From ad3d5caa39942aaf5711ac982d57fadffe655106 Mon Sep 17 00:00:00 2001 From: Sourabh Singh Date: Mon, 25 Nov 2024 11:02:22 +0000 Subject: [PATCH 01/49] cloudrun server test --- bin/lib/common.py | 19 ++- bin/run_test_server_c6n.py | 66 ++++++++ framework/infrastructure/c6n.py | 93 +++++++++++ framework/infrastructure/gcp/compute.py | 87 ++++++++++- .../infrastructure/gcp/network_services.py | 42 +++-- framework/infrastructure/traffic_director.py | 127 ++++++++++++++- .../test_app/runners/c6n/c6n_base_runner.py | 146 ++++++++++++++++++ .../runners/c6n/c6n_xds_server_runner.py | 106 +++++++++++++ framework/xds_c6n_testcase.py | 0 framework/xds_flags.py | 6 + framework/xds_k8s_flags.py | 12 ++ framework/xds_k8s_testcase.py | 102 ++++++++++++ .../client.cloudrun.deployment.yaml | 72 +++++++++ kubernetes-manifests/client.deployment.yaml | 2 +- tests/baseline_test_c6n_server.py | 65 ++++++++ tests/bootstrap_generator_test.py | 6 +- tests/url_map/affinity_test.py | 12 +- 17 files changed, 932 insertions(+), 31 deletions(-) create mode 100644 bin/run_test_server_c6n.py create mode 100644 framework/infrastructure/c6n.py create mode 100644 framework/test_app/runners/c6n/c6n_base_runner.py create mode 100644 framework/test_app/runners/c6n/c6n_xds_server_runner.py create mode 100644 framework/xds_c6n_testcase.py create mode 100644 kubernetes-manifests/client.cloudrun.deployment.yaml create mode 100644 tests/baseline_test_c6n_server.py diff --git a/bin/lib/common.py b/bin/lib/common.py index ebe6681e..6f755b5d 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,10 +22,12 @@ from framework import xds_flags from framework import xds_k8s_flags +from framework.infrastructure import c6n from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.test_app import client_app from framework.test_app import server_app +from framework.test_app.runners.c6n import c6n_xds_server_runner from framework.test_app.runners.k8s import gamma_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner @@ -45,6 +47,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner GammaServerRunner = gamma_server_runner.GammaServerRunner +CloudRunServerRunner=c6n_xds_server_runner.CloudRunServerRunner _XdsTestServer = server_app.XdsTestServer _XdsTestClient = client_app.XdsTestClient @@ -59,6 +62,10 @@ def gcp_api_manager(): return gcp.api.GcpApiManager() +@functools.cache +def c6n_api_manager(): + return c6n.CloudRunApiManager() + def td_attrs(): return dict( gcp_api_manager=gcp_api_manager(), @@ -117,7 +124,6 @@ def make_server_namespace( ) return k8s.KubernetesNamespace(k8s_api_manager(), namespace_name) - def make_server_runner( namespace: k8s.KubernetesNamespace, *, @@ -155,6 +161,17 @@ def make_server_runner( return server_runner(namespace, **runner_kwargs) +def make_c6n_server_runner() -> CloudRunServerRunner: + # CloudRunServerRunner arguments. + runner_kwargs = dict( + project = xds_flags.PROJECT.value, + service_name = xds_flags.SERVER_NAME.value, + image_name = xds_k8s_flags.SERVER_IMAGE.value, + network = xds_flags.NETWORK.value, + region = xds_flags.REGION.value, + ) + server_runner = CloudRunServerRunner + return server_runner(**runner_kwargs) def _ensure_atexit(signum, frame): """Needed to handle signals or atexit handler won't be called.""" diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_c6n.py new file mode 100644 index 00000000..91d4243b --- /dev/null +++ b/bin/run_test_server_c6n.py @@ -0,0 +1,66 @@ +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run test xds server. + +Typical usage examples: + + # Help. + ./run.sh ./bin/run_test_server_c6n.py --help + + # Run modes. + ./run.sh ./bin/run_test_server_c6n.py --mode=app_net + ./run.sh ./bin/run_test_server_c6n.py --mode=secure + + # Gamma run mode: uses HTTPRoute by default. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma + + # Gamma run mode: use GRPCRoute. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --gamma_route_kind=grpc + + # Running multipler server replicas. + ./run.sh ./bin/run_test_server_c6n.py --server_replica_count=3 + + # Cleanup: make sure to set the same mode used to create. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --cmd=cleanup +""" +import logging + +from absl import app +from absl import flags + +from bin.lib import common +from framework import xds_flags +from framework import xds_k8s_flags + +logger = logging.getLogger(__name__) + +flags.adopt_module_key_flags(xds_flags) +flags.adopt_module_key_flags(xds_k8s_flags) +flags.adopt_module_key_flags(common) + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + xds_flags.set_socket_default_timeout_from_flag() + + run_kwargs = dict() + server_runner = common.make_c6n_server_runner() + server_runner.run(**run_kwargs) + + +if __name__ == "__main__": + app.run(main) diff --git a/framework/infrastructure/c6n.py b/framework/infrastructure/c6n.py new file mode 100644 index 00000000..cd063cdb --- /dev/null +++ b/framework/infrastructure/c6n.py @@ -0,0 +1,93 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.cloud import run_v2 + +logger = logging.getLogger(__name__) + + +class CloudRunApiManager: + project: str + region: str + _parent: str + _client: run_v2.ServicesClient + _service: run_v2.Service + + def __init__(self, project: str, region: str): + if not project: + raise ValueError("Project ID cannot be empty or None.") + if not region: + raise ValueError("Region cannot be empty or None.") + + self.project = project + self.region = region + client_options = {"api_endpoint": f"{self.region}-run.googleapis.com"} + self._client = run_v2.ServicesClient(client_options=client_options) + self._parent = f"projects/{self.project}/locations/{self.region}" + self._service = None + + def deploy_service(self, service_name: str, image_name: str): + if not service_name: + raise ValueError("service_name cannot be empty or None") + if not image_name: + raise ValueError("image_name cannot be empty or None") + service_name = service_name[:49] + + service = run_v2.Service( + template=run_v2.RevisionTemplate( + containers=[ + run_v2.Container( + image=image_name, + ports=[ + run_v2.ContainerPort( + name="http1", + container_port=50051, + ), + ], + ) + ] + ), + ) + + request = run_v2.CreateServiceRequest( + parent=self._parent, service=service, service_id=service_name + ) + + try: + operation = self._client.create_service(request=request) + self._service = operation.result(timeout=300) + logger.info("Deployed service: %s", self._service.uri) + return self._service.uri + except Exception as e: + logger.exception("Error deploying service: %s", e) + raise + + def get_service_url(self): + if self._service is None: + raise RuntimeError("Cloud Run service not deployed yet.") + return self._service.uri + + def delete_service(self, service_name: str): + try: + request = run_v2.DeleteServiceRequest( + name=f"{self._parent}/services/{service_name}" + ) + operation = self._client.delete_service(request=request) + operation.result(timeout=300) + logger.info("Deleted service: %s", service_name) + except Exception as e: + logger.exception("Error deleting service: %s", e) + raise diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index d1f4f96c..643a81a9 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -158,7 +158,7 @@ def create_backend_service_traffic_director( body = { "name": name, "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director - "healthChecks": [health_check.url], + # "healthChecks": [health_check.url], "protocol": protocol.name, } # If add dualstack support is specified True, config the backend service @@ -210,8 +210,8 @@ def backend_service_patch_backends( backend_list = [ { "group": backend.url, - "balancingMode": "RATE", - "maxRatePerEndpoint": max_rate_per_endpoint, + # "balancingMode": "CONNECTION", + # "maxRatePerEndpoint": max_rate_per_endpoint, } for backend in backends ] @@ -558,6 +558,85 @@ def get_backend_service_backend_health(self, backend_service, backend): .execute() ) + def create_serverless_neg( + self, name: str, region: str, service_name: str, network: str + ): + """Creates a serverless NEG. + + Args: + name: The name of the NEG. + region: The region in which to create the NEG. + service_name: The name of the Cloud Run service. Format: "namespaces/{namespace}/services/{service}" + network: The network of the NEG. Format: "projects/{project}/global/networks/{network}" + + Returns: + The NEG selfLink URL + """ + name = name + "-neg" + neg_body = { + "name": name, + "networkEndpointType": "SERVERLESS", + "cloudRun": {"service": service_name}, + } + + try: + logger.info("Creating serverless NEG %s in %s", name, region) + operation = ( + self.api.regionNetworkEndpointGroups() + .insert(project=self.project, region=region, body=neg_body) + .execute() + ) + neg = self.get_serverless_network_endpoint_group(name, region) + print(neg) + return neg + + except Exception as e: + logger.exception("Error creating serverless NEG: %s", e) + raise + + def delete_serverless_neg(self, name: str, zone: str): + """Deletes a serverless NEG. + + Args: + name: The name of the NEG to delete. + zone: The zone of the NEG. + """ + try: + logger.info("Deleting serverless NEG %s in %s", name, zone) + operation = ( + self.api.networkEndpointGroups() + .delete( + project=self.project, zone=zone, networkEndpointGroup=name + ) + .execute() + ) + self._wait( + operation["name"], self._WAIT_FOR_OPERATION_SEC + ) # Wait for operation completion + + except googleapiclient.errors.HttpError as error: + if error.resp.status == 404: # NEG not found + logger.debug( + "NEG %s not found in zone %s. Skipping deletion.", + name, + zone, + ) + return + logger.exception("Error deleting serverless NEG: %s", error) + raise + except Exception as e: + logger.exception("Error deleting serverless NEG: %s", e) + raise + + def get_serverless_network_endpoint_group(self, name, region): + neg = ( + self.api.regionNetworkEndpointGroups() + .get(project=self.project, networkEndpointGroup=name, region=region) + .execute() + ) + # TODO(sergiitk): dataclass + return neg + def _get_resource( self, collection: discovery.Resource, **kwargs ) -> "GcpResource": @@ -651,6 +730,8 @@ def _execute( # pylint: disable=arguments-differ ) request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header request.add_response_callback(self._log_debug_header) + logger.info("Executing request: %s", request) + logger.info(request.to_json()) operation = request.execute(num_retries=self._GCP_API_RETRIES) logger.debug("Operation %s", operation) return self._wait(operation["name"], timeout_sec) diff --git a/framework/infrastructure/gcp/network_services.py b/framework/infrastructure/gcp/network_services.py index 86d4a97a..09132536 100644 --- a/framework/infrastructure/gcp/network_services.py +++ b/framework/infrastructure/gcp/network_services.py @@ -112,14 +112,19 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteMatch": return cls( - method=GrpcRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - GrpcRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), + method=( + GrpcRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None + ), + headers=( + tuple( + GrpcRoute.HeaderMatch.from_response(h) + for h in d["headers"] + ) + if "headers" in d + else () + ), ) @dataclasses.dataclass(frozen=True) @@ -224,14 +229,19 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteMatch": return cls( - method=HttpRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - HttpRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), + method=( + HttpRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None + ), + headers=( + tuple( + HttpRoute.HeaderMatch.from_response(h) + for h in d["headers"] + ) + if "headers" in d + else () + ), ) @dataclasses.dataclass(frozen=True) diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 0004ad9d..74b76bcb 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -14,7 +14,8 @@ import functools import logging import random -from typing import Any, Dict, Final, List, Optional +import re +from typing import Any, Dict, Final, List, Optional, Sequence import googleapiclient.errors from typing_extensions import TypeAlias @@ -263,6 +264,8 @@ def create_backend_service( ) self.backend_service = resource self.backend_service_protocol = protocol + logger.info("backend service protocal") + logger.info(self.backend_service_protocol) def load_backend_service(self): name = self.make_resource_name(self.BACKEND_SERVICE_NAME) @@ -1282,3 +1285,125 @@ def _get_certificate_provider(cls): "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, }, } + + +class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): + MESH_NAME = "grpc-mesh" + SERVER_TLS_POLICY_NAME = "server-tls-policy" + CLIENT_TLS_POLICY_NAME = "client-tls-policy" + AUTHZ_POLICY_NAME = "authz-policy" + ENDPOINT_POLICY = "endpoint-policy" + CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" + + netsec: _NetworkSecurityV1Beta1 + netsvc: _NetworkServicesV1Beta1 + + def __init__( + self, + gcp_api_manager: gcp.api.GcpApiManager, + project: str, + *, + resource_prefix: str, + resource_suffix: Optional[str] = None, + network: str = "default", + compute_api_version: str = "v1", + enable_dualstack: bool = False, + ): + super().__init__( + gcp_api_manager, + project, + resource_prefix=resource_prefix, + resource_suffix=resource_suffix, + network=network, + compute_api_version=compute_api_version, + enable_dualstack=enable_dualstack, + ) + + # API + self.netsvc = gcp.network_services.NetworkServicesV1( + gcp_api_manager, project + ) + + # Managed resources + # TODO(gnossen) PTAL at the pylint error + self.grpc_route: Optional[GrpcRoute] = None + self.http_route: Optional[HttpRoute] = None + self.mesh: Optional[Mesh] = None + + # Managed resources + self.server_tls_policy: Optional[ServerTlsPolicy] = None + self.client_tls_policy: Optional[ClientTlsPolicy] = None + self.authz_policy: Optional[AuthorizationPolicy] = None + self.endpoint_policy: Optional[EndpointPolicy] = None + + def backend_service_add_backends( + self, + backends: Sequence[str], + region: Optional[str] = None, + balancing_mode: str = "CONNECTION", + max_rate_per_endpoint: Optional[int] = None, + capacity_scaler: float = 1.0, + *, + circuit_breakers: Optional[dict[str, int]] = None, + ): + + new_backends = [] + for backend in backends: + new_backend = { + "group": backend, + "balancingMode": balancing_mode, + "maxRatePerEndpoint": max_rate_per_endpoint, + "capacityScaler": capacity_scaler, + } + + if circuit_breakers is not None: + new_backend["circuitBreakers"] = circuit_breakers + + new_backends.append(new_backend) + + # self.backends.update(new_backends) + backend_service = self.backend_service + + logging.info( + "Adding backends to Backend Service %s: %r", + backend_service.name, + new_backends, + ) + + self.compute.backend_service_patch_backends( + backend_service, + backends, + ) + + def create_grpc_route(self, src_host: str, src_port: int): + hostname = src_host + route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" + logger.info("Creating gRPC route: %s", route_name) + + try: + backend_service_url = self.backend_service.url.split("v1/")[-1] + # URL of the backend service + route_body = { + "name": route_name, + "hostnames": [hostname], + "rules": [ + { + "matches": [], # Empty matches means any rpc + "action": { + "destinations": [ + {"serviceName": backend_service_url} + ] + }, + } + ], + } + if hasattr(self, "mesh") and self.mesh: + route_body["meshes"] = self.mesh.url + + resource = self.netsvc.create_grpc_route(route_name, route_body) + self.grpc_route = self.netsvc.get_grpc_route(route_name) + logger.info("gRPC Route created successfully: %s", self.grpc_route) + + except Exception as e: # Catching generic exceptions for now + logger.exception("Error creating gRPC route: %s", e) + raise diff --git a/framework/test_app/runners/c6n/c6n_base_runner.py b/framework/test_app/runners/c6n/c6n_base_runner.py new file mode 100644 index 00000000..95bb2cc2 --- /dev/null +++ b/framework/test_app/runners/c6n/c6n_base_runner.py @@ -0,0 +1,146 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Common functionality for running xDS Test Client and Server on CloudRun. +""" +from abc import ABCMeta +import collections +import dataclasses +import datetime as dt +import logging +from typing import Optional + +import framework +from framework.infrastructure import c6n +from framework.test_app.runners import base_runner + +logger = logging.getLogger(__name__) + +_RunnerError = base_runner.RunnerError +_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml +_helper_datetime = framework.helpers.datetime +_datetime = dt.datetime +_timedelta = dt.timedelta + + +@dataclasses.dataclass(frozen=True) +class RunHistory: + revision_id: str + time_start_requested: _datetime + time_start_completed: Optional[_datetime] + time_stopped: _datetime + + +class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): + """Runs xDS Test Client/Server on Cloud Run.""" + + project: str + service_name: str + image_name: str + network: Optional[str] = None + tag: str = "latest" + region: Optional[str] = None + _current_revision: Optional[str] = None + gcp_project: Optional[str] = None + gcp_ui_url: Optional[str] = None + + run_history: collections.deque[RunHistory] + + time_start_requested: Optional[dt.datetime] = None + time_start_completed: Optional[dt.datetime] = None + time_stopped: Optional[dt.datetime] = None + + def __init__( + self, + project: str, + service_name: str, + image_name: str, + network: Optional[str] = None, + region: Optional[str] = None, + ) -> None: + super().__init__() + + self.project = project + self.service_name = service_name + self.image_name = image_name + self.network = network + self.region = region + self.current_revision = None + self.gcp_project = None + self.gcp_ui_url = None + self.run_history = collections.deque() + + # Persistent across many runs. + self.run_history = collections.deque() + + # Mutable state associated with each run. + self._reset_state() + + # Highlighter. + self._highlighter = _HighlighterYaml() + + self._initalize_cloudrun_api_manager() + + def _initalize_cloudrun_api_manager(self): + """Initializes the CloudRunApiManager.""" + self.cloudrun_api_manager = c6n.CloudRunApiManager( + project=self.project, region=self.region + ) + + def run(self, **kwargs): + if self.time_start_requested and not self.time_stopped: + if self.time_start_completed: + raise RuntimeError( + f"Service {self.service_name}: has already been started " + f"at {self.time_start_completed.isoformat()}" + ) + raise RuntimeError( + f"Service {self.service_name}: start has already been " + f"requested at {self.time_start_requested.isoformat()}" + ) + + self._reset_state() + self.time_start_requested = dt.datetime.now(tz=dt.timezone.utc) + self._current_revision = self.cloudrun_api_manager.deploy_service( + self.service_name, self.image_name + ) + + def _start_completed(self): + self.time_start_completed = dt.datetime.now(tz=dt.timezone.utc) + + def _stop(self): + self.time_stopped = dt.datetime.now(tz=dt.timezone.utc) + if self.time_start_requested: + run_history = RunHistory( + revision_id=self._current_revision, + time_start_requested=self.time_start_requested, + time_start_completed=self.time_start_completed, + time_stopped=self.time_stopped, + ) + self.run_history.append(run_history) + + @classmethod + def _get_workload_identity_member_name( + cls, project, namespace_name, service_account_name + ): + """ + Returns workload identity member name used to authenticate Kubernetes + service accounts. + + https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + """ + return ( + f"serviceAccount:{project}.svc.id.goog" + f"[{namespace_name}/{service_account_name}]" + ) diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/c6n/c6n_xds_server_runner.py new file mode 100644 index 00000000..5fece4ce --- /dev/null +++ b/framework/test_app/runners/c6n/c6n_xds_server_runner.py @@ -0,0 +1,106 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run xDS Test Client on Kubernetes. +""" +import dataclasses +import logging +from typing import List, Optional +import uuid + +from absl import flags +from absl import logging +from typing_extensions import override + +from framework import xds_flags +from framework import xds_k8s_flags +from framework.infrastructure import c6n +from framework.test_app.runners.c6n import c6n_base_runner +from framework.test_app.server_app import XdsTestServer + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass(frozen=True) +class CloudRunDeploymentArgs: + """Arguments for deploying a server to Cloud Run.""" + + env_vars: dict = dataclasses.field(default_factory=dict) + max_instances: int = 10 # Example: Maximum number of instances + min_instances: int = 0 # Example: Minimum number of instances + service_account_email: str = "" # Email address of the service account + timeout_seconds: int = 300 # Timeout for requests + revision_suffix: Optional[str] = None + + def as_dict(self): + return { + "env_vars": self.env_vars, + "max_instances": self.max_instances, + "min_instances": self.min_instances, + "service_account_email": self.service_account_email, + "timeout_seconds": self.timeout_seconds, + } + + +class CloudRunServerRunner(c6n_base_runner.CloudRunBaseRunner): + """Manages xDS Test Servers running on Cloud Run.""" + + def __init__( + self, + project: str, + service_name: str, + image_name: str, + network: str, + region: str, + ): + super().__init__( + project, + service_name, + image_name, + network=network, + region=region, + ) + # Mutable state associated with each run. + self._reset_state() + + @override + def _reset_state(self): + super()._reset_state() + self.service = None + self.pods_to_servers = {} + self.replica_count = 0 + + @override + def run(self, **kwargs) -> List[XdsTestServer]: + """Deploys and manages the xDS Test Server on Cloud Run.""" + logger.info(self.service_name) + logger.info(self.image_name) + deployed_service_url = self.cloudrun_api_manager.deploy_service( + service_name=self.service_name, + image_name=self.image_name, + ) + servers = [ + XdsTestServer( + ip="0.0.0.0", rpc_port=0, hostname=deployed_service_url + ) + ] + self.servers = servers # Add servers to the list + return servers + + def get_service_url(self): + return self.cloudrun_api_manager.get_service_url() + + @override + def cleanup(self, *, force=False): + return super().cleanup(force=force) diff --git a/framework/xds_c6n_testcase.py b/framework/xds_c6n_testcase.py new file mode 100644 index 00000000..e69de29b diff --git a/framework/xds_flags.py b/framework/xds_flags.py index 3960b27f..4656593d 100644 --- a/framework/xds_flags.py +++ b/framework/xds_flags.py @@ -196,6 +196,12 @@ help="Enable support for Dual Stack resources to the framework.", ) +REGION = flags.DEFINE_string( + "region", + default="us-central1", + help="The region for deployment", +) + def set_socket_default_timeout_from_flag() -> None: """A helper to configure default socket timeout from a flag. diff --git a/framework/xds_k8s_flags.py b/framework/xds_k8s_flags.py index 8070642b..ef729c37 100644 --- a/framework/xds_k8s_flags.py +++ b/framework/xds_k8s_flags.py @@ -33,6 +33,18 @@ help="Traffic Director gRPC Bootstrap Docker image", ) +CLOUDRUN_SERVER_IMAGE = flags.DEFINE_string( + "cloudrun_server_image", + default=None, + help="Cloudrun Server Docker image name", +) + +CLOUDRUN_CLIENT_IMAGE = flags.DEFINE_string( + "cloudrun_client_image", + default=None, + help="Cloudrun Client Docker image name", +) + # Test app SERVER_IMAGE = flags.DEFINE_string( "server_image", default=None, help="Server Docker image name" diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index c302af89..0a9688c4 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -39,11 +39,13 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director +from framework.infrastructure.gcp.compute import ComputeV1 from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing from framework.test_app import client_app from framework.test_app import server_app +from framework.test_app.runners.c6n import c6n_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner from framework.test_cases import base_testcase @@ -63,6 +65,8 @@ TrafficDirectorManager = traffic_director.TrafficDirectorManager TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager +CloudRunServerRunner = c6n_xds_server_runner.CloudRunServerRunner +TrafficDirectorCloudRunManager = traffic_director.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -198,6 +202,7 @@ def setUpClass(cls): cls.resource_suffix = xds_flags.RESOURCE_SUFFIX.value # Test server + cls.cloudrun_server_image = xds_k8s_flags.CLOUDRUN_SERVER_IMAGE.value cls.server_image = xds_k8s_flags.SERVER_IMAGE.value cls.server_name = xds_flags.SERVER_NAME.value cls.server_port = xds_flags.SERVER_PORT.value @@ -206,6 +211,7 @@ def setUpClass(cls): cls.server_xds_port = xds_flags.SERVER_XDS_PORT.value # Test client + cls.cloudrun_client_image = xds_k8s_flags.CLOUDRUN_CLIENT_IMAGE.value cls.client_image = xds_k8s_flags.CLIENT_IMAGE.value cls.client_name = xds_flags.CLIENT_NAME.value cls.client_port = xds_flags.CLIENT_PORT.value @@ -1380,3 +1386,99 @@ def debug_cert(cert): return "missing" sha1 = hashlib.sha1(cert) return f"sha1={sha1.hexdigest()}, len={len(cert)}" + + +class CloudRunXdsKubernetesTestCase(RegularXdsKubernetesTestCase): + server_runner: CloudRunServerRunner + td: TrafficDirectorCloudRunManager + region: str + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) + cls.compute_v1 = ComputeV1( + cls.gcp_api_manager, cls.project, version=cls.compute_api_version + ) + cls.region = xds_flags.REGION.value + + def initTrafficDirectorManager(self) -> TrafficDirectorManager: + return TrafficDirectorCloudRunManager( + self.gcp_api_manager, + project=self.project, + resource_prefix=self.resource_prefix, + resource_suffix=self.resource_suffix, + network=self.network, + compute_api_version=self.compute_api_version, + enable_dualstack=self.enable_dualstack, + ) + + def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: + return KubernetesClientRunner( + k8s.KubernetesNamespace( + self.k8s_api_manager, self.client_namespace + ), + deployment_name=self.client_name, + deployment_template="client.cloudrun.deployment.yaml", + image_name=self.cloudrun_client_image, + td_bootstrap_image=self.td_bootstrap_image, + gcp_project=self.project, + gcp_api_manager=self.gcp_api_manager, + gcp_service_account=self.gcp_service_account, + xds_server_uri=self.xds_server_uri, + network=self.network, + debug_use_port_forwarding=self.debug_use_port_forwarding, + enable_workload_identity=self.enable_workload_identity, + stats_port=self.client_port, + reuse_namespace=self.server_namespace == self.client_namespace, + **kwargs, + ) + + def setupServerBackends( + self, + *, + wait_for_healthy_status=True, + server_runner=None, + max_rate_per_endpoint: Optional[int] = None, + ): + if server_runner is None: + server_runner = self.server_runner + service_url = server_runner.get_service_url() + self.td.backend_service_add_backends([service_url]) + + def startTestServers( + self, server_runner=None, **kwargs + ) -> List[XdsTestServer]: + logger.info(self.cloudrun_server_image) + if server_runner is None: + self.server_runner = CloudRunServerRunner( + project=self.project, + service_name=self.server_namespace, + image_name=self.cloudrun_server_image, + network=self.network, + region=self.region, + ) + test_servers = self.server_runner.run() + for test_server in test_servers: + test_server.set_xds_address( + self.server_xds_host, self.server_xds_port + ) + return test_servers + + def startTestClient( + self, test_server: XdsTestServer, **kwargs + ) -> XdsTestClient: + return self._start_test_client(test_server.xds_uri, **kwargs) + + def backend_service_add_serverless_neg_backends(self): + logger.info("Creating serverless NEG") + neg = self.compute_v1.create_serverless_neg( + self.server_namespace, + self.region, + self.server_namespace, + "serverless", + ) + return neg + + def tearDown(self): + logger.info("----- TestMethod %s teardown -----", self.test_name) diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml new file mode 100644 index 00000000..87a28c95 --- /dev/null +++ b/kubernetes-manifests/client.cloudrun.deployment.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psm-grpc-client + labels: + app: psm-grpc-client +spec: + replicas: 1 + selector: + matchLabels: + app: psm-grpc-client + template: + metadata: + labels: + app: psm-grpc-client + spec: + containers: + - name: psm-grpc-client + image: docker.io/grpc/xds-example-cpp-client:v1.69.x + imagePullPolicy: Always + args: + - "--xds_creds" + - "--target=xds:///psm-grpc-server" + env: + - name: GRPC_TRACE + value: "http1" + - name: GRPC_XDS_BOOTSTRAP + value: "/tmp/grpc-xds/td-grpc-bootstrap.json" + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER + value: "true" + + - name: CSM_WORKLOAD_NAME + value: psm-grpc-client + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: psm-grpc-client + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) + volumeMounts: + - mountPath: /tmp/grpc-xds/ + name: grpc-td-conf + readOnly: true + initContainers: + - name: grpc-td-init + image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 + imagePullPolicy: Always + args: + - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" + - "--vpc-network-name=default" + - "--xds-server-uri=trafficdirector.googleapis.com:443" + - "--config-mesh=${config_mesh}" + - "--is-trusted-xds-server-experimental=true" + volumeMounts: + - mountPath: /tmp/bootstrap/ + name: grpc-td-conf + volumes: + - name: grpc-td-conf + emptyDir: + medium: Memory +... \ No newline at end of file diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 09a18e43..67ef56d2 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -36,7 +36,7 @@ spec: ## driver waiting for the container to start. failureThreshold: 1000 args: - - "--server=${server_target}" + - "--server=psm-grpc-server:8080" - "--stats_port=${stats_port}" - "--qps=${qps}" - "--rpc=${rpc}" diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_c6n_server.py new file mode 100644 index 00000000..466101c5 --- /dev/null +++ b/tests/baseline_test_c6n_server.py @@ -0,0 +1,65 @@ +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from absl.testing import absltest + +from framework import xds_k8s_testcase + +logger = logging.getLogger(__name__) + +_XdsTestServer = xds_k8s_testcase.XdsTestServer +_XdsTestClient = xds_k8s_testcase.XdsTestClient + + +class BaselineTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): + def test_traffic_director_grpc_setup(self): + with self.subTest("0_create_mesh"): + self.td.create_mesh() + + with self.subTest("1_start_cloudrun_test_server"): + test_server: _XdsTestServer = self.startTestServers()[0] + + with self.subTest("2_create_serverless_neg"): + neg = self.backend_service_add_serverless_neg_backends() + + with self.subTest("3_create_backend_service"): + self.td.create_backend_service( + protocol=self.compute_v1.BackendServiceProtocol.HTTP2 + ) + + with self.subTest("4_add_server_backends_to_backend_service"): + neg_resource = self.compute_v1.GcpResource( + name=neg["name"], + url=neg["selfLink"], + ) + self.td.backend_service_add_backends([neg_resource], self.region) + + with self.subTest("5_create_grpc_route"): + self.td.create_grpc_route(self.server_xds_host, self.server_port) + + with self.subTest("7_start_test_client"): + test_client: _XdsTestClient = self.startTestClient( + test_server, config_mesh=self.td.mesh.name + ) + + with self.subTest("8_test_client_xds_config_exists"): + self.assertXdsConfigExists(test_client) + + with self.subTest("9_test_server_received_rpcs_from_test_client"): + self.assertSuccessfulRpcs(test_client) + + +if __name__ == "__main__": + absltest.main(failfast=True) diff --git a/tests/bootstrap_generator_test.py b/tests/bootstrap_generator_test.py index 70fe4c4b..471dde8f 100644 --- a/tests/bootstrap_generator_test.py +++ b/tests/bootstrap_generator_test.py @@ -38,9 +38,9 @@ # Constants GCR_PROD: Final[str] = "gcr.io/trafficdirector-prod/td-grpc-bootstrap" -GCR_TESTING: Final[ - str -] = "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" +GCR_TESTING: Final[str] = ( + "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" +) # Returns a list of bootstrap generator versions to be tested along with their diff --git a/tests/url_map/affinity_test.py b/tests/url_map/affinity_test.py index 2bd93405..e3e2102c 100644 --- a/tests/url_map/affinity_test.py +++ b/tests/url_map/affinity_test.py @@ -101,9 +101,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() + path_matcher["defaultService"] = ( + GcpResourceManager().affinity_backend_service() + ) return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -176,9 +176,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() + path_matcher["defaultService"] = ( + GcpResourceManager().affinity_backend_service() + ) return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): From 685995712a21a0bce47a24411c62d2556730fbe2 Mon Sep 17 00:00:00 2001 From: Sourabh Singh Date: Mon, 25 Nov 2024 11:02:22 +0000 Subject: [PATCH 02/49] cloudrun server test --- bin/lib/common.py | 19 ++- bin/run_test_server_c6n.py | 66 ++++++++ framework/infrastructure/c6n.py | 93 +++++++++++ framework/infrastructure/gcp/compute.py | 87 ++++++++++- .../infrastructure/gcp/network_services.py | 42 +++-- framework/infrastructure/traffic_director.py | 127 ++++++++++++++- .../test_app/runners/c6n/c6n_base_runner.py | 146 ++++++++++++++++++ .../runners/c6n/c6n_xds_server_runner.py | 106 +++++++++++++ framework/xds_c6n_testcase.py | 0 framework/xds_flags.py | 6 + framework/xds_k8s_flags.py | 12 ++ framework/xds_k8s_testcase.py | 102 ++++++++++++ .../client.cloudrun.deployment.yaml | 72 +++++++++ kubernetes-manifests/client.deployment.yaml | 2 +- tests/baseline_test_c6n_server.py | 65 ++++++++ tests/bootstrap_generator_test.py | 6 +- tests/url_map/affinity_test.py | 12 +- 17 files changed, 932 insertions(+), 31 deletions(-) create mode 100644 bin/run_test_server_c6n.py create mode 100644 framework/infrastructure/c6n.py create mode 100644 framework/test_app/runners/c6n/c6n_base_runner.py create mode 100644 framework/test_app/runners/c6n/c6n_xds_server_runner.py create mode 100644 framework/xds_c6n_testcase.py create mode 100644 kubernetes-manifests/client.cloudrun.deployment.yaml create mode 100644 tests/baseline_test_c6n_server.py diff --git a/bin/lib/common.py b/bin/lib/common.py index ebe6681e..6f755b5d 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,10 +22,12 @@ from framework import xds_flags from framework import xds_k8s_flags +from framework.infrastructure import c6n from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.test_app import client_app from framework.test_app import server_app +from framework.test_app.runners.c6n import c6n_xds_server_runner from framework.test_app.runners.k8s import gamma_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner @@ -45,6 +47,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner GammaServerRunner = gamma_server_runner.GammaServerRunner +CloudRunServerRunner=c6n_xds_server_runner.CloudRunServerRunner _XdsTestServer = server_app.XdsTestServer _XdsTestClient = client_app.XdsTestClient @@ -59,6 +62,10 @@ def gcp_api_manager(): return gcp.api.GcpApiManager() +@functools.cache +def c6n_api_manager(): + return c6n.CloudRunApiManager() + def td_attrs(): return dict( gcp_api_manager=gcp_api_manager(), @@ -117,7 +124,6 @@ def make_server_namespace( ) return k8s.KubernetesNamespace(k8s_api_manager(), namespace_name) - def make_server_runner( namespace: k8s.KubernetesNamespace, *, @@ -155,6 +161,17 @@ def make_server_runner( return server_runner(namespace, **runner_kwargs) +def make_c6n_server_runner() -> CloudRunServerRunner: + # CloudRunServerRunner arguments. + runner_kwargs = dict( + project = xds_flags.PROJECT.value, + service_name = xds_flags.SERVER_NAME.value, + image_name = xds_k8s_flags.SERVER_IMAGE.value, + network = xds_flags.NETWORK.value, + region = xds_flags.REGION.value, + ) + server_runner = CloudRunServerRunner + return server_runner(**runner_kwargs) def _ensure_atexit(signum, frame): """Needed to handle signals or atexit handler won't be called.""" diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_c6n.py new file mode 100644 index 00000000..91d4243b --- /dev/null +++ b/bin/run_test_server_c6n.py @@ -0,0 +1,66 @@ +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run test xds server. + +Typical usage examples: + + # Help. + ./run.sh ./bin/run_test_server_c6n.py --help + + # Run modes. + ./run.sh ./bin/run_test_server_c6n.py --mode=app_net + ./run.sh ./bin/run_test_server_c6n.py --mode=secure + + # Gamma run mode: uses HTTPRoute by default. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma + + # Gamma run mode: use GRPCRoute. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --gamma_route_kind=grpc + + # Running multipler server replicas. + ./run.sh ./bin/run_test_server_c6n.py --server_replica_count=3 + + # Cleanup: make sure to set the same mode used to create. + ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --cmd=cleanup +""" +import logging + +from absl import app +from absl import flags + +from bin.lib import common +from framework import xds_flags +from framework import xds_k8s_flags + +logger = logging.getLogger(__name__) + +flags.adopt_module_key_flags(xds_flags) +flags.adopt_module_key_flags(xds_k8s_flags) +flags.adopt_module_key_flags(common) + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + xds_flags.set_socket_default_timeout_from_flag() + + run_kwargs = dict() + server_runner = common.make_c6n_server_runner() + server_runner.run(**run_kwargs) + + +if __name__ == "__main__": + app.run(main) diff --git a/framework/infrastructure/c6n.py b/framework/infrastructure/c6n.py new file mode 100644 index 00000000..cd063cdb --- /dev/null +++ b/framework/infrastructure/c6n.py @@ -0,0 +1,93 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.cloud import run_v2 + +logger = logging.getLogger(__name__) + + +class CloudRunApiManager: + project: str + region: str + _parent: str + _client: run_v2.ServicesClient + _service: run_v2.Service + + def __init__(self, project: str, region: str): + if not project: + raise ValueError("Project ID cannot be empty or None.") + if not region: + raise ValueError("Region cannot be empty or None.") + + self.project = project + self.region = region + client_options = {"api_endpoint": f"{self.region}-run.googleapis.com"} + self._client = run_v2.ServicesClient(client_options=client_options) + self._parent = f"projects/{self.project}/locations/{self.region}" + self._service = None + + def deploy_service(self, service_name: str, image_name: str): + if not service_name: + raise ValueError("service_name cannot be empty or None") + if not image_name: + raise ValueError("image_name cannot be empty or None") + service_name = service_name[:49] + + service = run_v2.Service( + template=run_v2.RevisionTemplate( + containers=[ + run_v2.Container( + image=image_name, + ports=[ + run_v2.ContainerPort( + name="http1", + container_port=50051, + ), + ], + ) + ] + ), + ) + + request = run_v2.CreateServiceRequest( + parent=self._parent, service=service, service_id=service_name + ) + + try: + operation = self._client.create_service(request=request) + self._service = operation.result(timeout=300) + logger.info("Deployed service: %s", self._service.uri) + return self._service.uri + except Exception as e: + logger.exception("Error deploying service: %s", e) + raise + + def get_service_url(self): + if self._service is None: + raise RuntimeError("Cloud Run service not deployed yet.") + return self._service.uri + + def delete_service(self, service_name: str): + try: + request = run_v2.DeleteServiceRequest( + name=f"{self._parent}/services/{service_name}" + ) + operation = self._client.delete_service(request=request) + operation.result(timeout=300) + logger.info("Deleted service: %s", service_name) + except Exception as e: + logger.exception("Error deleting service: %s", e) + raise diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index d1f4f96c..643a81a9 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -158,7 +158,7 @@ def create_backend_service_traffic_director( body = { "name": name, "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director - "healthChecks": [health_check.url], + # "healthChecks": [health_check.url], "protocol": protocol.name, } # If add dualstack support is specified True, config the backend service @@ -210,8 +210,8 @@ def backend_service_patch_backends( backend_list = [ { "group": backend.url, - "balancingMode": "RATE", - "maxRatePerEndpoint": max_rate_per_endpoint, + # "balancingMode": "CONNECTION", + # "maxRatePerEndpoint": max_rate_per_endpoint, } for backend in backends ] @@ -558,6 +558,85 @@ def get_backend_service_backend_health(self, backend_service, backend): .execute() ) + def create_serverless_neg( + self, name: str, region: str, service_name: str, network: str + ): + """Creates a serverless NEG. + + Args: + name: The name of the NEG. + region: The region in which to create the NEG. + service_name: The name of the Cloud Run service. Format: "namespaces/{namespace}/services/{service}" + network: The network of the NEG. Format: "projects/{project}/global/networks/{network}" + + Returns: + The NEG selfLink URL + """ + name = name + "-neg" + neg_body = { + "name": name, + "networkEndpointType": "SERVERLESS", + "cloudRun": {"service": service_name}, + } + + try: + logger.info("Creating serverless NEG %s in %s", name, region) + operation = ( + self.api.regionNetworkEndpointGroups() + .insert(project=self.project, region=region, body=neg_body) + .execute() + ) + neg = self.get_serverless_network_endpoint_group(name, region) + print(neg) + return neg + + except Exception as e: + logger.exception("Error creating serverless NEG: %s", e) + raise + + def delete_serverless_neg(self, name: str, zone: str): + """Deletes a serverless NEG. + + Args: + name: The name of the NEG to delete. + zone: The zone of the NEG. + """ + try: + logger.info("Deleting serverless NEG %s in %s", name, zone) + operation = ( + self.api.networkEndpointGroups() + .delete( + project=self.project, zone=zone, networkEndpointGroup=name + ) + .execute() + ) + self._wait( + operation["name"], self._WAIT_FOR_OPERATION_SEC + ) # Wait for operation completion + + except googleapiclient.errors.HttpError as error: + if error.resp.status == 404: # NEG not found + logger.debug( + "NEG %s not found in zone %s. Skipping deletion.", + name, + zone, + ) + return + logger.exception("Error deleting serverless NEG: %s", error) + raise + except Exception as e: + logger.exception("Error deleting serverless NEG: %s", e) + raise + + def get_serverless_network_endpoint_group(self, name, region): + neg = ( + self.api.regionNetworkEndpointGroups() + .get(project=self.project, networkEndpointGroup=name, region=region) + .execute() + ) + # TODO(sergiitk): dataclass + return neg + def _get_resource( self, collection: discovery.Resource, **kwargs ) -> "GcpResource": @@ -651,6 +730,8 @@ def _execute( # pylint: disable=arguments-differ ) request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header request.add_response_callback(self._log_debug_header) + logger.info("Executing request: %s", request) + logger.info(request.to_json()) operation = request.execute(num_retries=self._GCP_API_RETRIES) logger.debug("Operation %s", operation) return self._wait(operation["name"], timeout_sec) diff --git a/framework/infrastructure/gcp/network_services.py b/framework/infrastructure/gcp/network_services.py index 86d4a97a..09132536 100644 --- a/framework/infrastructure/gcp/network_services.py +++ b/framework/infrastructure/gcp/network_services.py @@ -112,14 +112,19 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteMatch": return cls( - method=GrpcRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - GrpcRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), + method=( + GrpcRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None + ), + headers=( + tuple( + GrpcRoute.HeaderMatch.from_response(h) + for h in d["headers"] + ) + if "headers" in d + else () + ), ) @dataclasses.dataclass(frozen=True) @@ -224,14 +229,19 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteMatch": return cls( - method=HttpRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - HttpRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), + method=( + HttpRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None + ), + headers=( + tuple( + HttpRoute.HeaderMatch.from_response(h) + for h in d["headers"] + ) + if "headers" in d + else () + ), ) @dataclasses.dataclass(frozen=True) diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 0004ad9d..74b76bcb 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -14,7 +14,8 @@ import functools import logging import random -from typing import Any, Dict, Final, List, Optional +import re +from typing import Any, Dict, Final, List, Optional, Sequence import googleapiclient.errors from typing_extensions import TypeAlias @@ -263,6 +264,8 @@ def create_backend_service( ) self.backend_service = resource self.backend_service_protocol = protocol + logger.info("backend service protocal") + logger.info(self.backend_service_protocol) def load_backend_service(self): name = self.make_resource_name(self.BACKEND_SERVICE_NAME) @@ -1282,3 +1285,125 @@ def _get_certificate_provider(cls): "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, }, } + + +class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): + MESH_NAME = "grpc-mesh" + SERVER_TLS_POLICY_NAME = "server-tls-policy" + CLIENT_TLS_POLICY_NAME = "client-tls-policy" + AUTHZ_POLICY_NAME = "authz-policy" + ENDPOINT_POLICY = "endpoint-policy" + CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" + + netsec: _NetworkSecurityV1Beta1 + netsvc: _NetworkServicesV1Beta1 + + def __init__( + self, + gcp_api_manager: gcp.api.GcpApiManager, + project: str, + *, + resource_prefix: str, + resource_suffix: Optional[str] = None, + network: str = "default", + compute_api_version: str = "v1", + enable_dualstack: bool = False, + ): + super().__init__( + gcp_api_manager, + project, + resource_prefix=resource_prefix, + resource_suffix=resource_suffix, + network=network, + compute_api_version=compute_api_version, + enable_dualstack=enable_dualstack, + ) + + # API + self.netsvc = gcp.network_services.NetworkServicesV1( + gcp_api_manager, project + ) + + # Managed resources + # TODO(gnossen) PTAL at the pylint error + self.grpc_route: Optional[GrpcRoute] = None + self.http_route: Optional[HttpRoute] = None + self.mesh: Optional[Mesh] = None + + # Managed resources + self.server_tls_policy: Optional[ServerTlsPolicy] = None + self.client_tls_policy: Optional[ClientTlsPolicy] = None + self.authz_policy: Optional[AuthorizationPolicy] = None + self.endpoint_policy: Optional[EndpointPolicy] = None + + def backend_service_add_backends( + self, + backends: Sequence[str], + region: Optional[str] = None, + balancing_mode: str = "CONNECTION", + max_rate_per_endpoint: Optional[int] = None, + capacity_scaler: float = 1.0, + *, + circuit_breakers: Optional[dict[str, int]] = None, + ): + + new_backends = [] + for backend in backends: + new_backend = { + "group": backend, + "balancingMode": balancing_mode, + "maxRatePerEndpoint": max_rate_per_endpoint, + "capacityScaler": capacity_scaler, + } + + if circuit_breakers is not None: + new_backend["circuitBreakers"] = circuit_breakers + + new_backends.append(new_backend) + + # self.backends.update(new_backends) + backend_service = self.backend_service + + logging.info( + "Adding backends to Backend Service %s: %r", + backend_service.name, + new_backends, + ) + + self.compute.backend_service_patch_backends( + backend_service, + backends, + ) + + def create_grpc_route(self, src_host: str, src_port: int): + hostname = src_host + route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" + logger.info("Creating gRPC route: %s", route_name) + + try: + backend_service_url = self.backend_service.url.split("v1/")[-1] + # URL of the backend service + route_body = { + "name": route_name, + "hostnames": [hostname], + "rules": [ + { + "matches": [], # Empty matches means any rpc + "action": { + "destinations": [ + {"serviceName": backend_service_url} + ] + }, + } + ], + } + if hasattr(self, "mesh") and self.mesh: + route_body["meshes"] = self.mesh.url + + resource = self.netsvc.create_grpc_route(route_name, route_body) + self.grpc_route = self.netsvc.get_grpc_route(route_name) + logger.info("gRPC Route created successfully: %s", self.grpc_route) + + except Exception as e: # Catching generic exceptions for now + logger.exception("Error creating gRPC route: %s", e) + raise diff --git a/framework/test_app/runners/c6n/c6n_base_runner.py b/framework/test_app/runners/c6n/c6n_base_runner.py new file mode 100644 index 00000000..95bb2cc2 --- /dev/null +++ b/framework/test_app/runners/c6n/c6n_base_runner.py @@ -0,0 +1,146 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Common functionality for running xDS Test Client and Server on CloudRun. +""" +from abc import ABCMeta +import collections +import dataclasses +import datetime as dt +import logging +from typing import Optional + +import framework +from framework.infrastructure import c6n +from framework.test_app.runners import base_runner + +logger = logging.getLogger(__name__) + +_RunnerError = base_runner.RunnerError +_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml +_helper_datetime = framework.helpers.datetime +_datetime = dt.datetime +_timedelta = dt.timedelta + + +@dataclasses.dataclass(frozen=True) +class RunHistory: + revision_id: str + time_start_requested: _datetime + time_start_completed: Optional[_datetime] + time_stopped: _datetime + + +class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): + """Runs xDS Test Client/Server on Cloud Run.""" + + project: str + service_name: str + image_name: str + network: Optional[str] = None + tag: str = "latest" + region: Optional[str] = None + _current_revision: Optional[str] = None + gcp_project: Optional[str] = None + gcp_ui_url: Optional[str] = None + + run_history: collections.deque[RunHistory] + + time_start_requested: Optional[dt.datetime] = None + time_start_completed: Optional[dt.datetime] = None + time_stopped: Optional[dt.datetime] = None + + def __init__( + self, + project: str, + service_name: str, + image_name: str, + network: Optional[str] = None, + region: Optional[str] = None, + ) -> None: + super().__init__() + + self.project = project + self.service_name = service_name + self.image_name = image_name + self.network = network + self.region = region + self.current_revision = None + self.gcp_project = None + self.gcp_ui_url = None + self.run_history = collections.deque() + + # Persistent across many runs. + self.run_history = collections.deque() + + # Mutable state associated with each run. + self._reset_state() + + # Highlighter. + self._highlighter = _HighlighterYaml() + + self._initalize_cloudrun_api_manager() + + def _initalize_cloudrun_api_manager(self): + """Initializes the CloudRunApiManager.""" + self.cloudrun_api_manager = c6n.CloudRunApiManager( + project=self.project, region=self.region + ) + + def run(self, **kwargs): + if self.time_start_requested and not self.time_stopped: + if self.time_start_completed: + raise RuntimeError( + f"Service {self.service_name}: has already been started " + f"at {self.time_start_completed.isoformat()}" + ) + raise RuntimeError( + f"Service {self.service_name}: start has already been " + f"requested at {self.time_start_requested.isoformat()}" + ) + + self._reset_state() + self.time_start_requested = dt.datetime.now(tz=dt.timezone.utc) + self._current_revision = self.cloudrun_api_manager.deploy_service( + self.service_name, self.image_name + ) + + def _start_completed(self): + self.time_start_completed = dt.datetime.now(tz=dt.timezone.utc) + + def _stop(self): + self.time_stopped = dt.datetime.now(tz=dt.timezone.utc) + if self.time_start_requested: + run_history = RunHistory( + revision_id=self._current_revision, + time_start_requested=self.time_start_requested, + time_start_completed=self.time_start_completed, + time_stopped=self.time_stopped, + ) + self.run_history.append(run_history) + + @classmethod + def _get_workload_identity_member_name( + cls, project, namespace_name, service_account_name + ): + """ + Returns workload identity member name used to authenticate Kubernetes + service accounts. + + https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + """ + return ( + f"serviceAccount:{project}.svc.id.goog" + f"[{namespace_name}/{service_account_name}]" + ) diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/c6n/c6n_xds_server_runner.py new file mode 100644 index 00000000..5fece4ce --- /dev/null +++ b/framework/test_app/runners/c6n/c6n_xds_server_runner.py @@ -0,0 +1,106 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run xDS Test Client on Kubernetes. +""" +import dataclasses +import logging +from typing import List, Optional +import uuid + +from absl import flags +from absl import logging +from typing_extensions import override + +from framework import xds_flags +from framework import xds_k8s_flags +from framework.infrastructure import c6n +from framework.test_app.runners.c6n import c6n_base_runner +from framework.test_app.server_app import XdsTestServer + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass(frozen=True) +class CloudRunDeploymentArgs: + """Arguments for deploying a server to Cloud Run.""" + + env_vars: dict = dataclasses.field(default_factory=dict) + max_instances: int = 10 # Example: Maximum number of instances + min_instances: int = 0 # Example: Minimum number of instances + service_account_email: str = "" # Email address of the service account + timeout_seconds: int = 300 # Timeout for requests + revision_suffix: Optional[str] = None + + def as_dict(self): + return { + "env_vars": self.env_vars, + "max_instances": self.max_instances, + "min_instances": self.min_instances, + "service_account_email": self.service_account_email, + "timeout_seconds": self.timeout_seconds, + } + + +class CloudRunServerRunner(c6n_base_runner.CloudRunBaseRunner): + """Manages xDS Test Servers running on Cloud Run.""" + + def __init__( + self, + project: str, + service_name: str, + image_name: str, + network: str, + region: str, + ): + super().__init__( + project, + service_name, + image_name, + network=network, + region=region, + ) + # Mutable state associated with each run. + self._reset_state() + + @override + def _reset_state(self): + super()._reset_state() + self.service = None + self.pods_to_servers = {} + self.replica_count = 0 + + @override + def run(self, **kwargs) -> List[XdsTestServer]: + """Deploys and manages the xDS Test Server on Cloud Run.""" + logger.info(self.service_name) + logger.info(self.image_name) + deployed_service_url = self.cloudrun_api_manager.deploy_service( + service_name=self.service_name, + image_name=self.image_name, + ) + servers = [ + XdsTestServer( + ip="0.0.0.0", rpc_port=0, hostname=deployed_service_url + ) + ] + self.servers = servers # Add servers to the list + return servers + + def get_service_url(self): + return self.cloudrun_api_manager.get_service_url() + + @override + def cleanup(self, *, force=False): + return super().cleanup(force=force) diff --git a/framework/xds_c6n_testcase.py b/framework/xds_c6n_testcase.py new file mode 100644 index 00000000..e69de29b diff --git a/framework/xds_flags.py b/framework/xds_flags.py index 45e606f5..f601f6e8 100644 --- a/framework/xds_flags.py +++ b/framework/xds_flags.py @@ -199,6 +199,12 @@ help="Enable support for Dual Stack resources to the framework.", ) +REGION = flags.DEFINE_string( + "region", + default="us-central1", + help="The region for deployment", +) + def set_socket_default_timeout_from_flag() -> None: """A helper to configure default socket timeout from a flag. diff --git a/framework/xds_k8s_flags.py b/framework/xds_k8s_flags.py index 8070642b..ef729c37 100644 --- a/framework/xds_k8s_flags.py +++ b/framework/xds_k8s_flags.py @@ -33,6 +33,18 @@ help="Traffic Director gRPC Bootstrap Docker image", ) +CLOUDRUN_SERVER_IMAGE = flags.DEFINE_string( + "cloudrun_server_image", + default=None, + help="Cloudrun Server Docker image name", +) + +CLOUDRUN_CLIENT_IMAGE = flags.DEFINE_string( + "cloudrun_client_image", + default=None, + help="Cloudrun Client Docker image name", +) + # Test app SERVER_IMAGE = flags.DEFINE_string( "server_image", default=None, help="Server Docker image name" diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 1c602e7d..ccae817f 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -40,11 +40,13 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director +from framework.infrastructure.gcp.compute import ComputeV1 from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing from framework.test_app import client_app from framework.test_app import server_app +from framework.test_app.runners.c6n import c6n_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner from framework.test_cases import base_testcase @@ -64,6 +66,8 @@ TrafficDirectorManager = traffic_director.TrafficDirectorManager TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager +CloudRunServerRunner = c6n_xds_server_runner.CloudRunServerRunner +TrafficDirectorCloudRunManager = traffic_director.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -213,6 +217,7 @@ def setUpClass(cls): cls.resource_suffix = xds_flags.RESOURCE_SUFFIX.value # Test server + cls.cloudrun_server_image = xds_k8s_flags.CLOUDRUN_SERVER_IMAGE.value cls.server_image = xds_k8s_flags.SERVER_IMAGE.value cls.server_name = xds_flags.SERVER_NAME.value cls.server_port = xds_flags.SERVER_PORT.value @@ -221,6 +226,7 @@ def setUpClass(cls): cls.server_xds_port = xds_flags.SERVER_XDS_PORT.value # Test client + cls.cloudrun_client_image = xds_k8s_flags.CLOUDRUN_CLIENT_IMAGE.value cls.client_image = xds_k8s_flags.CLIENT_IMAGE.value cls.client_name = xds_flags.CLIENT_NAME.value cls.client_port = xds_flags.CLIENT_PORT.value @@ -1395,3 +1401,99 @@ def debug_cert(cert): return "missing" sha1 = hashlib.sha1(cert) return f"sha1={sha1.hexdigest()}, len={len(cert)}" + + +class CloudRunXdsKubernetesTestCase(RegularXdsKubernetesTestCase): + server_runner: CloudRunServerRunner + td: TrafficDirectorCloudRunManager + region: str + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) + cls.compute_v1 = ComputeV1( + cls.gcp_api_manager, cls.project, version=cls.compute_api_version + ) + cls.region = xds_flags.REGION.value + + def initTrafficDirectorManager(self) -> TrafficDirectorManager: + return TrafficDirectorCloudRunManager( + self.gcp_api_manager, + project=self.project, + resource_prefix=self.resource_prefix, + resource_suffix=self.resource_suffix, + network=self.network, + compute_api_version=self.compute_api_version, + enable_dualstack=self.enable_dualstack, + ) + + def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: + return KubernetesClientRunner( + k8s.KubernetesNamespace( + self.k8s_api_manager, self.client_namespace + ), + deployment_name=self.client_name, + deployment_template="client.cloudrun.deployment.yaml", + image_name=self.cloudrun_client_image, + td_bootstrap_image=self.td_bootstrap_image, + gcp_project=self.project, + gcp_api_manager=self.gcp_api_manager, + gcp_service_account=self.gcp_service_account, + xds_server_uri=self.xds_server_uri, + network=self.network, + debug_use_port_forwarding=self.debug_use_port_forwarding, + enable_workload_identity=self.enable_workload_identity, + stats_port=self.client_port, + reuse_namespace=self.server_namespace == self.client_namespace, + **kwargs, + ) + + def setupServerBackends( + self, + *, + wait_for_healthy_status=True, + server_runner=None, + max_rate_per_endpoint: Optional[int] = None, + ): + if server_runner is None: + server_runner = self.server_runner + service_url = server_runner.get_service_url() + self.td.backend_service_add_backends([service_url]) + + def startTestServers( + self, server_runner=None, **kwargs + ) -> List[XdsTestServer]: + logger.info(self.cloudrun_server_image) + if server_runner is None: + self.server_runner = CloudRunServerRunner( + project=self.project, + service_name=self.server_namespace, + image_name=self.cloudrun_server_image, + network=self.network, + region=self.region, + ) + test_servers = self.server_runner.run() + for test_server in test_servers: + test_server.set_xds_address( + self.server_xds_host, self.server_xds_port + ) + return test_servers + + def startTestClient( + self, test_server: XdsTestServer, **kwargs + ) -> XdsTestClient: + return self._start_test_client(test_server.xds_uri, **kwargs) + + def backend_service_add_serverless_neg_backends(self): + logger.info("Creating serverless NEG") + neg = self.compute_v1.create_serverless_neg( + self.server_namespace, + self.region, + self.server_namespace, + "serverless", + ) + return neg + + def tearDown(self): + logger.info("----- TestMethod %s teardown -----", self.test_name) diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml new file mode 100644 index 00000000..87a28c95 --- /dev/null +++ b/kubernetes-manifests/client.cloudrun.deployment.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psm-grpc-client + labels: + app: psm-grpc-client +spec: + replicas: 1 + selector: + matchLabels: + app: psm-grpc-client + template: + metadata: + labels: + app: psm-grpc-client + spec: + containers: + - name: psm-grpc-client + image: docker.io/grpc/xds-example-cpp-client:v1.69.x + imagePullPolicy: Always + args: + - "--xds_creds" + - "--target=xds:///psm-grpc-server" + env: + - name: GRPC_TRACE + value: "http1" + - name: GRPC_XDS_BOOTSTRAP + value: "/tmp/grpc-xds/td-grpc-bootstrap.json" + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER + value: "true" + + - name: CSM_WORKLOAD_NAME + value: psm-grpc-client + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: psm-grpc-client + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) + volumeMounts: + - mountPath: /tmp/grpc-xds/ + name: grpc-td-conf + readOnly: true + initContainers: + - name: grpc-td-init + image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 + imagePullPolicy: Always + args: + - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" + - "--vpc-network-name=default" + - "--xds-server-uri=trafficdirector.googleapis.com:443" + - "--config-mesh=${config_mesh}" + - "--is-trusted-xds-server-experimental=true" + volumeMounts: + - mountPath: /tmp/bootstrap/ + name: grpc-td-conf + volumes: + - name: grpc-td-conf + emptyDir: + medium: Memory +... \ No newline at end of file diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 09a18e43..67ef56d2 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -36,7 +36,7 @@ spec: ## driver waiting for the container to start. failureThreshold: 1000 args: - - "--server=${server_target}" + - "--server=psm-grpc-server:8080" - "--stats_port=${stats_port}" - "--qps=${qps}" - "--rpc=${rpc}" diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_c6n_server.py new file mode 100644 index 00000000..466101c5 --- /dev/null +++ b/tests/baseline_test_c6n_server.py @@ -0,0 +1,65 @@ +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from absl.testing import absltest + +from framework import xds_k8s_testcase + +logger = logging.getLogger(__name__) + +_XdsTestServer = xds_k8s_testcase.XdsTestServer +_XdsTestClient = xds_k8s_testcase.XdsTestClient + + +class BaselineTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): + def test_traffic_director_grpc_setup(self): + with self.subTest("0_create_mesh"): + self.td.create_mesh() + + with self.subTest("1_start_cloudrun_test_server"): + test_server: _XdsTestServer = self.startTestServers()[0] + + with self.subTest("2_create_serverless_neg"): + neg = self.backend_service_add_serverless_neg_backends() + + with self.subTest("3_create_backend_service"): + self.td.create_backend_service( + protocol=self.compute_v1.BackendServiceProtocol.HTTP2 + ) + + with self.subTest("4_add_server_backends_to_backend_service"): + neg_resource = self.compute_v1.GcpResource( + name=neg["name"], + url=neg["selfLink"], + ) + self.td.backend_service_add_backends([neg_resource], self.region) + + with self.subTest("5_create_grpc_route"): + self.td.create_grpc_route(self.server_xds_host, self.server_port) + + with self.subTest("7_start_test_client"): + test_client: _XdsTestClient = self.startTestClient( + test_server, config_mesh=self.td.mesh.name + ) + + with self.subTest("8_test_client_xds_config_exists"): + self.assertXdsConfigExists(test_client) + + with self.subTest("9_test_server_received_rpcs_from_test_client"): + self.assertSuccessfulRpcs(test_client) + + +if __name__ == "__main__": + absltest.main(failfast=True) diff --git a/tests/bootstrap_generator_test.py b/tests/bootstrap_generator_test.py index 744e2999..6e7bfa0a 100644 --- a/tests/bootstrap_generator_test.py +++ b/tests/bootstrap_generator_test.py @@ -38,9 +38,9 @@ # Constants GCR_PROD: Final[str] = "gcr.io/trafficdirector-prod/td-grpc-bootstrap" -GCR_TESTING: Final[ - str -] = "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" +GCR_TESTING: Final[str] = ( + "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" +) # Returns a list of bootstrap generator versions to be tested along with their diff --git a/tests/url_map/affinity_test.py b/tests/url_map/affinity_test.py index 2bd93405..e3e2102c 100644 --- a/tests/url_map/affinity_test.py +++ b/tests/url_map/affinity_test.py @@ -101,9 +101,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() + path_matcher["defaultService"] = ( + GcpResourceManager().affinity_backend_service() + ) return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -176,9 +176,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() + path_matcher["defaultService"] = ( + GcpResourceManager().affinity_backend_service() + ) return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): From 39c5cbe5417ba1e180e8ff665e317d411542cdd3 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 11 Feb 2025 15:14:29 +0530 Subject: [PATCH 03/49] check patch --- framework/infrastructure/c6n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/framework/infrastructure/c6n.py b/framework/infrastructure/c6n.py index cd063cdb..c412e049 100644 --- a/framework/infrastructure/c6n.py +++ b/framework/infrastructure/c6n.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +#emchandwani : check import logging from google.cloud import run_v2 From 87317ec10deb7c5cca3ebde330ce14eb687d6359 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 11 Feb 2025 16:46:27 +0530 Subject: [PATCH 04/49] correct logging import and remove clourun server flag --- framework/test_app/runners/c6n/c6n_xds_server_runner.py | 2 +- framework/xds_k8s_flags.py | 6 ------ framework/xds_k8s_testcase.py | 5 ++--- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/c6n/c6n_xds_server_runner.py index 5fece4ce..e01118a0 100644 --- a/framework/test_app/runners/c6n/c6n_xds_server_runner.py +++ b/framework/test_app/runners/c6n/c6n_xds_server_runner.py @@ -20,7 +20,7 @@ import uuid from absl import flags -from absl import logging + from typing_extensions import override from framework import xds_flags diff --git a/framework/xds_k8s_flags.py b/framework/xds_k8s_flags.py index ef729c37..203aaac2 100644 --- a/framework/xds_k8s_flags.py +++ b/framework/xds_k8s_flags.py @@ -33,12 +33,6 @@ help="Traffic Director gRPC Bootstrap Docker image", ) -CLOUDRUN_SERVER_IMAGE = flags.DEFINE_string( - "cloudrun_server_image", - default=None, - help="Cloudrun Server Docker image name", -) - CLOUDRUN_CLIENT_IMAGE = flags.DEFINE_string( "cloudrun_client_image", default=None, diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index ccae817f..a5be24cf 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -217,7 +217,6 @@ def setUpClass(cls): cls.resource_suffix = xds_flags.RESOURCE_SUFFIX.value # Test server - cls.cloudrun_server_image = xds_k8s_flags.CLOUDRUN_SERVER_IMAGE.value cls.server_image = xds_k8s_flags.SERVER_IMAGE.value cls.server_name = xds_flags.SERVER_NAME.value cls.server_port = xds_flags.SERVER_PORT.value @@ -1464,12 +1463,12 @@ def setupServerBackends( def startTestServers( self, server_runner=None, **kwargs ) -> List[XdsTestServer]: - logger.info(self.cloudrun_server_image) + logger.info(self.server_image) if server_runner is None: self.server_runner = CloudRunServerRunner( project=self.project, service_name=self.server_namespace, - image_name=self.cloudrun_server_image, + image_name=self.server_image, network=self.network, region=self.region, ) From a9f88b04f910a8a9377057cba432131aa862c7b1 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 17 Feb 2025 15:17:29 +0530 Subject: [PATCH 05/49] client test not working --- bin/lib/common.py | 2 +- bin/run_test_server_c6n.py | 2 +- framework/infrastructure/gcp/api.py | 1 - framework/infrastructure/{ => gcp}/c6n.py | 14 ++-- framework/infrastructure/gcp/compute.py | 40 +++++++---- framework/infrastructure/traffic_director.py | 14 ++-- .../test_app/runners/c6n/c6n_base_runner.py | 21 +++--- .../runners/c6n/c6n_xds_server_runner.py | 17 ++--- framework/xds_k8s_testcase.py | 10 +-- .../client.cloudrun.deployment.yaml | 55 ++++++++------ kubernetes-manifests/client.deployment.yaml | 2 +- kubernetes-manifests/eshita.yaml | 71 +++++++++++++++++++ tests/baseline_test_c6n_server.py | 8 +-- 13 files changed, 179 insertions(+), 78 deletions(-) rename framework/infrastructure/{ => gcp}/c6n.py (83%) create mode 100644 kubernetes-manifests/eshita.yaml diff --git a/bin/lib/common.py b/bin/lib/common.py index 6f755b5d..1f3fab49 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,7 +22,7 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure import c6n +from framework.infrastructure.gcp import c6n from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.test_app import client_app diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_c6n.py index 91d4243b..7f842e80 100644 --- a/bin/run_test_server_c6n.py +++ b/bin/run_test_server_c6n.py @@ -1,4 +1,4 @@ -# Copyright 2020 gRPC authors. +# Copyright 2025 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index a0be2db1..aafad314 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -514,7 +514,6 @@ def wait_for_operation( class GcpStandardCloudApiResource(GcpProjectApiResource, metaclass=abc.ABCMeta): GLOBAL_LOCATION = "global" - def parent(self, location: Optional[str] = GLOBAL_LOCATION): if location is None: location = self.GLOBAL_LOCATION diff --git a/framework/infrastructure/c6n.py b/framework/infrastructure/gcp/c6n.py similarity index 83% rename from framework/infrastructure/c6n.py rename to framework/infrastructure/gcp/c6n.py index c412e049..d1d0db55 100644 --- a/framework/infrastructure/c6n.py +++ b/framework/infrastructure/gcp/c6n.py @@ -13,14 +13,18 @@ # limitations under the License. #emchandwani : check +import abc import logging from google.cloud import run_v2 +from framework.infrastructure.gcp.api import GcpProjectApiResource + logger = logging.getLogger(__name__) -class CloudRunApiManager: +class CloudRunApiManager(GcpProjectApiResource, metaclass=abc.ABCMeta): + DEFAULT_TEST_PORT = 8080 project: str region: str _parent: str @@ -40,7 +44,7 @@ def __init__(self, project: str, region: str): self._parent = f"projects/{self.project}/locations/{self.region}" self._service = None - def deploy_service(self, service_name: str, image_name: str): + def deploy_service(self, service_name: str, image_name: str, *,test_port: int = DEFAULT_TEST_PORT): if not service_name: raise ValueError("service_name cannot be empty or None") if not image_name: @@ -55,7 +59,8 @@ def deploy_service(self, service_name: str, image_name: str): ports=[ run_v2.ContainerPort( name="http1", - container_port=50051, + # emchandwani : change to self.server_xds_port + container_port=test_port, ), ], ) @@ -69,7 +74,8 @@ def deploy_service(self, service_name: str, image_name: str): try: operation = self._client.create_service(request=request) - self._service = operation.result(timeout=300) + # operation=GcpProjectApiResource.wait_for_operation(operation_request=request,test_success_fn=logger.info("done")) + self._service = operation.result(timeout=600) logger.info("Deployed service: %s", self._service.uri) return self._service.uri except Exception as e: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 643a81a9..34182a7c 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -152,15 +152,20 @@ def create_backend_service_traffic_director( locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, enable_dualstack: bool = False, + is_cloudrun: bool = False, ) -> "GcpResource": if not isinstance(protocol, self.BackendServiceProtocol): raise TypeError(f"Unexpected Backend Service protocol: {protocol}") body = { "name": name, "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director - # "healthChecks": [health_check.url], "protocol": protocol.name, } + # If it is not for cloud run, add heath check since cloud run does not + # support health check. + if not is_cloudrun: + body["healthChecks"] = [health_check.url] + # If add dualstack support is specified True, config the backend service # to support IPv6 if enable_dualstack: @@ -204,17 +209,26 @@ def backend_service_patch_backends( max_rate_per_endpoint: Optional[int] = None, *, circuit_breakers: Optional[dict[str, int]] = None, + is_cloudrun: bool = False, ): - if max_rate_per_endpoint is None: - max_rate_per_endpoint = 5 - backend_list = [ - { - "group": backend.url, - # "balancingMode": "CONNECTION", - # "maxRatePerEndpoint": max_rate_per_endpoint, - } - for backend in backends - ] + if is_cloudrun: + backend_list = [ + {"group": backend.url} + for backend in backends + ] + else: + if max_rate_per_endpoint is None: + max_rate_per_endpoint = 5 + + backend_list = [ + { + "group": backend.url, + "balancingMode": "RATE", + "maxRatePerEndpoint": max_rate_per_endpoint, + } + for backend in backends + ] + request = {"backends": backend_list} if circuit_breakers: @@ -559,7 +573,7 @@ def get_backend_service_backend_health(self, backend_service, backend): ) def create_serverless_neg( - self, name: str, region: str, service_name: str, network: str + self, name: str, region: str, service_name: str ): """Creates a serverless NEG. @@ -587,7 +601,7 @@ def create_serverless_neg( .execute() ) neg = self.get_serverless_network_endpoint_group(name, region) - print(neg) + logger.info(neg) return neg except Exception as e: diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 74b76bcb..c8c180a2 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -246,6 +246,7 @@ def create_backend_service( affinity_header: Optional[str] = None, locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, + is_cloudrun: Optional[bool] = False, ): if protocol is None: protocol = _BackendGRPC @@ -261,10 +262,11 @@ def create_backend_service( locality_lb_policies=locality_lb_policies, outlier_detection=outlier_detection, enable_dualstack=self.enable_dualstack, + is_cloudrun=is_cloudrun, ) self.backend_service = resource self.backend_service_protocol = protocol - logger.info("backend service protocal") + logger.info("backend service protocol") logger.info(self.backend_service_protocol) def load_backend_service(self): @@ -1320,9 +1322,11 @@ def __init__( ) # API - self.netsvc = gcp.network_services.NetworkServicesV1( - gcp_api_manager, project - ) + # netsvc: gcp.network_services.NetworkServicesV1 + # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) + # self.netsvc = gcp.network_services.NetworkServicesV1( + # gcp_api_manager, project + # ) # Managed resources # TODO(gnossen) PTAL at the pylint error @@ -1373,6 +1377,7 @@ def backend_service_add_backends( self.compute.backend_service_patch_backends( backend_service, backends, + is_cloudrun=True, ) def create_grpc_route(self, src_host: str, src_port: int): @@ -1399,7 +1404,6 @@ def create_grpc_route(self, src_host: str, src_port: int): } if hasattr(self, "mesh") and self.mesh: route_body["meshes"] = self.mesh.url - resource = self.netsvc.create_grpc_route(route_name, route_body) self.grpc_route = self.netsvc.get_grpc_route(route_name) logger.info("gRPC Route created successfully: %s", self.grpc_route) diff --git a/framework/test_app/runners/c6n/c6n_base_runner.py b/framework/test_app/runners/c6n/c6n_base_runner.py index 95bb2cc2..efe37a3f 100644 --- a/framework/test_app/runners/c6n/c6n_base_runner.py +++ b/framework/test_app/runners/c6n/c6n_base_runner.py @@ -22,7 +22,7 @@ from typing import Optional import framework -from framework.infrastructure import c6n +from framework.infrastructure.gcp import c6n from framework.test_app.runners import base_runner logger = logging.getLogger(__name__) @@ -51,8 +51,8 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): network: Optional[str] = None tag: str = "latest" region: Optional[str] = None - _current_revision: Optional[str] = None - gcp_project: Optional[str] = None + current_revision: Optional[str] = None + # gcp_project: Optional[str] = None gcp_ui_url: Optional[str] = None run_history: collections.deque[RunHistory] @@ -77,9 +77,8 @@ def __init__( self.network = network self.region = region self.current_revision = None - self.gcp_project = None + # self.gcp_project = None self.gcp_ui_url = None - self.run_history = collections.deque() # Persistent across many runs. self.run_history = collections.deque() @@ -111,19 +110,19 @@ def run(self, **kwargs): ) self._reset_state() - self.time_start_requested = dt.datetime.now(tz=dt.timezone.utc) - self._current_revision = self.cloudrun_api_manager.deploy_service( - self.service_name, self.image_name + self.time_start_requested = _datetime.now() + self.current_revision = self.cloudrun_api_manager.deploy_service( + self.service_name, self.image_name, ) def _start_completed(self): - self.time_start_completed = dt.datetime.now(tz=dt.timezone.utc) + self.time_start_completed = _datetime.now() def _stop(self): - self.time_stopped = dt.datetime.now(tz=dt.timezone.utc) + self.time_stopped = _datetime.now() if self.time_start_requested: run_history = RunHistory( - revision_id=self._current_revision, + revision_id=self.current_revision, time_start_requested=self.time_start_requested, time_start_completed=self.time_start_completed, time_stopped=self.time_stopped, diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/c6n/c6n_xds_server_runner.py index e01118a0..bd4003fc 100644 --- a/framework/test_app/runners/c6n/c6n_xds_server_runner.py +++ b/framework/test_app/runners/c6n/c6n_xds_server_runner.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Run xDS Test Client on Kubernetes. +Run xDS Test Server on Cloud Run. """ import dataclasses import logging @@ -25,7 +25,7 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure import c6n +from framework.infrastructure.gcp import c6n from framework.test_app.runners.c6n import c6n_base_runner from framework.test_app.server_app import XdsTestServer @@ -86,16 +86,15 @@ def run(self, **kwargs) -> List[XdsTestServer]: """Deploys and manages the xDS Test Server on Cloud Run.""" logger.info(self.service_name) logger.info(self.image_name) - deployed_service_url = self.cloudrun_api_manager.deploy_service( - service_name=self.service_name, - image_name=self.image_name, - ) + + super().run(**kwargs) servers = [ XdsTestServer( - ip="0.0.0.0", rpc_port=0, hostname=deployed_service_url + ip="0.0.0.0", rpc_port=0, hostname=self.current_revision ) ] self.servers = servers # Add servers to the list + self._start_completed() return servers def get_service_url(self): @@ -103,4 +102,6 @@ def get_service_url(self): @override def cleanup(self, *, force=False): - return super().cleanup(force=force) + super().cleanup(force=force) + self._stop() + return diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index a5be24cf..7222579b 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -225,7 +225,6 @@ def setUpClass(cls): cls.server_xds_port = xds_flags.SERVER_XDS_PORT.value # Test client - cls.cloudrun_client_image = xds_k8s_flags.CLOUDRUN_CLIENT_IMAGE.value cls.client_image = xds_k8s_flags.CLIENT_IMAGE.value cls.client_name = xds_flags.CLIENT_NAME.value cls.client_port = xds_flags.CLIENT_PORT.value @@ -1433,8 +1432,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: self.k8s_api_manager, self.client_namespace ), deployment_name=self.client_name, - deployment_template="client.cloudrun.deployment.yaml", - image_name=self.cloudrun_client_image, + image_name=self.client_image, td_bootstrap_image=self.td_bootstrap_image, gcp_project=self.project, gcp_api_manager=self.gcp_api_manager, @@ -1443,6 +1441,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: network=self.network, debug_use_port_forwarding=self.debug_use_port_forwarding, enable_workload_identity=self.enable_workload_identity, + deployment_template="client.cloudrun.deployment.yaml", stats_port=self.client_port, reuse_namespace=self.server_namespace == self.client_namespace, **kwargs, @@ -1460,7 +1459,7 @@ def setupServerBackends( service_url = server_runner.get_service_url() self.td.backend_service_add_backends([service_url]) - def startTestServers( + def startTestServers( self, server_runner=None, **kwargs ) -> List[XdsTestServer]: logger.info(self.server_image) @@ -1482,6 +1481,8 @@ def startTestServers( def startTestClient( self, test_server: XdsTestServer, **kwargs ) -> XdsTestClient: + # logger.info("emchandwani : test server : %v",test_server) + logger.info("emchandwani : xds URI : %s",test_server.xds_uri) return self._start_test_client(test_server.xds_uri, **kwargs) def backend_service_add_serverless_neg_backends(self): @@ -1490,7 +1491,6 @@ def backend_service_add_serverless_neg_backends(self): self.server_namespace, self.region, self.server_namespace, - "serverless", ) return neg diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml index 87a28c95..ffc0c6f2 100644 --- a/kubernetes-manifests/client.cloudrun.deployment.yaml +++ b/kubernetes-manifests/client.cloudrun.deployment.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -17,14 +16,14 @@ spec: spec: containers: - name: psm-grpc-client - image: docker.io/grpc/xds-example-cpp-client:v1.69.x + image: ${image_name} imagePullPolicy: Always args: - "--xds_creds" - "--target=xds:///psm-grpc-server" env: - name: GRPC_TRACE - value: "http1" + value: "xds_client" - name: GRPC_XDS_BOOTSTRAP value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE @@ -33,35 +32,45 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER value: "true" - - - name: CSM_WORKLOAD_NAME - value: psm-grpc-client - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CONTAINER_NAME - value: psm-grpc-client - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) +% if csm_workload_name: + - name: CSM_WORKLOAD_NAME + value: ${csm_workload_name} + % endif + % if csm_canonical_service_name: + - name: CSM_CANONICAL_SERVICE_NAME + value: ${csm_canonical_service_name} + % endif + % if enable_csm_observability: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME) + % endif volumeMounts: - mountPath: /tmp/grpc-xds/ name: grpc-td-conf readOnly: true initContainers: - name: grpc-td-init - image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 + image: ${td_bootstrap_image} imagePullPolicy: Always args: - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=default" - - "--xds-server-uri=trafficdirector.googleapis.com:443" - - "--config-mesh=${config_mesh}" + - "--vpc-network-name=${network}" + % if xds_server_uri: + - "--xds-server-uri=${xds_server_uri}" + % endif + % if config_mesh: + - "--config-mesh-experimental=${config_mesh}" + % endif - "--is-trusted-xds-server-experimental=true" + # - "--gcp-project=emchandwani-default" volumeMounts: - mountPath: /tmp/bootstrap/ name: grpc-td-conf @@ -69,4 +78,4 @@ spec: - name: grpc-td-conf emptyDir: medium: Memory -... \ No newline at end of file +... diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 67ef56d2..09a18e43 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -36,7 +36,7 @@ spec: ## driver waiting for the container to start. failureThreshold: 1000 args: - - "--server=psm-grpc-server:8080" + - "--server=${server_target}" - "--stats_port=${stats_port}" - "--qps=${qps}" - "--rpc=${rpc}" diff --git a/kubernetes-manifests/eshita.yaml b/kubernetes-manifests/eshita.yaml new file mode 100644 index 00000000..1b61019e --- /dev/null +++ b/kubernetes-manifests/eshita.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psm-grpc-client + labels: + app: psm-grpc-client +spec: + replicas: 1 + selector: + matchLabels: + app: psm-grpc-client + template: + metadata: + labels: + app: psm-grpc-client + spec: + containers: + - name: psm-grpc-client + image: us-docker.pkg.dev/grpc-testing/psm-interop/java-client:v1.63.x + imagePullPolicy: Always + args: + - "--xds_creds" + - "--target=xds:///psm-grpc-server" + env: + - name: GRPC_TRACE + value: "xds_client" + - name: GRPC_XDS_BOOTSTRAP + value: "/tmp/grpc-xds/td-grpc-bootstrap.json" + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER + value: "true" + + - name: CSM_WORKLOAD_NAME + value: psm-grpc-client + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: psm-grpc-client + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=\$(POD_NAME),k8s.namespace.name=\$(NAMESPACE_NAME),k8s.container.name=\$(CONTAINER_NAME) + volumeMounts: + - mountPath: /tmp/grpc-xds/ + name: grpc-td-conf + readOnly: true + initContainers: + - name: grpc-td-init + image: us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:ad35743ff01c0dc16a785e421591d7d52f3cdda4 + imagePullPolicy: Always + args: + - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" + - "--vpc-network-name=default" + - "--config-mesh-experimental=emchandwani-psm-grpc-mesh-20250217-0927-24ueq" + - "--is-trusted-xds-server-experimental=true" + # - "--gcp-project=emchandwani-default" + volumeMounts: + - mountPath: /tmp/bootstrap/ + name: grpc-td-conf + volumes: + - name: grpc-td-conf + emptyDir: + medium: Memory +... \ No newline at end of file diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_c6n_server.py index 466101c5..1c3b89d4 100644 --- a/tests/baseline_test_c6n_server.py +++ b/tests/baseline_test_c6n_server.py @@ -23,8 +23,8 @@ _XdsTestClient = xds_k8s_testcase.XdsTestClient -class BaselineTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): - def test_traffic_director_grpc_setup(self): +class CloudRunServerBaselineTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): + def test_GKE_client_cloudrun_service(self): with self.subTest("0_create_mesh"): self.td.create_mesh() @@ -35,9 +35,7 @@ def test_traffic_director_grpc_setup(self): neg = self.backend_service_add_serverless_neg_backends() with self.subTest("3_create_backend_service"): - self.td.create_backend_service( - protocol=self.compute_v1.BackendServiceProtocol.HTTP2 - ) + self.td.create_backend_service(protocol=self.compute_v1.BackendServiceProtocol.HTTP2,is_cloudrun=True) with self.subTest("4_add_server_backends_to_backend_service"): neg_resource = self.compute_v1.GcpResource( From 0142857dd548d04d6a1cf8d93a0e25f6357b14c0 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 21 Feb 2025 11:56:16 +0530 Subject: [PATCH 06/49] changes --- config/common.cfg | 2 +- framework/infrastructure/gcp/c6n.py | 2 +- framework/infrastructure/traffic_director.py | 64 ++++++++++--------- framework/xds_k8s_testcase.py | 4 +- .../client.cloudrun.deployment.yaml | 57 ++++++++--------- kubernetes-manifests/client.deployment.yaml | 21 ++---- tests/baseline_test_c6n_server.py | 2 +- 7 files changed, 70 insertions(+), 82 deletions(-) diff --git a/config/common.cfg b/config/common.cfg index 1bc8dff8..df6cc89d 100644 --- a/config/common.cfg +++ b/config/common.cfg @@ -1,5 +1,5 @@ --resource_prefix=psm-interop ---td_bootstrap_image=us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:ad35743ff01c0dc16a785e421591d7d52f3cdda4 +--td_bootstrap_image=us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:e4622a2cb6956f85da18eb0aa9daec839c2e38b5 # The canonical implementation of the xDS test server. # Can be used in tests where language-specific xDS test server does not exist, diff --git a/framework/infrastructure/gcp/c6n.py b/framework/infrastructure/gcp/c6n.py index d1d0db55..897b46db 100644 --- a/framework/infrastructure/gcp/c6n.py +++ b/framework/infrastructure/gcp/c6n.py @@ -75,7 +75,7 @@ def deploy_service(self, service_name: str, image_name: str, *,test_port: int = try: operation = self._client.create_service(request=request) # operation=GcpProjectApiResource.wait_for_operation(operation_request=request,test_success_fn=logger.info("done")) - self._service = operation.result(timeout=600) + self._service = operation.result(timeout=800) logger.info("Deployed service: %s", self._service.uri) return self._service.uri except Exception as e: diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index c8c180a2..02d58d14 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -1380,34 +1380,36 @@ def backend_service_add_backends( is_cloudrun=True, ) - def create_grpc_route(self, src_host: str, src_port: int): - hostname = src_host - route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" - logger.info("Creating gRPC route: %s", route_name) - - try: - backend_service_url = self.backend_service.url.split("v1/")[-1] - # URL of the backend service - route_body = { - "name": route_name, - "hostnames": [hostname], - "rules": [ - { - "matches": [], # Empty matches means any rpc - "action": { - "destinations": [ - {"serviceName": backend_service_url} - ] - }, - } - ], - } - if hasattr(self, "mesh") and self.mesh: - route_body["meshes"] = self.mesh.url - resource = self.netsvc.create_grpc_route(route_name, route_body) - self.grpc_route = self.netsvc.get_grpc_route(route_name) - logger.info("gRPC Route created successfully: %s", self.grpc_route) - - except Exception as e: # Catching generic exceptions for now - logger.exception("Error creating gRPC route: %s", e) - raise + # def create_grpc_route(self, src_host: str, src_port: int): + # self.create_grpc_route(src_host, src_port) + # hostname = f"{src_host}:{src_port}" + # route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" + # logger.info("Creating gRPC route: %s", route_name) + # service_name=self.netsvc.resource_full_name( + # self.backend_service.name, "backendServices" + # ) + # try: + # # backend_service_url = self.backend_service.url.split("v1/")[-1] + # # URL of the backend service + # route_body = { + # "name": route_name, + # "hostnames": [hostname], + # "rules": [ + # { + # "action": { + # "destinations": [ + # {"serviceName": service_name} + # ] + # }, + # } + # ], + # } + # if hasattr(self, "mesh") and self.mesh: + # route_body["meshes"] = self.mesh.url + # resource = self.netsvc.create_grpc_route(route_name, route_body) + # self.grpc_route = self.netsvc.get_grpc_route(route_name) + # logger.info("gRPC Route created successfully: %s", self.grpc_route) + + # except Exception as e: # Catching generic exceptions for now + # logger.exception("Error creating gRPC route: %s", e) + # raise diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 7222579b..d640a214 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1441,7 +1441,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: network=self.network, debug_use_port_forwarding=self.debug_use_port_forwarding, enable_workload_identity=self.enable_workload_identity, - deployment_template="client.cloudrun.deployment.yaml", + deployment_template="client.deployment.yaml", stats_port=self.client_port, reuse_namespace=self.server_namespace == self.client_namespace, **kwargs, @@ -1483,7 +1483,7 @@ def startTestClient( ) -> XdsTestClient: # logger.info("emchandwani : test server : %v",test_server) logger.info("emchandwani : xds URI : %s",test_server.xds_uri) - return self._start_test_client(test_server.xds_uri, **kwargs) + return self._start_test_client(test_server.xds_uri,secure_mode=True, **kwargs) def backend_service_add_serverless_neg_backends(self): logger.info("Creating serverless NEG") diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml index ffc0c6f2..8c45c7ab 100644 --- a/kubernetes-manifests/client.cloudrun.deployment.yaml +++ b/kubernetes-manifests/client.cloudrun.deployment.yaml @@ -14,13 +14,15 @@ spec: labels: app: psm-grpc-client spec: + % if service_account_name: + serviceAccountName: ${service_account_name} + % endif containers: - name: psm-grpc-client image: ${image_name} imagePullPolicy: Always - args: - - "--xds_creds" - - "--target=xds:///psm-grpc-server" + args: + - "--server=${server_target}" env: - name: GRPC_TRACE value: "xds_client" @@ -32,45 +34,35 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER value: "true" -% if csm_workload_name: - - name: CSM_WORKLOAD_NAME - value: ${csm_workload_name} - % endif - % if csm_canonical_service_name: - - name: CSM_CANONICAL_SERVICE_NAME - value: ${csm_canonical_service_name} - % endif - % if enable_csm_observability: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME) - % endif + + - name: CSM_WORKLOAD_NAME + value: psm-grpc-client + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: psm-grpc-client + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) volumeMounts: - mountPath: /tmp/grpc-xds/ name: grpc-td-conf readOnly: true initContainers: - name: grpc-td-init - image: ${td_bootstrap_image} + image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 imagePullPolicy: Always args: - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - % if config_mesh: - - "--config-mesh-experimental=${config_mesh}" - % endif + - "--vpc-network-name=default" + - "--xds-server-uri=trafficdirector.googleapis.com:443" + - "--config-mesh=${config_mesh}" - "--is-trusted-xds-server-experimental=true" - # - "--gcp-project=emchandwani-default" volumeMounts: - mountPath: /tmp/bootstrap/ name: grpc-td-conf @@ -78,4 +70,5 @@ spec: - name: grpc-td-conf emptyDir: medium: Memory + ... diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 09a18e43..0566575c 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -54,24 +54,16 @@ spec: ports: - containerPort: ${stats_port} env: + - name: GRPC_TRACE + value: "xds_client" - name: GRPC_XDS_BOOTSTRAP value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE value: "true" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS value: "true" - - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER value: "true" - - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - % if enable_dualstack: - - name: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS - value: "true" - % endif % if csm_workload_name: - name: CSM_WORKLOAD_NAME value: ${csm_workload_name} @@ -114,11 +106,12 @@ spec: - "--xds-server-uri=${xds_server_uri}" % endif % if config_mesh: - - "--config-mesh-experimental=${config_mesh}" + - "--config-mesh=${config_mesh}" % endif % if generate_mesh_id: - "--generate-mesh-id-experimental" % endif + - "--is-trusted-xds-server-experimental=true" resources: limits: cpu: 100m diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_c6n_server.py index 1c3b89d4..9773893e 100644 --- a/tests/baseline_test_c6n_server.py +++ b/tests/baseline_test_c6n_server.py @@ -45,7 +45,7 @@ def test_GKE_client_cloudrun_service(self): self.td.backend_service_add_backends([neg_resource], self.region) with self.subTest("5_create_grpc_route"): - self.td.create_grpc_route(self.server_xds_host, self.server_port) + self.td.create_grpc_route(self.server_xds_host, self.server_xds_port) with self.subTest("7_start_test_client"): test_client: _XdsTestClient = self.startTestClient( From 579f78256d55b80d2adfa55e83bce99edaf2ea6a Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 01:38:29 +0530 Subject: [PATCH 07/49] running tests --- bin/lib/common.py | 8 +- bin/run_test_server_c6n.py | 16 +-- .../gcp/{c6n.py => cloud_run.py} | 8 +- framework/infrastructure/gcp/compute.py | 1 - .../infrastructure/gcp/network_services.py | 44 +++--- framework/infrastructure/traffic_director.py | 128 +----------------- .../traffic_director_cloudrun.py | 119 ++++++++++++++++ .../cloud_run_base_runner.py} | 33 ++--- .../cloud_run_xds_server_runner.py} | 17 ++- .../runners/k8s/k8s_xds_client_runner.py | 5 +- framework/xds_c6n_testcase.py | 0 framework/xds_flags.py | 2 +- framework/xds_k8s_testcase.py | 93 +++++++++++-- .../client.cloudrun.deployment.yaml | 74 ---------- kubernetes-manifests/client.deployment.yaml | 21 +++ ...r.py => baseline_test_cloud_run_server.py} | 2 +- tests/bootstrap_generator_test.py | 8 +- tests/url_map/affinity_test.py | 14 +- 18 files changed, 297 insertions(+), 296 deletions(-) rename framework/infrastructure/gcp/{c6n.py => cloud_run.py} (91%) create mode 100644 framework/infrastructure/traffic_director_cloudrun.py rename framework/test_app/runners/{c6n/c6n_base_runner.py => cloud_run/cloud_run_base_runner.py} (85%) rename framework/test_app/runners/{c6n/c6n_xds_server_runner.py => cloud_run/cloud_run_xds_server_runner.py} (88%) delete mode 100644 framework/xds_c6n_testcase.py delete mode 100644 kubernetes-manifests/client.cloudrun.deployment.yaml rename tests/{baseline_test_c6n_server.py => baseline_test_cloud_run_server.py} (95%) diff --git a/bin/lib/common.py b/bin/lib/common.py index 1f3fab49..5da21bd7 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,12 +22,12 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure.gcp import c6n +from framework.infrastructure.gcp import cloud_run from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.test_app import client_app from framework.test_app import server_app -from framework.test_app.runners.c6n import c6n_xds_server_runner +from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import gamma_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner @@ -47,7 +47,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner GammaServerRunner = gamma_server_runner.GammaServerRunner -CloudRunServerRunner=c6n_xds_server_runner.CloudRunServerRunner +CloudRunServerRunner=cloud_run_xds_server_runner.CloudRunServerRunner _XdsTestServer = server_app.XdsTestServer _XdsTestClient = client_app.XdsTestClient @@ -64,7 +64,7 @@ def gcp_api_manager(): @functools.cache def c6n_api_manager(): - return c6n.CloudRunApiManager() + return cloud_run.CloudRunApiManager() def td_attrs(): return dict( diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_c6n.py index 7f842e80..c6b32253 100644 --- a/bin/run_test_server_c6n.py +++ b/bin/run_test_server_c6n.py @@ -17,23 +17,23 @@ Typical usage examples: # Help. - ./run.sh ./bin/run_test_server_c6n.py --help + ./run.sh ./bin/run_test_server_cloud_run.py --help # Run modes. - ./run.sh ./bin/run_test_server_c6n.py --mode=app_net - ./run.sh ./bin/run_test_server_c6n.py --mode=secure + ./run.sh ./bin/run_test_server_cloud_run.py --mode=app_net + ./run.sh ./bin/run_test_server_cloud_run.py --mode=secure # Gamma run mode: uses HTTPRoute by default. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma # Gamma run mode: use GRPCRoute. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --gamma_route_kind=grpc + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --gamma_route_kind=grpc # Running multipler server replicas. - ./run.sh ./bin/run_test_server_c6n.py --server_replica_count=3 + ./run.sh ./bin/run_test_server_cloud_run.py --server_replica_count=3 # Cleanup: make sure to set the same mode used to create. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --cmd=cleanup + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --cmd=cleanup """ import logging @@ -58,7 +58,7 @@ def main(argv): xds_flags.set_socket_default_timeout_from_flag() run_kwargs = dict() - server_runner = common.make_c6n_server_runner() + server_runner = common.make_cloud_run_server_runner() server_runner.run(**run_kwargs) diff --git a/framework/infrastructure/gcp/c6n.py b/framework/infrastructure/gcp/cloud_run.py similarity index 91% rename from framework/infrastructure/gcp/c6n.py rename to framework/infrastructure/gcp/cloud_run.py index 897b46db..204eeaa2 100644 --- a/framework/infrastructure/gcp/c6n.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#emchandwani : check import abc import logging @@ -58,8 +56,7 @@ def deploy_service(self, service_name: str, image_name: str, *,test_port: int = image=image_name, ports=[ run_v2.ContainerPort( - name="http1", - # emchandwani : change to self.server_xds_port + name="h2c", container_port=test_port, ), ], @@ -74,8 +71,7 @@ def deploy_service(self, service_name: str, image_name: str, *,test_port: int = try: operation = self._client.create_service(request=request) - # operation=GcpProjectApiResource.wait_for_operation(operation_request=request,test_success_fn=logger.info("done")) - self._service = operation.result(timeout=800) + self._service = operation.result(timeout=600) logger.info("Deployed service: %s", self._service.uri) return self._service.uri except Exception as e: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 34182a7c..92429c79 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -581,7 +581,6 @@ def create_serverless_neg( name: The name of the NEG. region: The region in which to create the NEG. service_name: The name of the Cloud Run service. Format: "namespaces/{namespace}/services/{service}" - network: The network of the NEG. Format: "projects/{project}/global/networks/{network}" Returns: The NEG selfLink URL diff --git a/framework/infrastructure/gcp/network_services.py b/framework/infrastructure/gcp/network_services.py index 09132536..d13e60fa 100644 --- a/framework/infrastructure/gcp/network_services.py +++ b/framework/infrastructure/gcp/network_services.py @@ -112,19 +112,14 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteMatch": return cls( - method=( - GrpcRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None - ), - headers=( - tuple( - GrpcRoute.HeaderMatch.from_response(h) - for h in d["headers"] - ) - if "headers" in d - else () - ), + method=GrpcRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None, + headers=tuple( + GrpcRoute.HeaderMatch.from_response(h) for h in d["headers"] + ) + if "headers" in d + else (), ) @dataclasses.dataclass(frozen=True) @@ -229,19 +224,14 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteMatch": return cls( - method=( - HttpRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None - ), - headers=( - tuple( - HttpRoute.HeaderMatch.from_response(h) - for h in d["headers"] - ) - if "headers" in d - else () - ), + method=HttpRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None, + headers=tuple( + HttpRoute.HeaderMatch.from_response(h) for h in d["headers"] + ) + if "headers" in d + else (), ) @dataclasses.dataclass(frozen=True) @@ -484,4 +474,4 @@ def delete_http_route(self, name: str) -> bool: return self._delete_resource( collection=self._api_locations.httpRoutes(), full_name=self.resource_full_name(name, self.HTTP_ROUTES), - ) + ) \ No newline at end of file diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 02d58d14..5d6b2f2a 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -1286,130 +1286,4 @@ def _get_certificate_provider(cls): "certificateProviderInstance": { "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, }, - } - - -class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): - MESH_NAME = "grpc-mesh" - SERVER_TLS_POLICY_NAME = "server-tls-policy" - CLIENT_TLS_POLICY_NAME = "client-tls-policy" - AUTHZ_POLICY_NAME = "authz-policy" - ENDPOINT_POLICY = "endpoint-policy" - CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" - - netsec: _NetworkSecurityV1Beta1 - netsvc: _NetworkServicesV1Beta1 - - def __init__( - self, - gcp_api_manager: gcp.api.GcpApiManager, - project: str, - *, - resource_prefix: str, - resource_suffix: Optional[str] = None, - network: str = "default", - compute_api_version: str = "v1", - enable_dualstack: bool = False, - ): - super().__init__( - gcp_api_manager, - project, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - network=network, - compute_api_version=compute_api_version, - enable_dualstack=enable_dualstack, - ) - - # API - # netsvc: gcp.network_services.NetworkServicesV1 - # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) - # self.netsvc = gcp.network_services.NetworkServicesV1( - # gcp_api_manager, project - # ) - - # Managed resources - # TODO(gnossen) PTAL at the pylint error - self.grpc_route: Optional[GrpcRoute] = None - self.http_route: Optional[HttpRoute] = None - self.mesh: Optional[Mesh] = None - - # Managed resources - self.server_tls_policy: Optional[ServerTlsPolicy] = None - self.client_tls_policy: Optional[ClientTlsPolicy] = None - self.authz_policy: Optional[AuthorizationPolicy] = None - self.endpoint_policy: Optional[EndpointPolicy] = None - - def backend_service_add_backends( - self, - backends: Sequence[str], - region: Optional[str] = None, - balancing_mode: str = "CONNECTION", - max_rate_per_endpoint: Optional[int] = None, - capacity_scaler: float = 1.0, - *, - circuit_breakers: Optional[dict[str, int]] = None, - ): - - new_backends = [] - for backend in backends: - new_backend = { - "group": backend, - "balancingMode": balancing_mode, - "maxRatePerEndpoint": max_rate_per_endpoint, - "capacityScaler": capacity_scaler, - } - - if circuit_breakers is not None: - new_backend["circuitBreakers"] = circuit_breakers - - new_backends.append(new_backend) - - # self.backends.update(new_backends) - backend_service = self.backend_service - - logging.info( - "Adding backends to Backend Service %s: %r", - backend_service.name, - new_backends, - ) - - self.compute.backend_service_patch_backends( - backend_service, - backends, - is_cloudrun=True, - ) - - # def create_grpc_route(self, src_host: str, src_port: int): - # self.create_grpc_route(src_host, src_port) - # hostname = f"{src_host}:{src_port}" - # route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" - # logger.info("Creating gRPC route: %s", route_name) - # service_name=self.netsvc.resource_full_name( - # self.backend_service.name, "backendServices" - # ) - # try: - # # backend_service_url = self.backend_service.url.split("v1/")[-1] - # # URL of the backend service - # route_body = { - # "name": route_name, - # "hostnames": [hostname], - # "rules": [ - # { - # "action": { - # "destinations": [ - # {"serviceName": service_name} - # ] - # }, - # } - # ], - # } - # if hasattr(self, "mesh") and self.mesh: - # route_body["meshes"] = self.mesh.url - # resource = self.netsvc.create_grpc_route(route_name, route_body) - # self.grpc_route = self.netsvc.get_grpc_route(route_name) - # logger.info("gRPC Route created successfully: %s", self.grpc_route) - - # except Exception as e: # Catching generic exceptions for now - # logger.exception("Error creating gRPC route: %s", e) - # raise + } \ No newline at end of file diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloudrun.py new file mode 100644 index 00000000..20ff2fa1 --- /dev/null +++ b/framework/infrastructure/traffic_director_cloudrun.py @@ -0,0 +1,119 @@ +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import Optional, Sequence +from framework.infrastructure.gcp.network_security import NetworkSecurityV1Beta1 +from framework.infrastructure.gcp.network_services import NetworkServicesV1Beta1 +from framework.infrastructure.traffic_director import TrafficDirectorAppNetManager +from framework.infrastructure import gcp + +# _NetworkSecurityV1Beta1 = gcp.network_security.NetworkSecurityV1Beta1 +ServerTlsPolicy = gcp.network_security.ServerTlsPolicy +ClientTlsPolicy = gcp.network_security.ClientTlsPolicy +AuthorizationPolicy = gcp.network_security.AuthorizationPolicy + +# Network Services +# _NetworkServicesV1Beta1 = gcp.network_services.NetworkServicesV1Beta1 +EndpointPolicy = gcp.network_services.EndpointPolicy +GrpcRoute = gcp.network_services.GrpcRoute +HttpRoute = gcp.network_services.HttpRoute +Mesh = gcp.network_services.Mesh + +class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): + MESH_NAME = "grpc-mesh" + SERVER_TLS_POLICY_NAME = "server-tls-policy" + CLIENT_TLS_POLICY_NAME = "client-tls-policy" + AUTHZ_POLICY_NAME = "authz-policy" + ENDPOINT_POLICY = "endpoint-policy" + CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" + + # netsec: _NetworkSecurityV1Beta1 + # netsvc: _NetworkServicesV1Beta1 + + def __init__( + self, + gcp_api_manager: gcp.api.GcpApiManager, + project: str, + *, + resource_prefix: str, + resource_suffix: Optional[str] = None, + network: str = "default", + compute_api_version: str = "v1", + enable_dualstack: bool = False, + ): + super().__init__( + gcp_api_manager, + project, + resource_prefix=resource_prefix, + resource_suffix=resource_suffix, + network=network, + compute_api_version=compute_api_version, + enable_dualstack=enable_dualstack, + ) + + # API + # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) + # self.netsvc = _NetworkSecurityV1Beta1(gcp_api_manager, project) + + # Managed resources + # TODO(gnossen) PTAL at the pylint error + self.grpc_route: Optional[GrpcRoute] = None + self.http_route: Optional[HttpRoute] = None + self.mesh: Optional[Mesh] = None + + # Managed resources + self.server_tls_policy: Optional[ServerTlsPolicy] = None + self.client_tls_policy: Optional[ClientTlsPolicy] = None + self.authz_policy: Optional[AuthorizationPolicy] = None + self.endpoint_policy: Optional[EndpointPolicy] = None + + def backend_service_add_backends( + self, + backends: Sequence[str], + region: Optional[str] = None, + balancing_mode: str = "CONNECTION", + max_rate_per_endpoint: Optional[int] = None, + capacity_scaler: float = 1.0, + *, + circuit_breakers: Optional[dict[str, int]] = None, + ): + + new_backends = [] + for backend in backends: + new_backend = { + "group": backend, + "balancingMode": balancing_mode, + "maxRatePerEndpoint": max_rate_per_endpoint, + "capacityScaler": capacity_scaler, + } + + if circuit_breakers is not None: + new_backend["circuitBreakers"] = circuit_breakers + + new_backends.append(new_backend) + + # self.backends.update(new_backends) + backend_service = self.backend_service + + logging.info( + "Adding backends to Backend Service %s: %r", + backend_service.name, + new_backends, + ) + + self.compute.backend_service_patch_backends( + backend_service, + backends, + is_cloudrun=True, + ) \ No newline at end of file diff --git a/framework/test_app/runners/c6n/c6n_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py similarity index 85% rename from framework/test_app/runners/c6n/c6n_base_runner.py rename to framework/test_app/runners/cloud_run/cloud_run_base_runner.py index efe37a3f..ab229fb5 100644 --- a/framework/test_app/runners/c6n/c6n_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -22,7 +22,7 @@ from typing import Optional import framework -from framework.infrastructure.gcp import c6n +from framework.infrastructure.gcp import cloud_run from framework.test_app.runners import base_runner logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): image_name: str network: Optional[str] = None tag: str = "latest" - region: Optional[str] = None + region: str = "us-central1" current_revision: Optional[str] = None # gcp_project: Optional[str] = None gcp_ui_url: Optional[str] = None @@ -66,8 +66,8 @@ def __init__( project: str, service_name: str, image_name: str, + region: str, network: Optional[str] = None, - region: Optional[str] = None, ) -> None: super().__init__() @@ -77,7 +77,6 @@ def __init__( self.network = network self.region = region self.current_revision = None - # self.gcp_project = None self.gcp_ui_url = None # Persistent across many runs. @@ -93,7 +92,7 @@ def __init__( def _initalize_cloudrun_api_manager(self): """Initializes the CloudRunApiManager.""" - self.cloudrun_api_manager = c6n.CloudRunApiManager( + self.cloudrun_api_manager = cloud_run.CloudRunApiManager( project=self.project, region=self.region ) @@ -129,17 +128,13 @@ def _stop(self): ) self.run_history.append(run_history) - @classmethod - def _get_workload_identity_member_name( - cls, project, namespace_name, service_account_name - ): - """ - Returns workload identity member name used to authenticate Kubernetes - service accounts. - - https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity - """ - return ( - f"serviceAccount:{project}.svc.id.goog" - f"[{namespace_name}/{service_account_name}]" - ) + def stop(self): + """Deletes Cloud Run Service""" + logger.info("Deleting Cloud Run service: %s", self.service_name) + try: + self.cloudrun_api_manager.delete_service(self.service_name) + logger.info("Cloud Run service %s deleted", self.service_name) + except Exception as e: + logger.warning( + "Cloud Run service %s deletion failed: %s", self.service_name, e + ) \ No newline at end of file diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py similarity index 88% rename from framework/test_app/runners/c6n/c6n_xds_server_runner.py rename to framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index bd4003fc..12b29865 100644 --- a/framework/test_app/runners/c6n/c6n_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -1,4 +1,4 @@ -# Copyright 2022 gRPC authors. +# Copyright 2025 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,8 +25,8 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure.gcp import c6n -from framework.test_app.runners.c6n import c6n_base_runner +from framework.infrastructure.gcp import cloud_run +from framework.test_app.runners.cloud_run import cloud_run_base_runner from framework.test_app.server_app import XdsTestServer logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ def as_dict(self): } -class CloudRunServerRunner(c6n_base_runner.CloudRunBaseRunner): +class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" def __init__( @@ -102,6 +102,9 @@ def get_service_url(self): @override def cleanup(self, *, force=False): - super().cleanup(force=force) - self._stop() - return + try: + if self.service: + self.stop() + self.service_name=None + finally: + self._stop() diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index b1f4e142..25430bac 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -131,12 +131,13 @@ def run( # pylint: disable=arguments-differ log_to_stdout: bool = False, request_payload_size: int = 0, response_payload_size: int = 0, + is_trusted_xds_server_experimental: bool=False ) -> client_app.XdsTestClient: logger.info( ( 'Deploying xDS test client "%s" to k8s namespace %s: ' "server_target=%s rpc=%s qps=%s metadata=%r secure_mode=%s " - "print_response=%s" + "print_response=%s is_trusted_xds_server_experimental=%s" ), self.deployment_name, self.k8s_namespace.name, @@ -146,6 +147,7 @@ def run( # pylint: disable=arguments-differ metadata, secure_mode, print_response, + is_trusted_xds_server_experimental ) super().run() @@ -191,6 +193,7 @@ def run( # pylint: disable=arguments-differ config_mesh=config_mesh, generate_mesh_id=generate_mesh_id, print_response=print_response, + is_trusted_xds_server_experimental=is_trusted_xds_server_experimental, **self.deployment_args.as_dict(), ) diff --git a/framework/xds_c6n_testcase.py b/framework/xds_c6n_testcase.py deleted file mode 100644 index e69de29b..00000000 diff --git a/framework/xds_flags.py b/framework/xds_flags.py index f601f6e8..b727e0de 100644 --- a/framework/xds_flags.py +++ b/framework/xds_flags.py @@ -201,7 +201,7 @@ REGION = flags.DEFINE_string( "region", - default="us-central1", + default=None, help="The region for deployment", ) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index d640a214..24560209 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -40,13 +40,14 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director +import framework.infrastructure.traffic_director_cloudrun as td_cloudrun from framework.infrastructure.gcp.compute import ComputeV1 from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing from framework.test_app import client_app from framework.test_app import server_app -from framework.test_app.runners.c6n import c6n_xds_server_runner +from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner from framework.test_cases import base_testcase @@ -66,8 +67,8 @@ TrafficDirectorManager = traffic_director.TrafficDirectorManager TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager -CloudRunServerRunner = c6n_xds_server_runner.CloudRunServerRunner -TrafficDirectorCloudRunManager = traffic_director.TrafficDirectorCloudRunManager +CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner +TrafficDirectorCloudRunManager = td_cloudrun.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -1404,7 +1405,6 @@ def debug_cert(cert): class CloudRunXdsKubernetesTestCase(RegularXdsKubernetesTestCase): server_runner: CloudRunServerRunner td: TrafficDirectorCloudRunManager - region: str @classmethod def setUpClass(cls): @@ -1479,11 +1479,16 @@ def startTestServers( return test_servers def startTestClient( - self, test_server: XdsTestServer, **kwargs - ) -> XdsTestClient: - # logger.info("emchandwani : test server : %v",test_server) - logger.info("emchandwani : xds URI : %s",test_server.xds_uri) - return self._start_test_client(test_server.xds_uri,secure_mode=True, **kwargs) + self, + test_server: XdsTestServer, + **kwargs, + ) -> XdsTestClient: + return self._start_test_client( + server_target=test_server.xds_uri, + secure_mode=True, + **kwargs, + ) + def backend_service_add_serverless_neg_backends(self): logger.info("Creating serverless NEG") @@ -1492,7 +1497,77 @@ def backend_service_add_serverless_neg_backends(self): self.region, self.server_namespace, ) + self.neg=neg return neg + + def assertXdsConfigExists(self, test_client: XdsTestClient): + config = test_client.csds.fetch_client_status(log_level=logging.INFO) + self.assertIsNotNone(config) + seen = set() + want = frozenset( + [ + "listener_config", + "cluster_config", + "route_config", + ] + ) + for xds_config in config.xds_config: + seen.add(xds_config.WhichOneof("per_xds_config")) + for generic_xds_config in config.generic_xds_configs: + if re.search(r"\.Listener$", generic_xds_config.type_url): + seen.add("listener_config") + elif re.search( + r"\.RouteConfiguration$", generic_xds_config.type_url + ): + seen.add("route_config") + elif re.search(r"\.Cluster$", generic_xds_config.type_url): + seen.add("cluster_config") + + logger.debug( + "Received xDS config dump: %s", + json_format.MessageToJson(config, indent=2), + ) + self.assertSameElements(want, seen) + + def cleanup(self): + self.td.cleanup(force=self.force_cleanup) + self.server_runner.cleanup(force=self.force_cleanup) + self.client_runner.cleanup( + force=self.force_cleanup, force_namespace=self.force_cleanup + ) + def tearDown(self): logger.info("----- TestMethod %s teardown -----", self.test_name) + logger.debug("Getting pods restart times") + client_restarts: int = 0 + try: + client_restarts = self.client_runner.get_pod_restarts( + self.client_runner.deployment + ) + except (retryers.RetryError, k8s.NotFound) as e: + logger.exception(e) + + retryer = retryers.constant_retryer( + wait_fixed=_timedelta(seconds=10), + attempts=3, + log_level=logging.INFO, + ) + try: + retryer(self.cleanup) + except retryers.RetryError: + logger.exception("Got error during teardown") + finally: + logger.info("----- Test client/server logs -----") + self.client_runner.logs_explorer_run_history_links() + + # Fail if any of the pods restarted. + self.assertEqual( + client_restarts, + 0, + msg=( + "Client container unexpectedly restarted" + f" {client_restarts} times during test. In most cases, this" + " is caused by the test client app crash." + ), + ) diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml deleted file mode 100644 index 8c45c7ab..00000000 --- a/kubernetes-manifests/client.cloudrun.deployment.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: psm-grpc-client - labels: - app: psm-grpc-client -spec: - replicas: 1 - selector: - matchLabels: - app: psm-grpc-client - template: - metadata: - labels: - app: psm-grpc-client - spec: - % if service_account_name: - serviceAccountName: ${service_account_name} - % endif - containers: - - name: psm-grpc-client - image: ${image_name} - imagePullPolicy: Always - args: - - "--server=${server_target}" - env: - - name: GRPC_TRACE - value: "xds_client" - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER - value: "true" - - - name: CSM_WORKLOAD_NAME - value: psm-grpc-client - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CONTAINER_NAME - value: psm-grpc-client - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - initContainers: - - name: grpc-td-init - image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=default" - - "--xds-server-uri=trafficdirector.googleapis.com:443" - - "--config-mesh=${config_mesh}" - - "--is-trusted-xds-server-experimental=true" - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory - -... diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 0566575c..b5cfa2b9 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -41,6 +41,9 @@ spec: - "--qps=${qps}" - "--rpc=${rpc}" - "--metadata=${metadata}" + % if secure_mode: + - "--secure_mode=${secure_mode}" + % endif % if request_payload_size > 0: - "--request_payload_size=${request_payload_size}" % endif @@ -64,6 +67,22 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH + value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY + value: "true" + - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST + value: "true" + % if enable_dualstack: + - name: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS + value: "true" + % endif % if csm_workload_name: - name: CSM_WORKLOAD_NAME value: ${csm_workload_name} @@ -111,7 +130,9 @@ spec: % if generate_mesh_id: - "--generate-mesh-id-experimental" % endif + % if is_trusted_xds_server_experimental: - "--is-trusted-xds-server-experimental=true" + % endif resources: limits: cpu: 100m diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_cloud_run_server.py similarity index 95% rename from tests/baseline_test_c6n_server.py rename to tests/baseline_test_cloud_run_server.py index 9773893e..9295977f 100644 --- a/tests/baseline_test_c6n_server.py +++ b/tests/baseline_test_cloud_run_server.py @@ -49,7 +49,7 @@ def test_GKE_client_cloudrun_service(self): with self.subTest("7_start_test_client"): test_client: _XdsTestClient = self.startTestClient( - test_server, config_mesh=self.td.mesh.name + test_server, config_mesh=self.td.mesh.name,is_trusted_xds_server_experimental=True ) with self.subTest("8_test_client_xds_config_exists"): diff --git a/tests/bootstrap_generator_test.py b/tests/bootstrap_generator_test.py index 6e7bfa0a..c7b53ef3 100644 --- a/tests/bootstrap_generator_test.py +++ b/tests/bootstrap_generator_test.py @@ -38,9 +38,9 @@ # Constants GCR_PROD: Final[str] = "gcr.io/trafficdirector-prod/td-grpc-bootstrap" -GCR_TESTING: Final[str] = ( - "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" -) +GCR_TESTING: Final[ + str +] = "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" # Returns a list of bootstrap generator versions to be tested along with their @@ -230,4 +230,4 @@ def test_baseline_in_server_with_bootstrap_version(self, version, image): if __name__ == "__main__": - absltest.main() + absltest.main() \ No newline at end of file diff --git a/tests/url_map/affinity_test.py b/tests/url_map/affinity_test.py index e3e2102c..c180ccd1 100644 --- a/tests/url_map/affinity_test.py +++ b/tests/url_map/affinity_test.py @@ -101,9 +101,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher["defaultService"] = ( - GcpResourceManager().affinity_backend_service() - ) + path_matcher[ + "defaultService" + ] = GcpResourceManager().affinity_backend_service() return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -176,9 +176,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher["defaultService"] = ( - GcpResourceManager().affinity_backend_service() - ) + path_matcher[ + "defaultService" + ] = GcpResourceManager().affinity_backend_service() return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -273,4 +273,4 @@ def rpc_distribution_validate(self, test_client: XdsTestClient): # RPCs are sent to another backend if __name__ == "__main__": - absltest.main() + absltest.main() \ No newline at end of file From 64cd9f1f08d1b275192436221a00658ed96e9071 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 01:49:39 +0530 Subject: [PATCH 08/49] run black and isort --- bin/lib/common.py | 2 +- framework/infrastructure/gcp/api.py | 1 + framework/infrastructure/gcp/cloud_run.py | 8 +++++- framework/infrastructure/gcp/compute.py | 10 ++----- .../infrastructure/gcp/network_services.py | 2 +- framework/infrastructure/traffic_director.py | 2 +- .../traffic_director_cloudrun.py | 11 +++++--- .../cloud_run/cloud_run_base_runner.py | 5 ++-- .../cloud_run/cloud_run_xds_server_runner.py | 5 ++-- .../runners/k8s/k8s_xds_client_runner.py | 4 +-- framework/xds_k8s_testcase.py | 28 +++++++++---------- tests/baseline_test_cloud_run_server.py | 17 ++++++++--- tests/bootstrap_generator_test.py | 2 +- tests/url_map/affinity_test.py | 2 +- 14 files changed, 55 insertions(+), 44 deletions(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index 5da21bd7..06aee079 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,9 +22,9 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure.gcp import cloud_run from framework.infrastructure import gcp from framework.infrastructure import k8s +from framework.infrastructure.gcp import cloud_run from framework.test_app import client_app from framework.test_app import server_app from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index aafad314..a0be2db1 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -514,6 +514,7 @@ def wait_for_operation( class GcpStandardCloudApiResource(GcpProjectApiResource, metaclass=abc.ABCMeta): GLOBAL_LOCATION = "global" + def parent(self, location: Optional[str] = GLOBAL_LOCATION): if location is None: location = self.GLOBAL_LOCATION diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 204eeaa2..beb5c74e 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -42,7 +42,13 @@ def __init__(self, project: str, region: str): self._parent = f"projects/{self.project}/locations/{self.region}" self._service = None - def deploy_service(self, service_name: str, image_name: str, *,test_port: int = DEFAULT_TEST_PORT): + def deploy_service( + self, + service_name: str, + image_name: str, + *, + test_port: int = DEFAULT_TEST_PORT, + ): if not service_name: raise ValueError("service_name cannot be empty or None") if not image_name: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 92429c79..8288c098 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -212,10 +212,7 @@ def backend_service_patch_backends( is_cloudrun: bool = False, ): if is_cloudrun: - backend_list = [ - {"group": backend.url} - for backend in backends - ] + backend_list = [{"group": backend.url} for backend in backends] else: if max_rate_per_endpoint is None: max_rate_per_endpoint = 5 @@ -229,7 +226,6 @@ def backend_service_patch_backends( for backend in backends ] - request = {"backends": backend_list} if circuit_breakers: request["circuitBreakers"] = circuit_breakers @@ -572,9 +568,7 @@ def get_backend_service_backend_health(self, backend_service, backend): .execute() ) - def create_serverless_neg( - self, name: str, region: str, service_name: str - ): + def create_serverless_neg(self, name: str, region: str, service_name: str): """Creates a serverless NEG. Args: diff --git a/framework/infrastructure/gcp/network_services.py b/framework/infrastructure/gcp/network_services.py index d13e60fa..86d4a97a 100644 --- a/framework/infrastructure/gcp/network_services.py +++ b/framework/infrastructure/gcp/network_services.py @@ -474,4 +474,4 @@ def delete_http_route(self, name: str) -> bool: return self._delete_resource( collection=self._api_locations.httpRoutes(), full_name=self.resource_full_name(name, self.HTTP_ROUTES), - ) \ No newline at end of file + ) diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 5d6b2f2a..987cadb2 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -1286,4 +1286,4 @@ def _get_certificate_provider(cls): "certificateProviderInstance": { "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, }, - } \ No newline at end of file + } diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloudrun.py index 20ff2fa1..f6aa598f 100644 --- a/framework/infrastructure/traffic_director_cloudrun.py +++ b/framework/infrastructure/traffic_director_cloudrun.py @@ -13,10 +13,13 @@ # limitations under the License. import logging from typing import Optional, Sequence + +from framework.infrastructure import gcp from framework.infrastructure.gcp.network_security import NetworkSecurityV1Beta1 from framework.infrastructure.gcp.network_services import NetworkServicesV1Beta1 -from framework.infrastructure.traffic_director import TrafficDirectorAppNetManager -from framework.infrastructure import gcp +from framework.infrastructure.traffic_director import ( + TrafficDirectorAppNetManager, +) # _NetworkSecurityV1Beta1 = gcp.network_security.NetworkSecurityV1Beta1 ServerTlsPolicy = gcp.network_security.ServerTlsPolicy @@ -30,6 +33,7 @@ HttpRoute = gcp.network_services.HttpRoute Mesh = gcp.network_services.Mesh + class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): MESH_NAME = "grpc-mesh" SERVER_TLS_POLICY_NAME = "server-tls-policy" @@ -88,7 +92,6 @@ def backend_service_add_backends( *, circuit_breakers: Optional[dict[str, int]] = None, ): - new_backends = [] for backend in backends: new_backend = { @@ -116,4 +119,4 @@ def backend_service_add_backends( backend_service, backends, is_cloudrun=True, - ) \ No newline at end of file + ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index ab229fb5..27312838 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -111,7 +111,8 @@ def run(self, **kwargs): self._reset_state() self.time_start_requested = _datetime.now() self.current_revision = self.cloudrun_api_manager.deploy_service( - self.service_name, self.image_name, + self.service_name, + self.image_name, ) def _start_completed(self): @@ -137,4 +138,4 @@ def stop(self): except Exception as e: logger.warning( "Cloud Run service %s deletion failed: %s", self.service_name, e - ) \ No newline at end of file + ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 12b29865..4b85e4e4 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -20,7 +20,6 @@ import uuid from absl import flags - from typing_extensions import override from framework import xds_flags @@ -86,7 +85,7 @@ def run(self, **kwargs) -> List[XdsTestServer]: """Deploys and manages the xDS Test Server on Cloud Run.""" logger.info(self.service_name) logger.info(self.image_name) - + super().run(**kwargs) servers = [ XdsTestServer( @@ -105,6 +104,6 @@ def cleanup(self, *, force=False): try: if self.service: self.stop() - self.service_name=None + self.service_name = None finally: self._stop() diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index 25430bac..535c7b7b 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -131,7 +131,7 @@ def run( # pylint: disable=arguments-differ log_to_stdout: bool = False, request_payload_size: int = 0, response_payload_size: int = 0, - is_trusted_xds_server_experimental: bool=False + is_trusted_xds_server_experimental: bool = False, ) -> client_app.XdsTestClient: logger.info( ( @@ -147,7 +147,7 @@ def run( # pylint: disable=arguments-differ metadata, secure_mode, print_response, - is_trusted_xds_server_experimental + is_trusted_xds_server_experimental, ) super().run() diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 24560209..724c163e 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -40,8 +40,8 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director -import framework.infrastructure.traffic_director_cloudrun as td_cloudrun from framework.infrastructure.gcp.compute import ComputeV1 +import framework.infrastructure.traffic_director_cloudrun as td_cloudrun from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing @@ -1459,7 +1459,7 @@ def setupServerBackends( service_url = server_runner.get_service_url() self.td.backend_service_add_backends([service_url]) - def startTestServers( + def startTestServers( self, server_runner=None, **kwargs ) -> List[XdsTestServer]: logger.info(self.server_image) @@ -1479,16 +1479,15 @@ def startTestServers( return test_servers def startTestClient( - self, - test_server: XdsTestServer, - **kwargs, - ) -> XdsTestClient: - return self._start_test_client( - server_target=test_server.xds_uri, - secure_mode=True, - **kwargs, - ) - + self, + test_server: XdsTestServer, + **kwargs, + ) -> XdsTestClient: + return self._start_test_client( + server_target=test_server.xds_uri, + secure_mode=True, + **kwargs, + ) def backend_service_add_serverless_neg_backends(self): logger.info("Creating serverless NEG") @@ -1497,9 +1496,9 @@ def backend_service_add_serverless_neg_backends(self): self.region, self.server_namespace, ) - self.neg=neg + self.neg = neg return neg - + def assertXdsConfigExists(self, test_client: XdsTestClient): config = test_client.csds.fetch_client_status(log_level=logging.INFO) self.assertIsNotNone(config) @@ -1536,7 +1535,6 @@ def cleanup(self): force=self.force_cleanup, force_namespace=self.force_cleanup ) - def tearDown(self): logger.info("----- TestMethod %s teardown -----", self.test_name) logger.debug("Getting pods restart times") diff --git a/tests/baseline_test_cloud_run_server.py b/tests/baseline_test_cloud_run_server.py index 9295977f..e80de0be 100644 --- a/tests/baseline_test_cloud_run_server.py +++ b/tests/baseline_test_cloud_run_server.py @@ -23,7 +23,9 @@ _XdsTestClient = xds_k8s_testcase.XdsTestClient -class CloudRunServerBaselineTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): +class CloudRunServerBaselineTest( + xds_k8s_testcase.CloudRunXdsKubernetesTestCase +): def test_GKE_client_cloudrun_service(self): with self.subTest("0_create_mesh"): self.td.create_mesh() @@ -35,7 +37,10 @@ def test_GKE_client_cloudrun_service(self): neg = self.backend_service_add_serverless_neg_backends() with self.subTest("3_create_backend_service"): - self.td.create_backend_service(protocol=self.compute_v1.BackendServiceProtocol.HTTP2,is_cloudrun=True) + self.td.create_backend_service( + protocol=self.compute_v1.BackendServiceProtocol.HTTP2, + is_cloudrun=True, + ) with self.subTest("4_add_server_backends_to_backend_service"): neg_resource = self.compute_v1.GcpResource( @@ -45,11 +50,15 @@ def test_GKE_client_cloudrun_service(self): self.td.backend_service_add_backends([neg_resource], self.region) with self.subTest("5_create_grpc_route"): - self.td.create_grpc_route(self.server_xds_host, self.server_xds_port) + self.td.create_grpc_route( + self.server_xds_host, self.server_xds_port + ) with self.subTest("7_start_test_client"): test_client: _XdsTestClient = self.startTestClient( - test_server, config_mesh=self.td.mesh.name,is_trusted_xds_server_experimental=True + test_server, + config_mesh=self.td.mesh.name, + is_trusted_xds_server_experimental=True, ) with self.subTest("8_test_client_xds_config_exists"): diff --git a/tests/bootstrap_generator_test.py b/tests/bootstrap_generator_test.py index c7b53ef3..744e2999 100644 --- a/tests/bootstrap_generator_test.py +++ b/tests/bootstrap_generator_test.py @@ -230,4 +230,4 @@ def test_baseline_in_server_with_bootstrap_version(self, version, image): if __name__ == "__main__": - absltest.main() \ No newline at end of file + absltest.main() diff --git a/tests/url_map/affinity_test.py b/tests/url_map/affinity_test.py index c180ccd1..2bd93405 100644 --- a/tests/url_map/affinity_test.py +++ b/tests/url_map/affinity_test.py @@ -273,4 +273,4 @@ def rpc_distribution_validate(self, test_client: XdsTestClient): # RPCs are sent to another backend if __name__ == "__main__": - absltest.main() \ No newline at end of file + absltest.main() From 1a1c974ac5ba9992c55047d4d597e74f7c8bad33 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 02:11:39 +0530 Subject: [PATCH 09/49] pylint error --- bin/lib/common.py | 16 ++++++++-------- framework/infrastructure/gcp/compute.py | 11 +++++------ framework/infrastructure/traffic_director.py | 3 +-- .../infrastructure/traffic_director_cloudrun.py | 3 --- framework/xds_k8s_testcase.py | 8 ++++---- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index 06aee079..4131a0bc 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -47,7 +47,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner GammaServerRunner = gamma_server_runner.GammaServerRunner -CloudRunServerRunner=cloud_run_xds_server_runner.CloudRunServerRunner +CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner _XdsTestServer = server_app.XdsTestServer _XdsTestClient = client_app.XdsTestClient @@ -64,7 +64,7 @@ def gcp_api_manager(): @functools.cache def c6n_api_manager(): - return cloud_run.CloudRunApiManager() + return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, region=xds_flags.REGION.value) def td_attrs(): return dict( @@ -161,14 +161,14 @@ def make_server_runner( return server_runner(namespace, **runner_kwargs) -def make_c6n_server_runner() -> CloudRunServerRunner: +def make_cloud_run_server_runner() -> CloudRunServerRunner: # CloudRunServerRunner arguments. runner_kwargs = dict( - project = xds_flags.PROJECT.value, - service_name = xds_flags.SERVER_NAME.value, - image_name = xds_k8s_flags.SERVER_IMAGE.value, - network = xds_flags.NETWORK.value, - region = xds_flags.REGION.value, + project=xds_flags.PROJECT.value, + service_name=xds_flags.SERVER_NAME.value, + image_name=xds_k8s_flags.SERVER_IMAGE.value, + network=xds_flags.NETWORK.value, + region=xds_flags.REGION.value, ) server_runner = CloudRunServerRunner return server_runner(**runner_kwargs) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 8288c098..27770236 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -574,7 +574,8 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): Args: name: The name of the NEG. region: The region in which to create the NEG. - service_name: The name of the Cloud Run service. Format: "namespaces/{namespace}/services/{service}" + service_name: The name of the Cloud Run service. + service_name format is "namespaces/{namespace}/services/{service}" Returns: The NEG selfLink URL @@ -588,11 +589,9 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): try: logger.info("Creating serverless NEG %s in %s", name, region) - operation = ( - self.api.regionNetworkEndpointGroups() - .insert(project=self.project, region=region, body=neg_body) - .execute() - ) + self.api.regionNetworkEndpointGroups().insert( + project=self.project, region=region, body=neg_body + ).execute() neg = self.get_serverless_network_endpoint_group(name, region) logger.info(neg) return neg diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 987cadb2..e761cd88 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -14,8 +14,7 @@ import functools import logging import random -import re -from typing import Any, Dict, Final, List, Optional, Sequence +from typing import Any, Dict, Final, List, Optional import googleapiclient.errors from typing_extensions import TypeAlias diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloudrun.py index f6aa598f..1e63acbd 100644 --- a/framework/infrastructure/traffic_director_cloudrun.py +++ b/framework/infrastructure/traffic_director_cloudrun.py @@ -15,8 +15,6 @@ from typing import Optional, Sequence from framework.infrastructure import gcp -from framework.infrastructure.gcp.network_security import NetworkSecurityV1Beta1 -from framework.infrastructure.gcp.network_services import NetworkServicesV1Beta1 from framework.infrastructure.traffic_director import ( TrafficDirectorAppNetManager, ) @@ -85,7 +83,6 @@ def __init__( def backend_service_add_backends( self, backends: Sequence[str], - region: Optional[str] = None, balancing_mode: str = "CONNECTION", max_rate_per_endpoint: Optional[int] = None, capacity_scaler: float = 1.0, diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 724c163e..3573158c 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1450,17 +1450,18 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: def setupServerBackends( self, *, - wait_for_healthy_status=True, server_runner=None, max_rate_per_endpoint: Optional[int] = None, ): if server_runner is None: server_runner = self.server_runner service_url = server_runner.get_service_url() - self.td.backend_service_add_backends([service_url]) + self.td.backend_service_add_backends( + [service_url], max_rate_per_endpoint=max_rate_per_endpoint + ) def startTestServers( - self, server_runner=None, **kwargs + self, replica_count=1, server_runner=None, **kwargs ) -> List[XdsTestServer]: logger.info(self.server_image) if server_runner is None: @@ -1496,7 +1497,6 @@ def backend_service_add_serverless_neg_backends(self): self.region, self.server_namespace, ) - self.neg = neg return neg def assertXdsConfigExists(self, test_client: XdsTestClient): From 6d5e0de0693dac22d2f18327ad5d88abe37d288b Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 02:21:54 +0530 Subject: [PATCH 10/49] pylint error --- bin/lib/common.py | 3 ++- framework/xds_k8s_testcase.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index 4131a0bc..526ac348 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -64,7 +64,8 @@ def gcp_api_manager(): @functools.cache def c6n_api_manager(): - return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, region=xds_flags.REGION.value) + return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, + region=xds_flags.REGION.value) def td_attrs(): return dict( diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 3573158c..b0fa736a 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1450,6 +1450,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: def setupServerBackends( self, *, + wait_for_healthy_status=True, server_runner=None, max_rate_per_endpoint: Optional[int] = None, ): @@ -1459,6 +1460,10 @@ def setupServerBackends( self.td.backend_service_add_backends( [service_url], max_rate_per_endpoint=max_rate_per_endpoint ) + if wait_for_healthy_status: + self.td.wait_for_backends_healthy_status( + replica_count=server_runner.replica_count + ) def startTestServers( self, replica_count=1, server_runner=None, **kwargs From bc645774bca4ee1b8849a81c57de8a4e57118ee1 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 10:04:35 +0530 Subject: [PATCH 11/49] remove unwanted files --- kubernetes-manifests/eshita.yaml | 71 -------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 kubernetes-manifests/eshita.yaml diff --git a/kubernetes-manifests/eshita.yaml b/kubernetes-manifests/eshita.yaml deleted file mode 100644 index 1b61019e..00000000 --- a/kubernetes-manifests/eshita.yaml +++ /dev/null @@ -1,71 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: psm-grpc-client - labels: - app: psm-grpc-client -spec: - replicas: 1 - selector: - matchLabels: - app: psm-grpc-client - template: - metadata: - labels: - app: psm-grpc-client - spec: - containers: - - name: psm-grpc-client - image: us-docker.pkg.dev/grpc-testing/psm-interop/java-client:v1.63.x - imagePullPolicy: Always - args: - - "--xds_creds" - - "--target=xds:///psm-grpc-server" - env: - - name: GRPC_TRACE - value: "xds_client" - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER - value: "true" - - - name: CSM_WORKLOAD_NAME - value: psm-grpc-client - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CONTAINER_NAME - value: psm-grpc-client - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.pod.name=\$(POD_NAME),k8s.namespace.name=\$(NAMESPACE_NAME),k8s.container.name=\$(CONTAINER_NAME) - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - initContainers: - - name: grpc-td-init - image: us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:ad35743ff01c0dc16a785e421591d7d52f3cdda4 - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=default" - - "--config-mesh-experimental=emchandwani-psm-grpc-mesh-20250217-0927-24ueq" - - "--is-trusted-xds-server-experimental=true" - # - "--gcp-project=emchandwani-default" - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory -... \ No newline at end of file From b9be8ca31fc7f3b2fba4427d6a6450f65d573f7c Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Sat, 1 Mar 2025 00:10:33 +0530 Subject: [PATCH 12/49] address comments --- ...er_c6n.py => run_test_server_cloud_run.py} | 17 +-------- framework/infrastructure/gcp/cloud_run.py | 8 +++-- framework/infrastructure/gcp/compute.py | 4 +-- framework/infrastructure/traffic_director.py | 2 -- .../traffic_director_cloudrun.py | 36 ++----------------- .../cloud_run/cloud_run_xds_server_runner.py | 8 +++-- framework/xds_k8s_flags.py | 6 ---- framework/xds_k8s_testcase.py | 3 +- ...r.py => cloud_run_server_baseline_test.py} | 2 +- 9 files changed, 18 insertions(+), 68 deletions(-) rename bin/{run_test_server_c6n.py => run_test_server_cloud_run.py} (68%) rename tests/{baseline_test_cloud_run_server.py => cloud_run_server_baseline_test.py} (97%) diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_cloud_run.py similarity index 68% rename from bin/run_test_server_c6n.py rename to bin/run_test_server_cloud_run.py index c6b32253..d9576033 100644 --- a/bin/run_test_server_c6n.py +++ b/bin/run_test_server_cloud_run.py @@ -18,22 +18,7 @@ # Help. ./run.sh ./bin/run_test_server_cloud_run.py --help - - # Run modes. - ./run.sh ./bin/run_test_server_cloud_run.py --mode=app_net - ./run.sh ./bin/run_test_server_cloud_run.py --mode=secure - - # Gamma run mode: uses HTTPRoute by default. - ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma - - # Gamma run mode: use GRPCRoute. - ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --gamma_route_kind=grpc - - # Running multipler server replicas. - ./run.sh ./bin/run_test_server_cloud_run.py --server_replica_count=3 - - # Cleanup: make sure to set the same mode used to create. - ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --cmd=cleanup + """ import logging diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index beb5c74e..68accf5b 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -1,4 +1,4 @@ -# Copyright 2024 gRPC authors. +# Copyright 2025 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,9 +20,11 @@ logger = logging.getLogger(__name__) +DEFAULT_TEST_PORT = 8080 +DEFAULT_TIMEOUT = 600 + class CloudRunApiManager(GcpProjectApiResource, metaclass=abc.ABCMeta): - DEFAULT_TEST_PORT = 8080 project: str region: str _parent: str @@ -77,7 +79,7 @@ def deploy_service( try: operation = self._client.create_service(request=request) - self._service = operation.result(timeout=600) + self._service = operation.result(timeout=DEFAULT_TIMEOUT) logger.info("Deployed service: %s", self._service.uri) return self._service.uri except Exception as e: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 27770236..65e8a15d 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -162,7 +162,7 @@ def create_backend_service_traffic_director( "protocol": protocol.name, } # If it is not for cloud run, add heath check since cloud run does not - # support health check. + # support health check. if not is_cloudrun: body["healthChecks"] = [health_check.url] @@ -736,8 +736,6 @@ def _execute( # pylint: disable=arguments-differ ) request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header request.add_response_callback(self._log_debug_header) - logger.info("Executing request: %s", request) - logger.info(request.to_json()) operation = request.execute(num_retries=self._GCP_API_RETRIES) logger.debug("Operation %s", operation) return self._wait(operation["name"], timeout_sec) diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index e761cd88..b51bac14 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -265,8 +265,6 @@ def create_backend_service( ) self.backend_service = resource self.backend_service_protocol = protocol - logger.info("backend service protocol") - logger.info(self.backend_service_protocol) def load_backend_service(self): name = self.make_resource_name(self.BACKEND_SERVICE_NAME) diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloudrun.py index 1e63acbd..a96aba9c 100644 --- a/framework/infrastructure/traffic_director_cloudrun.py +++ b/framework/infrastructure/traffic_director_cloudrun.py @@ -19,29 +19,12 @@ TrafficDirectorAppNetManager, ) -# _NetworkSecurityV1Beta1 = gcp.network_security.NetworkSecurityV1Beta1 -ServerTlsPolicy = gcp.network_security.ServerTlsPolicy -ClientTlsPolicy = gcp.network_security.ClientTlsPolicy -AuthorizationPolicy = gcp.network_security.AuthorizationPolicy - -# Network Services -# _NetworkServicesV1Beta1 = gcp.network_services.NetworkServicesV1Beta1 -EndpointPolicy = gcp.network_services.EndpointPolicy GrpcRoute = gcp.network_services.GrpcRoute -HttpRoute = gcp.network_services.HttpRoute Mesh = gcp.network_services.Mesh class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): MESH_NAME = "grpc-mesh" - SERVER_TLS_POLICY_NAME = "server-tls-policy" - CLIENT_TLS_POLICY_NAME = "client-tls-policy" - AUTHZ_POLICY_NAME = "authz-policy" - ENDPOINT_POLICY = "endpoint-policy" - CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" - - # netsec: _NetworkSecurityV1Beta1 - # netsvc: _NetworkServicesV1Beta1 def __init__( self, @@ -64,22 +47,10 @@ def __init__( enable_dualstack=enable_dualstack, ) - # API - # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) - # self.netsvc = _NetworkSecurityV1Beta1(gcp_api_manager, project) - # Managed resources - # TODO(gnossen) PTAL at the pylint error self.grpc_route: Optional[GrpcRoute] = None - self.http_route: Optional[HttpRoute] = None self.mesh: Optional[Mesh] = None - # Managed resources - self.server_tls_policy: Optional[ServerTlsPolicy] = None - self.client_tls_policy: Optional[ClientTlsPolicy] = None - self.authz_policy: Optional[AuthorizationPolicy] = None - self.endpoint_policy: Optional[EndpointPolicy] = None - def backend_service_add_backends( self, backends: Sequence[str], @@ -103,17 +74,16 @@ def backend_service_add_backends( new_backends.append(new_backend) - # self.backends.update(new_backends) - backend_service = self.backend_service + # backend_service = self.backend_service logging.info( "Adding backends to Backend Service %s: %r", - backend_service.name, + self.backend_service.name, new_backends, ) self.compute.backend_service_patch_backends( - backend_service, + self.backend_service, backends, is_cloudrun=True, ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 4b85e4e4..c6d5ffe4 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -83,8 +83,11 @@ def _reset_state(self): @override def run(self, **kwargs) -> List[XdsTestServer]: """Deploys and manages the xDS Test Server on Cloud Run.""" - logger.info(self.service_name) - logger.info(self.image_name) + logger.info( + "Starting cloud run server with service %s and image %s", + self.service_name, + self.image_name, + ) super().run(**kwargs) servers = [ @@ -105,5 +108,6 @@ def cleanup(self, *, force=False): if self.service: self.stop() self.service_name = None + self.service = None finally: self._stop() diff --git a/framework/xds_k8s_flags.py b/framework/xds_k8s_flags.py index 203aaac2..8070642b 100644 --- a/framework/xds_k8s_flags.py +++ b/framework/xds_k8s_flags.py @@ -33,12 +33,6 @@ help="Traffic Director gRPC Bootstrap Docker image", ) -CLOUDRUN_CLIENT_IMAGE = flags.DEFINE_string( - "cloudrun_client_image", - default=None, - help="Cloudrun Client Docker image name", -) - # Test app SERVER_IMAGE = flags.DEFINE_string( "server_image", default=None, help="Server Docker image name" diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index b0fa736a..49580972 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1468,7 +1468,6 @@ def setupServerBackends( def startTestServers( self, replica_count=1, server_runner=None, **kwargs ) -> List[XdsTestServer]: - logger.info(self.server_image) if server_runner is None: self.server_runner = CloudRunServerRunner( project=self.project, @@ -1495,7 +1494,7 @@ def startTestClient( **kwargs, ) - def backend_service_add_serverless_neg_backends(self): + def backendServiceAddServerlessNegBackends(self): logger.info("Creating serverless NEG") neg = self.compute_v1.create_serverless_neg( self.server_namespace, diff --git a/tests/baseline_test_cloud_run_server.py b/tests/cloud_run_server_baseline_test.py similarity index 97% rename from tests/baseline_test_cloud_run_server.py rename to tests/cloud_run_server_baseline_test.py index e80de0be..786f9cfb 100644 --- a/tests/baseline_test_cloud_run_server.py +++ b/tests/cloud_run_server_baseline_test.py @@ -34,7 +34,7 @@ def test_GKE_client_cloudrun_service(self): test_server: _XdsTestServer = self.startTestServers()[0] with self.subTest("2_create_serverless_neg"): - neg = self.backend_service_add_serverless_neg_backends() + neg = self.backendServiceAddServerlessNegBackends() with self.subTest("3_create_backend_service"): self.td.create_backend_service( From 521bf5e9e9bf8f6eb0c232e4a6d4beb32b7414f2 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Sat, 1 Mar 2025 00:15:28 +0530 Subject: [PATCH 13/49] pylint --- bin/run_test_server_cloud_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_test_server_cloud_run.py b/bin/run_test_server_cloud_run.py index d9576033..5d9269a3 100644 --- a/bin/run_test_server_cloud_run.py +++ b/bin/run_test_server_cloud_run.py @@ -18,7 +18,7 @@ # Help. ./run.sh ./bin/run_test_server_cloud_run.py --help - + """ import logging From 8455a1e28375b28c086c0c2f7ab39cabe1cb0f9f Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 6 Mar 2025 21:57:12 +0530 Subject: [PATCH 14/49] address comments --- bin/lib/common.py | 2 +- framework/infrastructure/gcp/compute.py | 27 ++++++++++--------- framework/infrastructure/traffic_director.py | 4 +-- ...udrun.py => traffic_director_cloud_run.py} | 2 +- .../cloud_run/cloud_run_base_runner.py | 12 ++++----- .../cloud_run/cloud_run_xds_server_runner.py | 2 +- framework/xds_k8s_testcase.py | 21 +++++---------- tests/cloud_run_server_baseline_test.py | 8 +++--- 8 files changed, 36 insertions(+), 42 deletions(-) rename framework/infrastructure/{traffic_director_cloudrun.py => traffic_director_cloud_run.py} (98%) diff --git a/bin/lib/common.py b/bin/lib/common.py index 526ac348..bc24b938 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -63,7 +63,7 @@ def gcp_api_manager(): @functools.cache -def c6n_api_manager(): +def cloud_run_api_manager(): return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, region=xds_flags.REGION.value) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 65e8a15d..5897a1fe 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -152,7 +152,7 @@ def create_backend_service_traffic_director( locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, enable_dualstack: bool = False, - is_cloudrun: bool = False, + is_cloud_run: bool = False, ) -> "GcpResource": if not isinstance(protocol, self.BackendServiceProtocol): raise TypeError(f"Unexpected Backend Service protocol: {protocol}") @@ -163,7 +163,7 @@ def create_backend_service_traffic_director( } # If it is not for cloud run, add heath check since cloud run does not # support health check. - if not is_cloudrun: + if not is_cloud_run: body["healthChecks"] = [health_check.url] # If add dualstack support is specified True, config the backend service @@ -209,9 +209,9 @@ def backend_service_patch_backends( max_rate_per_endpoint: Optional[int] = None, *, circuit_breakers: Optional[dict[str, int]] = None, - is_cloudrun: bool = False, + is_cloud_run: bool = False, ): - if is_cloudrun: + if is_cloud_run: backend_list = [{"group": backend.url} for backend in backends] else: if max_rate_per_endpoint is None: @@ -600,7 +600,7 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): logger.exception("Error creating serverless NEG: %s", e) raise - def delete_serverless_neg(self, name: str, zone: str): + def delete_serverless_neg(self, name: str, region: str): """Deletes a serverless NEG. Args: @@ -608,11 +608,11 @@ def delete_serverless_neg(self, name: str, zone: str): zone: The zone of the NEG. """ try: - logger.info("Deleting serverless NEG %s in %s", name, zone) + logger.info("Deleting serverless NEG %s in %s", name, region) operation = ( - self.api.networkEndpointGroups() + self.api.regionNetworkEndpointGroups() .delete( - project=self.project, zone=zone, networkEndpointGroup=name + project=self.project, region=region, networkEndpointGroup=name ) .execute() ) @@ -625,7 +625,7 @@ def delete_serverless_neg(self, name: str, zone: str): logger.debug( "NEG %s not found in zone %s. Skipping deletion.", name, - zone, + region, ) return logger.exception("Error deleting serverless NEG: %s", error) @@ -635,12 +635,15 @@ def delete_serverless_neg(self, name: str, zone: str): raise def get_serverless_network_endpoint_group(self, name, region): - neg = ( + try: + neg = ( self.api.regionNetworkEndpointGroups() .get(project=self.project, networkEndpointGroup=name, region=region) .execute() - ) - # TODO(sergiitk): dataclass + ) + except Exception as e: + logger.exception("Error getting serverless NEG: %s", e) + raise return neg def _get_resource( diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index b51bac14..7343a0de 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -245,7 +245,7 @@ def create_backend_service( affinity_header: Optional[str] = None, locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, - is_cloudrun: Optional[bool] = False, + is_cloud_run: Optional[bool] = False, ): if protocol is None: protocol = _BackendGRPC @@ -261,7 +261,7 @@ def create_backend_service( locality_lb_policies=locality_lb_policies, outlier_detection=outlier_detection, enable_dualstack=self.enable_dualstack, - is_cloudrun=is_cloudrun, + is_cloud_run=is_cloud_run, ) self.backend_service = resource self.backend_service_protocol = protocol diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloud_run.py similarity index 98% rename from framework/infrastructure/traffic_director_cloudrun.py rename to framework/infrastructure/traffic_director_cloud_run.py index a96aba9c..d6cc21a9 100644 --- a/framework/infrastructure/traffic_director_cloudrun.py +++ b/framework/infrastructure/traffic_director_cloud_run.py @@ -85,5 +85,5 @@ def backend_service_add_backends( self.compute.backend_service_patch_backends( self.backend_service, backends, - is_cloudrun=True, + is_cloud_run=True, ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 27312838..da22f04c 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -1,4 +1,4 @@ -# Copyright 2022 gRPC authors. +# Copyright 2025 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -88,11 +88,11 @@ def __init__( # Highlighter. self._highlighter = _HighlighterYaml() - self._initalize_cloudrun_api_manager() + self._initalize_cloud_run_api_manager() - def _initalize_cloudrun_api_manager(self): + def _initalize_cloud_run_api_manager(self): """Initializes the CloudRunApiManager.""" - self.cloudrun_api_manager = cloud_run.CloudRunApiManager( + self.cloud_run_api_manager = cloud_run.CloudRunApiManager( project=self.project, region=self.region ) @@ -110,7 +110,7 @@ def run(self, **kwargs): self._reset_state() self.time_start_requested = _datetime.now() - self.current_revision = self.cloudrun_api_manager.deploy_service( + self.current_revision = self.cloud_run_api_manager.deploy_service( self.service_name, self.image_name, ) @@ -133,7 +133,7 @@ def stop(self): """Deletes Cloud Run Service""" logger.info("Deleting Cloud Run service: %s", self.service_name) try: - self.cloudrun_api_manager.delete_service(self.service_name) + self.cloud_run_api_manager.delete_service(self.service_name) logger.info("Cloud Run service %s deleted", self.service_name) except Exception as e: logger.warning( diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index c6d5ffe4..747e4f0d 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -100,7 +100,7 @@ def run(self, **kwargs) -> List[XdsTestServer]: return servers def get_service_url(self): - return self.cloudrun_api_manager.get_service_url() + return self.cloud_run_api_manager.get_service_url() @override def cleanup(self, *, force=False): diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 49580972..3ad25852 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -41,7 +41,7 @@ from framework.infrastructure import k8s from framework.infrastructure import traffic_director from framework.infrastructure.gcp.compute import ComputeV1 -import framework.infrastructure.traffic_director_cloudrun as td_cloudrun +import framework.infrastructure.traffic_director_cloud_run as td_cloud_run from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing @@ -68,7 +68,7 @@ TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner -TrafficDirectorCloudRunManager = td_cloudrun.TrafficDirectorCloudRunManager +TrafficDirectorCloudRunManager = td_cloud_run.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -1402,10 +1402,10 @@ def debug_cert(cert): return f"sha1={sha1.hexdigest()}, len={len(cert)}" -class CloudRunXdsKubernetesTestCase(RegularXdsKubernetesTestCase): +class CloudRunXdsKubernetesTestCase(SecurityXdsKubernetesTestCase): server_runner: CloudRunServerRunner td: TrafficDirectorCloudRunManager - + neg:Any @classmethod def setUpClass(cls): super().setUpClass() @@ -1483,17 +1483,6 @@ def startTestServers( ) return test_servers - def startTestClient( - self, - test_server: XdsTestServer, - **kwargs, - ) -> XdsTestClient: - return self._start_test_client( - server_target=test_server.xds_uri, - secure_mode=True, - **kwargs, - ) - def backendServiceAddServerlessNegBackends(self): logger.info("Creating serverless NEG") neg = self.compute_v1.create_serverless_neg( @@ -1501,6 +1490,7 @@ def backendServiceAddServerlessNegBackends(self): self.region, self.server_namespace, ) + self.neg=neg return neg def assertXdsConfigExists(self, test_client: XdsTestClient): @@ -1534,6 +1524,7 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): def cleanup(self): self.td.cleanup(force=self.force_cleanup) + self.compute_v1.delete_serverless_neg(self.neg["name"],self.region) self.server_runner.cleanup(force=self.force_cleanup) self.client_runner.cleanup( force=self.force_cleanup, force_namespace=self.force_cleanup diff --git a/tests/cloud_run_server_baseline_test.py b/tests/cloud_run_server_baseline_test.py index 786f9cfb..f67a3a3b 100644 --- a/tests/cloud_run_server_baseline_test.py +++ b/tests/cloud_run_server_baseline_test.py @@ -26,11 +26,11 @@ class CloudRunServerBaselineTest( xds_k8s_testcase.CloudRunXdsKubernetesTestCase ): - def test_GKE_client_cloudrun_service(self): + def test_GKE_client_cloud_run_service(self): with self.subTest("0_create_mesh"): self.td.create_mesh() - with self.subTest("1_start_cloudrun_test_server"): + with self.subTest("1_start_cloud_run_test_server"): test_server: _XdsTestServer = self.startTestServers()[0] with self.subTest("2_create_serverless_neg"): @@ -39,7 +39,7 @@ def test_GKE_client_cloudrun_service(self): with self.subTest("3_create_backend_service"): self.td.create_backend_service( protocol=self.compute_v1.BackendServiceProtocol.HTTP2, - is_cloudrun=True, + is_cloud_run=True, ) with self.subTest("4_add_server_backends_to_backend_service"): @@ -55,7 +55,7 @@ def test_GKE_client_cloudrun_service(self): ) with self.subTest("7_start_test_client"): - test_client: _XdsTestClient = self.startTestClient( + test_client: _XdsTestClient = self.startSecureTestClient( test_server, config_mesh=self.td.mesh.name, is_trusted_xds_server_experimental=True, From d04238eaf8d79205199b0254458cefee3aa2742e Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 6 Mar 2025 22:03:28 +0530 Subject: [PATCH 15/49] pylint --- framework/infrastructure/gcp/compute.py | 14 ++++++++++---- framework/xds_k8s_testcase.py | 9 +++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 5897a1fe..2c04509d 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -612,7 +612,9 @@ def delete_serverless_neg(self, name: str, region: str): operation = ( self.api.regionNetworkEndpointGroups() .delete( - project=self.project, region=region, networkEndpointGroup=name + project=self.project, + region=region, + networkEndpointGroup=name, ) .execute() ) @@ -637,9 +639,13 @@ def delete_serverless_neg(self, name: str, region: str): def get_serverless_network_endpoint_group(self, name, region): try: neg = ( - self.api.regionNetworkEndpointGroups() - .get(project=self.project, networkEndpointGroup=name, region=region) - .execute() + self.api.regionNetworkEndpointGroups() + .get( + project=self.project, + networkEndpointGroup=name, + region=region, + ) + .execute() ) except Exception as e: logger.exception("Error getting serverless NEG: %s", e) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 3ad25852..b4f7b066 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1405,7 +1405,8 @@ def debug_cert(cert): class CloudRunXdsKubernetesTestCase(SecurityXdsKubernetesTestCase): server_runner: CloudRunServerRunner td: TrafficDirectorCloudRunManager - neg:Any + neg: Any + @classmethod def setUpClass(cls): super().setUpClass() @@ -1466,7 +1467,7 @@ def setupServerBackends( ) def startTestServers( - self, replica_count=1, server_runner=None, **kwargs + self, server_runner=None ) -> List[XdsTestServer]: if server_runner is None: self.server_runner = CloudRunServerRunner( @@ -1490,7 +1491,7 @@ def backendServiceAddServerlessNegBackends(self): self.region, self.server_namespace, ) - self.neg=neg + self.neg = neg return neg def assertXdsConfigExists(self, test_client: XdsTestClient): @@ -1524,7 +1525,7 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): def cleanup(self): self.td.cleanup(force=self.force_cleanup) - self.compute_v1.delete_serverless_neg(self.neg["name"],self.region) + self.compute_v1.delete_serverless_neg(self.neg["name"], self.region) self.server_runner.cleanup(force=self.force_cleanup) self.client_runner.cleanup( force=self.force_cleanup, force_namespace=self.force_cleanup From 10e95894bd86f0e2f4b51ea37a4b9c07532b73da Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 6 Mar 2025 22:05:56 +0530 Subject: [PATCH 16/49] black --- framework/xds_k8s_testcase.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index b4f7b066..95583591 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1466,9 +1466,7 @@ def setupServerBackends( replica_count=server_runner.replica_count ) - def startTestServers( - self, server_runner=None - ) -> List[XdsTestServer]: + def startTestServers(self, server_runner=None) -> List[XdsTestServer]: if server_runner is None: self.server_runner = CloudRunServerRunner( project=self.project, From 91ddaccd81d891bf3525fdeadba995d0e519ddd7 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 10 Mar 2025 15:43:09 +0530 Subject: [PATCH 17/49] working cloudrun_server test --- framework/infrastructure/gcp/api.py | 18 ++- framework/infrastructure/gcp/cloud_run.py | 117 ++++++++++-------- framework/infrastructure/gcp/compute.py | 35 ++---- .../cloud_run/cloud_run_base_runner.py | 4 +- .../cloud_run/cloud_run_xds_server_runner.py | 15 +-- framework/xds_k8s_testcase.py | 50 +++----- 6 files changed, 113 insertions(+), 126 deletions(-) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index a0be2db1..2055c02e 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -190,6 +190,16 @@ def networksecurity(self, version): raise NotImplementedError(f"Network Security {version} not supported") + @functools.lru_cache(None) + def cloudrun(self, version): + api_name = "run" + if version == "v2": + return self._build_from_discovery_v2( + api_name, + version, + ) + raise NotImplementedError(f"Cloud run {version} not supported") + @functools.lru_cache(None) def networkservices(self, version): api_name = "networkservices" @@ -520,11 +530,11 @@ def parent(self, location: Optional[str] = GLOBAL_LOCATION): location = self.GLOBAL_LOCATION return f"projects/{self.project}/locations/{location}" - def resource_full_name(self, name, collection_name): - return f"{self.parent()}/{collection_name}/{name}" + def resource_full_name(self, name, collection_name,location: Optional[str] = None): + return f"{self.parent(location)}/{collection_name}/{name}" def _create_resource( - self, collection: discovery.Resource, body: dict, **kwargs + self, collection: discovery.Resource, body: dict,location:Optional[str]=None, **kwargs ): logger.info( "Creating %s resource:\n%s", @@ -532,7 +542,7 @@ def _create_resource( self.resource_pretty_format(body), ) create_req = collection.create( - parent=self.parent(), body=body, **kwargs + parent=self.parent(location), body=body, **kwargs ) self._execute(create_req) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 68accf5b..feb6adac 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -13,10 +13,8 @@ # limitations under the License. import abc import logging - -from google.cloud import run_v2 - -from framework.infrastructure.gcp.api import GcpProjectApiResource +from framework.infrastructure import gcp +from googleapiclient import discovery logger = logging.getLogger(__name__) @@ -24,25 +22,60 @@ DEFAULT_TIMEOUT = 600 -class CloudRunApiManager(GcpProjectApiResource, metaclass=abc.ABCMeta): +class CloudRunApiManager(gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta): project: str region: str _parent: str - _client: run_v2.ServicesClient - _service: run_v2.Service + service :discovery.Resource + api_manager: gcp.api.GcpApiManager + discovery_url="https://run.googleapis.com/$discovery/rest?" - def __init__(self, project: str, region: str): + def __init__(self, project: str,region: str): if not project: raise ValueError("Project ID cannot be empty or None.") if not region: raise ValueError("Region cannot be empty or None.") - + self.api_manager = gcp.api.GcpApiManager(v2_discovery_uri="https://run.googleapis.com/$discovery/rest?") self.project = project self.region = region - client_options = {"api_endpoint": f"{self.region}-run.googleapis.com"} - self._client = run_v2.ServicesClient(client_options=client_options) + service:discovery.Resource=self.api_manager.cloudrun("v2") + self.service=service self._parent = f"projects/{self.project}/locations/{self.region}" - self._service = None + super().__init__(self.service, project) + + @property + def api_name(self) -> str: + """Returns the API name for Cloud Run.""" + return "run" + + @property + def api_version(self) -> str: + """Returns the API version for Cloud Run.""" + return "v2" + + def create_cloud_run_resource(self,service:discovery.Resource,service_name:str,body:dict): + return self._create_resource( + collection=service.projects().locations().services(), + body=body, + serviceId=service_name, + location=self.region + ) + + def get_cloud_run_resource(self,service:discovery.Resource,service_name:str): + return self._get_resource( + collection=service.projects().locations().services(), + full_name=self.resource_full_name(service_name,"services",self.region) + ) + + def get_service_uri(self,service_name:str)->str: + response=self.get_cloud_run_resource(self.service,service_name) + return response.get('uri') + + def delete_cloud_run_resource(self,service:discovery.Resource,service_name:str): + return self._delete_resource( + collection=service.projects().locations().services(), + full_name=self.resource_full_name(service_name,"services",self.region) + ) def deploy_service( self, @@ -55,50 +88,32 @@ def deploy_service( raise ValueError("service_name cannot be empty or None") if not image_name: raise ValueError("image_name cannot be empty or None") - service_name = service_name[:49] - - service = run_v2.Service( - template=run_v2.RevisionTemplate( - containers=[ - run_v2.Container( - image=image_name, - ports=[ - run_v2.ContainerPort( - name="h2c", - container_port=test_port, - ), - ], - ) - ] - ), - ) - - request = run_v2.CreateServiceRequest( - parent=self._parent, service=service, service_id=service_name - ) - + try: - operation = self._client.create_service(request=request) - self._service = operation.result(timeout=DEFAULT_TIMEOUT) - logger.info("Deployed service: %s", self._service.uri) - return self._service.uri - except Exception as e: - logger.exception("Error deploying service: %s", e) - raise + service_body = {} + service_body={ + "launch_stage":"alpha", + "template": + { + "containers": [ + { + "image": image_name, + "ports": [{"containerPort": test_port, "name": "h2c"}], + } + ], + }, + } - def get_service_url(self): - if self._service is None: - raise RuntimeError("Cloud Run service not deployed yet.") - return self._service.uri + self.create_cloud_run_resource(self.service,service_name,service_body) + logger.info("Deploying Cloud Run service '%s'", service_name) + return self.get_service_uri(service_name) + + except Exception as e: # noqa pylint: disable=broad-except + logger.exception("Error deploying Cloud Run service: %s", e) def delete_service(self, service_name: str): try: - request = run_v2.DeleteServiceRequest( - name=f"{self._parent}/services/{service_name}" - ) - operation = self._client.delete_service(request=request) - operation.result(timeout=300) - logger.info("Deleted service: %s", service_name) + self.delete_cloud_run_resource(self.service,service_name) except Exception as e: logger.exception("Error deleting service: %s", e) raise diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 2c04509d..c52cea35 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -593,7 +593,7 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): project=self.project, region=region, body=neg_body ).execute() neg = self.get_serverless_network_endpoint_group(name, region) - logger.info(neg) + logger.info("Created serverless neg %s ", neg) return neg except Exception as e: @@ -609,19 +609,12 @@ def delete_serverless_neg(self, name: str, region: str): """ try: logger.info("Deleting serverless NEG %s in %s", name, region) - operation = ( - self.api.regionNetworkEndpointGroups() - .delete( + self.api.regionNetworkEndpointGroups().delete( project=self.project, region=region, networkEndpointGroup=name, - ) - .execute() - ) - self._wait( - operation["name"], self._WAIT_FOR_OPERATION_SEC - ) # Wait for operation completion - + ).execute() + logger.info("Deleted serverless neg : %s", name) except googleapiclient.errors.HttpError as error: if error.resp.status == 404: # NEG not found logger.debug( @@ -637,19 +630,15 @@ def delete_serverless_neg(self, name: str, region: str): raise def get_serverless_network_endpoint_group(self, name, region): - try: - neg = ( - self.api.regionNetworkEndpointGroups() - .get( - project=self.project, - networkEndpointGroup=name, - region=region, - ) - .execute() + neg = ( + self.api.regionNetworkEndpointGroups() + .get( + project=self.project, + networkEndpointGroup=name, + region=region, ) - except Exception as e: - logger.exception("Error getting serverless NEG: %s", e) - raise + .execute() + ) return neg def _get_resource( diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index da22f04c..234959e8 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -134,8 +134,8 @@ def stop(self): logger.info("Deleting Cloud Run service: %s", self.service_name) try: self.cloud_run_api_manager.delete_service(self.service_name) - logger.info("Cloud Run service %s deleted", self.service_name) - except Exception as e: + logger.info("Deleted cloud run service: %s", self.service_name) + except Exception as e: # noqa pylint: disable=broad-except logger.warning( "Cloud Run service %s deletion failed: %s", self.service_name, e ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 747e4f0d..48495d2a 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -17,14 +17,8 @@ import dataclasses import logging from typing import List, Optional -import uuid - -from absl import flags from typing_extensions import override -from framework import xds_flags -from framework import xds_k8s_flags -from framework.infrastructure.gcp import cloud_run from framework.test_app.runners.cloud_run import cloud_run_base_runner from framework.test_app.server_app import XdsTestServer @@ -100,14 +94,13 @@ def run(self, **kwargs) -> List[XdsTestServer]: return servers def get_service_url(self): - return self.cloud_run_api_manager.get_service_url() + return self.cloud_run_api_manager.get_service_uri(self.service_name) @override def cleanup(self, *, force=False): try: - if self.service: - self.stop() - self.service_name = None - self.service = None + self.stop() + self.service_name = None + self.service = None finally: self._stop() diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 95583591..b858416a 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -558,6 +558,12 @@ def _assertRpcsEventuallyGoToGivenServers( f"Unexpected server {server_hostname} received RPCs", ) + def assertEDSConfigExists(self, config:ClientConfig, seen:set , want:frozenset): + want=want.union(["endpoint_config"]) + for generic_xds_config in config.generic_xds_configs: + if re.search(r"\.ClusterLoadAssignment$", generic_xds_config.type_url): + seen.add("endpoint_config") + def assertXdsConfigExists(self, test_client: XdsTestClient): config = test_client.csds.fetch_client_status(log_level=logging.INFO) self.assertIsNotNone(config) @@ -567,7 +573,6 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): "listener_config", "cluster_config", "route_config", - "endpoint_config", ] ) for xds_config in config.xds_config: @@ -581,10 +586,9 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): seen.add("route_config") elif re.search(r"\.Cluster$", generic_xds_config.type_url): seen.add("cluster_config") - elif re.search( - r"\.ClusterLoadAssignment$", generic_xds_config.type_url - ): - seen.add("endpoint_config") + + self.assertEDSConfigExists(config,seen,want) + logger.debug( "Received xDS config dump: %s", json_format.MessageToJson(config, indent=2), @@ -1483,48 +1487,24 @@ def startTestServers(self, server_runner=None) -> List[XdsTestServer]: return test_servers def backendServiceAddServerlessNegBackends(self): + name=self.td.make_resource_name("neg") logger.info("Creating serverless NEG") neg = self.compute_v1.create_serverless_neg( - self.server_namespace, + name, self.region, self.server_namespace, ) self.neg = neg return neg - def assertXdsConfigExists(self, test_client: XdsTestClient): - config = test_client.csds.fetch_client_status(log_level=logging.INFO) - self.assertIsNotNone(config) - seen = set() - want = frozenset( - [ - "listener_config", - "cluster_config", - "route_config", - ] - ) - for xds_config in config.xds_config: - seen.add(xds_config.WhichOneof("per_xds_config")) - for generic_xds_config in config.generic_xds_configs: - if re.search(r"\.Listener$", generic_xds_config.type_url): - seen.add("listener_config") - elif re.search( - r"\.RouteConfiguration$", generic_xds_config.type_url - ): - seen.add("route_config") - elif re.search(r"\.Cluster$", generic_xds_config.type_url): - seen.add("cluster_config") - - logger.debug( - "Received xDS config dump: %s", - json_format.MessageToJson(config, indent=2), - ) - self.assertSameElements(want, seen) + def assertEDSConfigExists(self,config,seen,want): + """No-op for Cloud Run as EDS is not required.""" + _ = (seen, config, want) def cleanup(self): + self.server_runner.cleanup(force=self.force_cleanup) self.td.cleanup(force=self.force_cleanup) self.compute_v1.delete_serverless_neg(self.neg["name"], self.region) - self.server_runner.cleanup(force=self.force_cleanup) self.client_runner.cleanup( force=self.force_cleanup, force_namespace=self.force_cleanup ) From bc72f6384e6b41a31fb4b5c2f2c6329250bda854 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 10 Mar 2025 17:49:55 +0530 Subject: [PATCH 18/49] formatting --- framework/infrastructure/gcp/api.py | 16 ++- framework/infrastructure/gcp/cloud_run.py | 101 +++++++++++------- framework/infrastructure/gcp/compute.py | 8 +- .../cloud_run/cloud_run_base_runner.py | 2 +- .../cloud_run/cloud_run_xds_server_runner.py | 1 + framework/xds_k8s_testcase.py | 18 ++-- 6 files changed, 88 insertions(+), 58 deletions(-) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index 2055c02e..847a561f 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -195,9 +195,9 @@ def cloudrun(self, version): api_name = "run" if version == "v2": return self._build_from_discovery_v2( - api_name, - version, - ) + api_name, + version, + ) raise NotImplementedError(f"Cloud run {version} not supported") @functools.lru_cache(None) @@ -530,11 +530,17 @@ def parent(self, location: Optional[str] = GLOBAL_LOCATION): location = self.GLOBAL_LOCATION return f"projects/{self.project}/locations/{location}" - def resource_full_name(self, name, collection_name,location: Optional[str] = None): + def resource_full_name( + self, name, collection_name, location: Optional[str] = None + ): return f"{self.parent(location)}/{collection_name}/{name}" def _create_resource( - self, collection: discovery.Resource, body: dict,location:Optional[str]=None, **kwargs + self, + collection: discovery.Resource, + body: dict, + location: Optional[str] = None, + **kwargs, ): logger.info( "Creating %s resource:\n%s", diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index feb6adac..9406d07c 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -13,33 +13,39 @@ # limitations under the License. import abc import logging -from framework.infrastructure import gcp + from googleapiclient import discovery +from framework.infrastructure import gcp + logger = logging.getLogger(__name__) DEFAULT_TEST_PORT = 8080 DEFAULT_TIMEOUT = 600 -class CloudRunApiManager(gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta): +class CloudRunApiManager( + gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta +): project: str region: str _parent: str - service :discovery.Resource + service: discovery.Resource api_manager: gcp.api.GcpApiManager - discovery_url="https://run.googleapis.com/$discovery/rest?" + discovery_url = "https://run.googleapis.com/$discovery/rest?" - def __init__(self, project: str,region: str): + def __init__(self, project: str, region: str): if not project: raise ValueError("Project ID cannot be empty or None.") if not region: raise ValueError("Region cannot be empty or None.") - self.api_manager = gcp.api.GcpApiManager(v2_discovery_uri="https://run.googleapis.com/$discovery/rest?") + self.api_manager = gcp.api.GcpApiManager( + v2_discovery_uri="https://run.googleapis.com/$discovery/rest?" + ) self.project = project self.region = region - service:discovery.Resource=self.api_manager.cloudrun("v2") - self.service=service + service: discovery.Resource = self.api_manager.cloudrun("v2") + self.service = service self._parent = f"projects/{self.project}/locations/{self.region}" super().__init__(self.service, project) @@ -47,35 +53,45 @@ def __init__(self, project: str,region: str): def api_name(self) -> str: """Returns the API name for Cloud Run.""" return "run" - + @property def api_version(self) -> str: """Returns the API version for Cloud Run.""" return "v2" - - def create_cloud_run_resource(self,service:discovery.Resource,service_name:str,body:dict): - return self._create_resource( + + def create_cloud_run_resource( + self, service: discovery.Resource, service_name: str, body: dict + ): + return self._create_resource( collection=service.projects().locations().services(), body=body, serviceId=service_name, - location=self.region + location=self.region, ) - - def get_cloud_run_resource(self,service:discovery.Resource,service_name:str): + + def get_cloud_run_resource( + self, service: discovery.Resource, service_name: str + ): return self._get_resource( collection=service.projects().locations().services(), - full_name=self.resource_full_name(service_name,"services",self.region) + full_name=self.resource_full_name( + service_name, "services", self.region + ), ) - def get_service_uri(self,service_name:str)->str: - response=self.get_cloud_run_resource(self.service,service_name) - return response.get('uri') - - def delete_cloud_run_resource(self,service:discovery.Resource,service_name:str): + def get_service_uri(self, service_name: str) -> str: + response = self.get_cloud_run_resource(self.service, service_name) + return response.get("uri") + + def delete_cloud_run_resource( + self, service: discovery.Resource, service_name: str + ): return self._delete_resource( collection=service.projects().locations().services(), - full_name=self.resource_full_name(service_name,"services",self.region) - ) + full_name=self.resource_full_name( + service_name, "services", self.region + ), + ) def deploy_service( self, @@ -88,32 +104,35 @@ def deploy_service( raise ValueError("service_name cannot be empty or None") if not image_name: raise ValueError("image_name cannot be empty or None") - + try: service_body = {} - service_body={ - "launch_stage":"alpha", - "template": - { - "containers": [ - { - "image": image_name, - "ports": [{"containerPort": test_port, "name": "h2c"}], - } - ], - }, - } - - self.create_cloud_run_resource(self.service,service_name,service_body) + service_body = { + "launch_stage": "alpha", + "template": { + "containers": [ + { + "image": image_name, + "ports": [ + {"containerPort": test_port, "name": "h2c"} + ], + } + ], + }, + } + + self.create_cloud_run_resource( + self.service, service_name, service_body + ) logger.info("Deploying Cloud Run service '%s'", service_name) return self.get_service_uri(service_name) - - except Exception as e: # noqa pylint: disable=broad-except + + except Exception as e: # noqa pylint: disable=broad-except logger.exception("Error deploying Cloud Run service: %s", e) def delete_service(self, service_name: str): try: - self.delete_cloud_run_resource(self.service,service_name) + self.delete_cloud_run_resource(self.service, service_name) except Exception as e: logger.exception("Error deleting service: %s", e) raise diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index c52cea35..3aa99e33 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -610,10 +610,10 @@ def delete_serverless_neg(self, name: str, region: str): try: logger.info("Deleting serverless NEG %s in %s", name, region) self.api.regionNetworkEndpointGroups().delete( - project=self.project, - region=region, - networkEndpointGroup=name, - ).execute() + project=self.project, + region=region, + networkEndpointGroup=name, + ).execute() logger.info("Deleted serverless neg : %s", name) except googleapiclient.errors.HttpError as error: if error.resp.status == 404: # NEG not found diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 234959e8..92e33025 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -135,7 +135,7 @@ def stop(self): try: self.cloud_run_api_manager.delete_service(self.service_name) logger.info("Deleted cloud run service: %s", self.service_name) - except Exception as e: # noqa pylint: disable=broad-except + except Exception as e: # noqa pylint: disable=broad-except logger.warning( "Cloud Run service %s deletion failed: %s", self.service_name, e ) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 48495d2a..0c284161 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -17,6 +17,7 @@ import dataclasses import logging from typing import List, Optional + from typing_extensions import override from framework.test_app.runners.cloud_run import cloud_run_base_runner diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index b858416a..02b67c72 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -558,11 +558,15 @@ def _assertRpcsEventuallyGoToGivenServers( f"Unexpected server {server_hostname} received RPCs", ) - def assertEDSConfigExists(self, config:ClientConfig, seen:set , want:frozenset): - want=want.union(["endpoint_config"]) + def assertEDSConfigExists( + self, config: ClientConfig, seen: set, want: frozenset + ): + want = want.union(["endpoint_config"]) for generic_xds_config in config.generic_xds_configs: - if re.search(r"\.ClusterLoadAssignment$", generic_xds_config.type_url): - seen.add("endpoint_config") + if re.search( + r"\.ClusterLoadAssignment$", generic_xds_config.type_url + ): + seen.add("endpoint_config") def assertXdsConfigExists(self, test_client: XdsTestClient): config = test_client.csds.fetch_client_status(log_level=logging.INFO) @@ -587,7 +591,7 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): elif re.search(r"\.Cluster$", generic_xds_config.type_url): seen.add("cluster_config") - self.assertEDSConfigExists(config,seen,want) + self.assertEDSConfigExists(config, seen, want) logger.debug( "Received xDS config dump: %s", @@ -1487,7 +1491,7 @@ def startTestServers(self, server_runner=None) -> List[XdsTestServer]: return test_servers def backendServiceAddServerlessNegBackends(self): - name=self.td.make_resource_name("neg") + name = self.td.make_resource_name("neg") logger.info("Creating serverless NEG") neg = self.compute_v1.create_serverless_neg( name, @@ -1497,7 +1501,7 @@ def backendServiceAddServerlessNegBackends(self): self.neg = neg return neg - def assertEDSConfigExists(self,config,seen,want): + def assertEDSConfigExists(self, config, seen, want): """No-op for Cloud Run as EDS is not required.""" _ = (seen, config, want) From a70ea9b2d1e3295f492ed9630f3ac1c095dff2dc Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 10 Mar 2025 18:29:12 +0530 Subject: [PATCH 19/49] pylint --- framework/xds_k8s_testcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 02b67c72..da2515db 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -124,7 +124,8 @@ class TdPropagationRetryableError(Exception): """Indicates that TD config hasn't propagated yet, and it's safe to retry""" -class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase): +class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase): # pylint: disable=too-many-public-methods + lang_spec: TestConfig client_namespace: str client_runner: KubernetesClientRunner From 268539782f2032200f5c2a41d1e81ce29dc39e6a Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 10 Mar 2025 18:32:27 +0530 Subject: [PATCH 20/49] pylint --- framework/xds_k8s_testcase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index da2515db..3e61b440 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -124,8 +124,9 @@ class TdPropagationRetryableError(Exception): """Indicates that TD config hasn't propagated yet, and it's safe to retry""" -class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase): # pylint: disable=too-many-public-methods - +class XdsKubernetesBaseTestCase( + base_testcase.BaseTestCase +): # pylint: disable=too-many-public-methods lang_spec: TestConfig client_namespace: str client_runner: KubernetesClientRunner @@ -559,9 +560,8 @@ def _assertRpcsEventuallyGoToGivenServers( f"Unexpected server {server_hostname} received RPCs", ) - def assertEDSConfigExists( - self, config: ClientConfig, seen: set, want: frozenset - ): + @staticmethod + def assertEDSConfigExists(config: ClientConfig, seen: set, want: frozenset): want = want.union(["endpoint_config"]) for generic_xds_config in config.generic_xds_configs: if re.search( From 75184b9142cbc28ad09790ef8e2890b9cd369bd6 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 12 Mar 2025 02:49:29 +0530 Subject: [PATCH 21/49] address comments --- framework/infrastructure/gcp/cloud_run.py | 23 +++++++------------ framework/infrastructure/gcp/compute.py | 1 - .../traffic_director_cloud_run.py | 2 -- .../cloud_run/cloud_run_base_runner.py | 1 - 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 9406d07c..f8caddf3 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -32,7 +32,6 @@ class CloudRunApiManager( _parent: str service: discovery.Resource api_manager: gcp.api.GcpApiManager - discovery_url = "https://run.googleapis.com/$discovery/rest?" def __init__(self, project: str, region: str): if not project: @@ -62,12 +61,9 @@ def api_version(self) -> str: def create_cloud_run_resource( self, service: discovery.Resource, service_name: str, body: dict ): - return self._create_resource( - collection=service.projects().locations().services(), - body=body, - serviceId=service_name, - location=self.region, - ) + service.projects().locations().services().create( + parent=self._parent, serviceId=service_name, body=body + ).execute() def get_cloud_run_resource( self, service: discovery.Resource, service_name: str @@ -86,12 +82,9 @@ def get_service_uri(self, service_name: str) -> str: def delete_cloud_run_resource( self, service: discovery.Resource, service_name: str ): - return self._delete_resource( - collection=service.projects().locations().services(), - full_name=self.resource_full_name( - service_name, "services", self.region - ), - ) + service.projects().locations().services().delete( + name=self.resource_full_name(service_name, "services", self.region) + ).execute() def deploy_service( self, @@ -106,7 +99,6 @@ def deploy_service( raise ValueError("image_name cannot be empty or None") try: - service_body = {} service_body = { "launch_stage": "alpha", "template": { @@ -121,14 +113,15 @@ def deploy_service( }, } + logger.info("Deploying Cloud Run service '%s'", service_name) self.create_cloud_run_resource( self.service, service_name, service_body ) - logger.info("Deploying Cloud Run service '%s'", service_name) return self.get_service_uri(service_name) except Exception as e: # noqa pylint: disable=broad-except logger.exception("Error deploying Cloud Run service: %s", e) + raise def delete_service(self, service_name: str): try: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 3aa99e33..733db710 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -580,7 +580,6 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): Returns: The NEG selfLink URL """ - name = name + "-neg" neg_body = { "name": name, "networkEndpointType": "SERVERLESS", diff --git a/framework/infrastructure/traffic_director_cloud_run.py b/framework/infrastructure/traffic_director_cloud_run.py index d6cc21a9..fca17384 100644 --- a/framework/infrastructure/traffic_director_cloud_run.py +++ b/framework/infrastructure/traffic_director_cloud_run.py @@ -74,8 +74,6 @@ def backend_service_add_backends( new_backends.append(new_backend) - # backend_service = self.backend_service - logging.info( "Adding backends to Backend Service %s: %r", self.backend_service.name, diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 92e33025..dba0ddbd 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -52,7 +52,6 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): tag: str = "latest" region: str = "us-central1" current_revision: Optional[str] = None - # gcp_project: Optional[str] = None gcp_ui_url: Optional[str] = None run_history: collections.deque[RunHistory] From 0b235ee963cd42cc1a974ba81359ce96e85e46b4 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 12 Mar 2025 03:03:55 +0530 Subject: [PATCH 22/49] black --- .../cloud_run/cloud_run_xds_server_runner.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 0c284161..74191fb8 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -26,27 +26,6 @@ logger = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) -class CloudRunDeploymentArgs: - """Arguments for deploying a server to Cloud Run.""" - - env_vars: dict = dataclasses.field(default_factory=dict) - max_instances: int = 10 # Example: Maximum number of instances - min_instances: int = 0 # Example: Minimum number of instances - service_account_email: str = "" # Email address of the service account - timeout_seconds: int = 300 # Timeout for requests - revision_suffix: Optional[str] = None - - def as_dict(self): - return { - "env_vars": self.env_vars, - "max_instances": self.max_instances, - "min_instances": self.min_instances, - "service_account_email": self.service_account_email, - "timeout_seconds": self.timeout_seconds, - } - - class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" From 486c554216799b0aea61ae6aa7bd4a799bb7fdf2 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 12 Mar 2025 15:31:53 +0530 Subject: [PATCH 23/49] add cloudrunlogs --- .../cloud_run/cloud_run_base_runner.py | 65 ++++++++++++++++++- .../cloud_run/cloud_run_xds_server_runner.py | 8 +-- framework/xds_k8s_testcase.py | 2 + ..._test.py => cloud_run_csm_inbound_test.py} | 6 +- 4 files changed, 72 insertions(+), 9 deletions(-) rename tests/{cloud_run_server_baseline_test.py => cloud_run_csm_inbound_test.py} (94%) diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index dba0ddbd..3fb3ec58 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -66,6 +66,7 @@ def __init__( service_name: str, image_name: str, region: str, + gcp_ui_url: str, network: Optional[str] = None, ) -> None: super().__init__() @@ -76,7 +77,7 @@ def __init__( self.network = network self.region = region self.current_revision = None - self.gcp_ui_url = None + self.gcp_ui_url = gcp_ui_url # Persistent across many runs. self.run_history = collections.deque() @@ -128,6 +129,68 @@ def _stop(self): ) self.run_history.append(run_history) + @classmethod + def _logs_explorer_link( + cls, + *, + service_name: str, + gcp_project: str, + gcp_ui_url: str, + location: str, + start_time: Optional[_datetime] = None, + end_time: Optional[_datetime] = None, + cursor_time: Optional[_datetime] = None, + ): + """Output the link to test server/client logs in GCP Logs Explorer.""" + if not start_time: + start_time = _datetime.now() + if not end_time: + end_time = start_time + _timedelta(minutes=30) + + logs_start = _helper_datetime.iso8601_utc_time(start_time) + logs_end = _helper_datetime.iso8601_utc_time(end_time) + request = {"timeRange": f"{logs_start}/{logs_end}"} + if cursor_time: + request["cursorTimestamp"] = _helper_datetime.iso8601_utc_time( + cursor_time + ) + query = { + "resource.type": "cloud_run_revision", + "resource.labels.project_id": gcp_project, + "resource.labels.service_name": service_name, + "resource.labels.location": location, + } + + link = cls._logs_explorer_link_from_params( + gcp_ui_url=gcp_ui_url, + gcp_project=gcp_project, + query=query, + request=request, + ) + link_to = service_name + # A whitespace at the end to indicate the end of the url. + logger.info("GCP Logs Explorer link to %s:\n%s ", link_to, link) + + def logs_explorer_run_history_links(self): + """Prints a separate GCP Logs Explorer link for each run *completed* by + the runner. + + This excludes the current run, if it hasn't been completed. + """ + if not self.run_history: + logger.info("No completed deployments of %s", self.service_name) + return + for run in self.run_history: + self._logs_explorer_link( + service_name=self.service_name, + gcp_project=self.project, + gcp_ui_url=self.gcp_ui_url, + location=self.region, + start_time=run.time_start_requested, + cursor_time=run.time_start_completed, + end_time=run.time_stopped, + ) + def stop(self): """Deletes Cloud Run Service""" logger.info("Deleting Cloud Run service: %s", self.service_name) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 74191fb8..2a8ef4ac 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -14,12 +14,12 @@ """ Run xDS Test Server on Cloud Run. """ -import dataclasses import logging -from typing import List, Optional +from typing import List from typing_extensions import override +from framework.infrastructure import gcp from framework.test_app.runners.cloud_run import cloud_run_base_runner from framework.test_app.server_app import XdsTestServer @@ -36,6 +36,7 @@ def __init__( image_name: str, network: str, region: str, + gcp_api_manager: gcp.api.GcpApiManager, ): super().__init__( project, @@ -43,6 +44,7 @@ def __init__( image_name, network=network, region=region, + gcp_ui_url=gcp_api_manager.gcp_ui_url, ) # Mutable state associated with each run. self._reset_state() @@ -80,7 +82,5 @@ def get_service_url(self): def cleanup(self, *, force=False): try: self.stop() - self.service_name = None - self.service = None finally: self._stop() diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 3e61b440..979b8e6d 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1483,6 +1483,7 @@ def startTestServers(self, server_runner=None) -> List[XdsTestServer]: image_name=self.server_image, network=self.network, region=self.region, + gcp_api_manager=self.gcp_api_manager, ) test_servers = self.server_runner.run() for test_server in test_servers: @@ -1537,6 +1538,7 @@ def tearDown(self): finally: logger.info("----- Test client/server logs -----") self.client_runner.logs_explorer_run_history_links() + self.server_runner.logs_explorer_run_history_links() # Fail if any of the pods restarted. self.assertEqual( diff --git a/tests/cloud_run_server_baseline_test.py b/tests/cloud_run_csm_inbound_test.py similarity index 94% rename from tests/cloud_run_server_baseline_test.py rename to tests/cloud_run_csm_inbound_test.py index f67a3a3b..12dc60b1 100644 --- a/tests/cloud_run_server_baseline_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -23,10 +23,8 @@ _XdsTestClient = xds_k8s_testcase.XdsTestClient -class CloudRunServerBaselineTest( - xds_k8s_testcase.CloudRunXdsKubernetesTestCase -): - def test_GKE_client_cloud_run_service(self): +class CloudRunCsmInboundTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): + def test_gke_to_cloud_run_setup(self): with self.subTest("0_create_mesh"): self.td.create_mesh() From 47447cf137bf3e05550d5815660cad5d629a828e Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 19 Mar 2025 13:16:59 +0530 Subject: [PATCH 24/49] flag file --- .kokoro/psm_interop_kokoro_lib.sh | 43 +++++++++++++++++++++++++++++++ bin/lib/common.py | 4 +-- config/common-cloudrun.cfg | 2 ++ config/common.cfg | 2 +- framework/xds_flags.py | 6 ++--- framework/xds_k8s_testcase.py | 2 +- 6 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 config/common-cloudrun.cfg diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 50db8116..9181ac4c 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -343,6 +343,49 @@ psm::csm::run_test() { psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } +# --- CloudRun TESTS ------------------ + +####################################### +# CloudRun Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::cloudrun::setup() { + activate_gke_cluster GKE_CLUSTER_CLOUD_RUN +} + +####################################### +# Prepares the list of tests in CloudRun test suite. +# Globals: +# TESTS: Populated with tests in PSM CloudRun test suite. +####################################### +psm::dualstack::get_tests() { + TESTS=( + "cloud_run_csm_inbound_test.py" + ) +} + +####################################### +# Executes CloudRun test case +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# GRPC_LANGUAGE: The name of gRPC languages under test +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +psm::cloudrun::run_test() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + + PSM_TEST_FLAGS+=( + "--flagfile=config/common-cloudrun.cfg" + ) + + psm::run::finalize_test_flags "${test_name}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + # --- Common test run logic ----------- ####################################### diff --git a/bin/lib/common.py b/bin/lib/common.py index bc24b938..5f213014 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -65,7 +65,7 @@ def gcp_api_manager(): @functools.cache def cloud_run_api_manager(): return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, - region=xds_flags.REGION.value) + region=xds_flags.CLOUD_RUN_REGION.value) def td_attrs(): return dict( @@ -169,7 +169,7 @@ def make_cloud_run_server_runner() -> CloudRunServerRunner: service_name=xds_flags.SERVER_NAME.value, image_name=xds_k8s_flags.SERVER_IMAGE.value, network=xds_flags.NETWORK.value, - region=xds_flags.REGION.value, + region=xds_flags.CLOUD_RUN_REGION.value, ) server_runner = CloudRunServerRunner return server_runner(**runner_kwargs) diff --git a/config/common-cloudrun.cfg b/config/common-cloudrun.cfg new file mode 100644 index 00000000..e1742327 --- /dev/null +++ b/config/common-cloudrun.cfg @@ -0,0 +1,2 @@ +# Common config file for Cloudrun PSM tests. +--cloud_run_region=us-central1 diff --git a/config/common.cfg b/config/common.cfg index df6cc89d..1bc8dff8 100644 --- a/config/common.cfg +++ b/config/common.cfg @@ -1,5 +1,5 @@ --resource_prefix=psm-interop ---td_bootstrap_image=us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:e4622a2cb6956f85da18eb0aa9daec839c2e38b5 +--td_bootstrap_image=us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap:ad35743ff01c0dc16a785e421591d7d52f3cdda4 # The canonical implementation of the xDS test server. # Can be used in tests where language-specific xDS test server does not exist, diff --git a/framework/xds_flags.py b/framework/xds_flags.py index b727e0de..b05dd2f9 100644 --- a/framework/xds_flags.py +++ b/framework/xds_flags.py @@ -199,10 +199,10 @@ help="Enable support for Dual Stack resources to the framework.", ) -REGION = flags.DEFINE_string( - "region", +CLOUD_RUN_REGION = flags.DEFINE_string( + "cloud_run_region", default=None, - help="The region for deployment", + help="The region for cloud run deployment", ) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 979b8e6d..df2d405b 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1423,7 +1423,7 @@ def setUpClass(cls): cls.compute_v1 = ComputeV1( cls.gcp_api_manager, cls.project, version=cls.compute_api_version ) - cls.region = xds_flags.REGION.value + cls.region = xds_flags.CLOUD_RUN_REGION.value def initTrafficDirectorManager(self) -> TrafficDirectorManager: return TrafficDirectorCloudRunManager( From 2c270c293f14108945d710c0eea8d0c9e5fdfe07 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Mar 2025 21:48:14 +0530 Subject: [PATCH 25/49] create_resource --- framework/infrastructure/gcp/cloud_run.py | 33 ++++++++++++++--------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index f8caddf3..329fb52f 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -13,6 +13,7 @@ # limitations under the License. import abc import logging +from typing import Final from googleapiclient import discovery @@ -20,9 +21,10 @@ logger = logging.getLogger(__name__) -DEFAULT_TEST_PORT = 8080 -DEFAULT_TIMEOUT = 600 +# Type aliases +GcpResource = gcp.compute.ComputeV1.GcpResource +DEFAULT_TEST_PORT: Final[int] = 8080 class CloudRunApiManager( gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta @@ -58,14 +60,19 @@ def api_version(self) -> str: """Returns the API version for Cloud Run.""" return "v2" - def create_cloud_run_resource( - self, service: discovery.Resource, service_name: str, body: dict - ): - service.projects().locations().services().create( - parent=self._parent, serviceId=service_name, body=body - ).execute() + def create_service( + self, service: discovery.Resource, service_name: str, body: dict) -> GcpResource: + self._create_resource( + collection=service.projects().locations().services(), + location=self.region, + body=body, + serviceId=service_name, + ) + # service.projects().locations().services().create( + # parent=self._parent, serviceId=service_name, body=body + # ).execute() - def get_cloud_run_resource( + def get_service( self, service: discovery.Resource, service_name: str ): return self._get_resource( @@ -76,10 +83,10 @@ def get_cloud_run_resource( ) def get_service_uri(self, service_name: str) -> str: - response = self.get_cloud_run_resource(self.service, service_name) + response = self.get_service(self.service, service_name) return response.get("uri") - def delete_cloud_run_resource( + def _delete_service( self, service: discovery.Resource, service_name: str ): service.projects().locations().services().delete( @@ -114,7 +121,7 @@ def deploy_service( } logger.info("Deploying Cloud Run service '%s'", service_name) - self.create_cloud_run_resource( + self.create_service( self.service, service_name, service_body ) return self.get_service_uri(service_name) @@ -125,7 +132,7 @@ def deploy_service( def delete_service(self, service_name: str): try: - self.delete_cloud_run_resource(self.service, service_name) + self._delete_service(self.service, service_name) except Exception as e: logger.exception("Error deleting service: %s", e) raise From 65f29bfb5b8cdcfdade491762f16785489da9610 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 28 Mar 2025 03:34:21 +0530 Subject: [PATCH 26/49] address comments --- .kokoro/psm_interop_kokoro_lib.sh | 43 -------------------- framework/infrastructure/gcp/api.py | 7 +--- framework/infrastructure/gcp/cloud_run.py | 37 +++++++++++------ framework/infrastructure/gcp/compute.py | 28 ++++++------- framework/infrastructure/traffic_director.py | 2 - tests/cloud_run_csm_inbound_test.py | 3 +- 6 files changed, 39 insertions(+), 81 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 41cb5c9f..c0f20ca4 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -382,49 +382,6 @@ psm::csm::run_test() { psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } -# --- CloudRun TESTS ------------------ - -####################################### -# CloudRun Test Suite setup. -# Outputs: -# Prints activated cluster names. -####################################### -psm::cloudrun::setup() { - activate_gke_cluster GKE_CLUSTER_CLOUD_RUN -} - -####################################### -# Prepares the list of tests in CloudRun test suite. -# Globals: -# TESTS: Populated with tests in PSM CloudRun test suite. -####################################### -psm::cloudrun::get_tests() { - TESTS=( - "cloud_run_csm_inbound_test.py" - ) -} - -####################################### -# Executes CloudRun test case -# Globals: -# PSM_TEST_FLAGS: The array with flags for the test -# GRPC_LANGUAGE: The name of gRPC languages under test -# Arguments: -# Test case name -# Outputs: -# Writes the output of test execution to stdout, stderr -####################################### -psm::cloudrun::run_test() { - local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" - - PSM_TEST_FLAGS+=( - "--flagfile=config/common-cloudrun.cfg" - ) - - psm::run::finalize_test_flags "${test_name}" - psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" -} - # --- Common test run logic ----------- ####################################### diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index 847a561f..6270a275 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -194,10 +194,7 @@ def networksecurity(self, version): def cloudrun(self, version): api_name = "run" if version == "v2": - return self._build_from_discovery_v2( - api_name, - version, - ) + return self._build_from_discovery_v2(api_name,version) raise NotImplementedError(f"Cloud run {version} not supported") @functools.lru_cache(None) @@ -531,7 +528,7 @@ def parent(self, location: Optional[str] = GLOBAL_LOCATION): return f"projects/{self.project}/locations/{location}" def resource_full_name( - self, name, collection_name, location: Optional[str] = None + self, name, collection_name, location: str = GLOBAL_LOCATION ): return f"{self.parent(location)}/{collection_name}/{name}" diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 329fb52f..0dd77244 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -13,7 +13,8 @@ # limitations under the License. import abc import logging -from typing import Final +import dataclasses +from typing import Final, Dict, Any from googleapiclient import discovery @@ -26,6 +27,18 @@ DEFAULT_TEST_PORT: Final[int] = 8080 +@dataclasses.dataclass(frozen=True) +class CloudRun: + service_name: str + url: str + + @classmethod + def from_response(cls, name: str, d: Dict[str, Any]) -> "CloudRun": + return cls( + service_name=name, + url=d["urls"], + ) + class CloudRunApiManager( gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta ): @@ -62,29 +75,27 @@ def api_version(self) -> str: def create_service( self, service: discovery.Resource, service_name: str, body: dict) -> GcpResource: - self._create_resource( - collection=service.projects().locations().services(), - location=self.region, - body=body, - serviceId=service_name, - ) - # service.projects().locations().services().create( - # parent=self._parent, serviceId=service_name, body=body - # ).execute() + service.projects().locations().services().create( + parent=self._parent, serviceId=service_name, body=body + ).execute() def get_service( self, service: discovery.Resource, service_name: str - ): - return self._get_resource( + ) -> CloudRun: + result=self._get_resource( collection=service.projects().locations().services(), full_name=self.resource_full_name( service_name, "services", self.region ), ) + return CloudRun.from_response(self.resource_full_name( + service_name, "services", self.region + ), result) def get_service_uri(self, service_name: str) -> str: response = self.get_service(self.service, service_name) - return response.get("uri") + return response.url + def _delete_service( self, service: discovery.Resource, service_name: str diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 733db710..df2f0ecf 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -145,14 +145,13 @@ def delete_firewall_rule(self, name): def create_backend_service_traffic_director( self, name: str, - health_check: "GcpResource", + health_check: Optional["GcpResource"] = None, affinity_header: Optional[str] = None, protocol: Optional[BackendServiceProtocol] = None, subset_size: Optional[int] = None, locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, enable_dualstack: bool = False, - is_cloud_run: bool = False, ) -> "GcpResource": if not isinstance(protocol, self.BackendServiceProtocol): raise TypeError(f"Unexpected Backend Service protocol: {protocol}") @@ -163,7 +162,7 @@ def create_backend_service_traffic_director( } # If it is not for cloud run, add heath check since cloud run does not # support health check. - if not is_cloud_run: + if not health_check is None: body["healthChecks"] = [health_check.url] # If add dualstack support is specified True, config the backend service @@ -580,24 +579,21 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): Returns: The NEG selfLink URL """ + + # Note: Cloud Run is not the only service that uses serverless NEGs. + # See: https://cloud.google.com/load-balancing/docs/negs/serverless-neg-concepts neg_body = { "name": name, "networkEndpointType": "SERVERLESS", "cloudRun": {"service": service_name}, } - - try: - logger.info("Creating serverless NEG %s in %s", name, region) - self.api.regionNetworkEndpointGroups().insert( - project=self.project, region=region, body=neg_body - ).execute() - neg = self.get_serverless_network_endpoint_group(name, region) - logger.info("Created serverless neg %s ", neg) - return neg - - except Exception as e: - logger.exception("Error creating serverless NEG: %s", e) - raise + logger.info("Creating serverless NEG %s in %s", name, region) + self.api.regionNetworkEndpointGroups().insert( + project=self.project, region=region, body=neg_body + ).execute() + neg = self.get_serverless_network_endpoint_group(name, region) + logger.info("Created serverless neg %s ", neg) + return neg def delete_serverless_neg(self, name: str, region: str): """Deletes a serverless NEG. diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 7343a0de..0004ad9d 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -245,7 +245,6 @@ def create_backend_service( affinity_header: Optional[str] = None, locality_lb_policies: Optional[List[dict]] = None, outlier_detection: Optional[dict] = None, - is_cloud_run: Optional[bool] = False, ): if protocol is None: protocol = _BackendGRPC @@ -261,7 +260,6 @@ def create_backend_service( locality_lb_policies=locality_lb_policies, outlier_detection=outlier_detection, enable_dualstack=self.enable_dualstack, - is_cloud_run=is_cloud_run, ) self.backend_service = resource self.backend_service_protocol = protocol diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 31c09f56..75fb837d 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -23,7 +23,7 @@ class CloudRunCsmInboundTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): - def test_gke_to_cloud_run_setup(self): + def test_gke_to_cloud_run(self): with self.subTest("0_create_mesh"): self.td.create_mesh() @@ -36,7 +36,6 @@ def test_gke_to_cloud_run_setup(self): with self.subTest("3_create_backend_service"): self.td.create_backend_service( protocol=self.compute_v1.BackendServiceProtocol.HTTP2, - is_cloud_run=True, ) with self.subTest("4_add_server_backends_to_backend_service"): From 54e9052c54d4aaf811fd4e2a5a39b2753917bd1a Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 28 Mar 2025 03:36:31 +0530 Subject: [PATCH 27/49] black --- framework/infrastructure/gcp/api.py | 2 +- framework/infrastructure/gcp/cloud_run.py | 27 +++++++++++------------ tests/cloud_run_csm_inbound_test.py | 1 + 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index 6270a275..31c35e96 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -194,7 +194,7 @@ def networksecurity(self, version): def cloudrun(self, version): api_name = "run" if version == "v2": - return self._build_from_discovery_v2(api_name,version) + return self._build_from_discovery_v2(api_name, version) raise NotImplementedError(f"Cloud run {version} not supported") @functools.lru_cache(None) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 0dd77244..a15d74f5 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import abc -import logging import dataclasses -from typing import Final, Dict, Any +import logging +from typing import Any, Dict, Final from googleapiclient import discovery @@ -27,6 +27,7 @@ DEFAULT_TEST_PORT: Final[int] = 8080 + @dataclasses.dataclass(frozen=True) class CloudRun: service_name: str @@ -39,6 +40,7 @@ def from_response(cls, name: str, d: Dict[str, Any]) -> "CloudRun": url=d["urls"], ) + class CloudRunApiManager( gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta ): @@ -74,7 +76,8 @@ def api_version(self) -> str: return "v2" def create_service( - self, service: discovery.Resource, service_name: str, body: dict) -> GcpResource: + self, service: discovery.Resource, service_name: str, body: dict + ) -> GcpResource: service.projects().locations().services().create( parent=self._parent, serviceId=service_name, body=body ).execute() @@ -82,24 +85,22 @@ def create_service( def get_service( self, service: discovery.Resource, service_name: str ) -> CloudRun: - result=self._get_resource( + result = self._get_resource( collection=service.projects().locations().services(), full_name=self.resource_full_name( service_name, "services", self.region ), ) - return CloudRun.from_response(self.resource_full_name( - service_name, "services", self.region - ), result) + return CloudRun.from_response( + self.resource_full_name(service_name, "services", self.region), + result, + ) def get_service_uri(self, service_name: str) -> str: response = self.get_service(self.service, service_name) return response.url - - def _delete_service( - self, service: discovery.Resource, service_name: str - ): + def _delete_service(self, service: discovery.Resource, service_name: str): service.projects().locations().services().delete( name=self.resource_full_name(service_name, "services", self.region) ).execute() @@ -132,9 +133,7 @@ def deploy_service( } logger.info("Deploying Cloud Run service '%s'", service_name) - self.create_service( - self.service, service_name, service_body - ) + self.create_service(self.service, service_name, service_body) return self.get_service_uri(service_name) except Exception as e: # noqa pylint: disable=broad-except diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 75fb837d..f16067d5 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging + from absl.testing import absltest from framework import xds_k8s_testcase From e7012376f70adf497bedaeeb9147610bbd0f5f81 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 1 Apr 2025 16:53:43 +0530 Subject: [PATCH 28/49] minor changes --- .kokoro/psm_interop_kokoro_lib.sh | 5 +++++ framework/infrastructure/gcp/api.py | 2 +- framework/infrastructure/gcp/cloud_run.py | 5 ++--- framework/infrastructure/gcp/compute.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index c0f20ca4..1f22f945 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -195,6 +195,11 @@ psm::cloud_run::get_tests() { ####################################### psm::cloud_run::run_test() { local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + + PSM_TEST_FLAGS+=( + "--flagfile=config/common-cloudrun.cfg" + ) + psm::run::finalize_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index 31c35e96..ff78829c 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -528,7 +528,7 @@ def parent(self, location: Optional[str] = GLOBAL_LOCATION): return f"projects/{self.project}/locations/{location}" def resource_full_name( - self, name, collection_name, location: str = GLOBAL_LOCATION + self, name: str, collection_name: str, location: str = GLOBAL_LOCATION ): return f"{self.parent(location)}/{collection_name}/{name}" diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index a15d74f5..2426ca6a 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -26,6 +26,7 @@ GcpResource = gcp.compute.ComputeV1.GcpResource DEFAULT_TEST_PORT: Final[int] = 8080 +DISCOVERY_URI: Final[str] = "https://run.googleapis.com/$discovery/rest?" @dataclasses.dataclass(frozen=True) @@ -55,9 +56,7 @@ def __init__(self, project: str, region: str): raise ValueError("Project ID cannot be empty or None.") if not region: raise ValueError("Region cannot be empty or None.") - self.api_manager = gcp.api.GcpApiManager( - v2_discovery_uri="https://run.googleapis.com/$discovery/rest?" - ) + self.api_manager = gcp.api.GcpApiManager(v2_discovery_uri=DISCOVERY_URI) self.project = project self.region = region service: discovery.Resource = self.api_manager.cloudrun("v2") diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index df2f0ecf..c12a9a87 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -162,7 +162,7 @@ def create_backend_service_traffic_director( } # If it is not for cloud run, add heath check since cloud run does not # support health check. - if not health_check is None: + if health_check: body["healthChecks"] = [health_check.url] # If add dualstack support is specified True, config the backend service From cd54ed413c83cd5c2defe6847d52f992922eae91 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 3 Apr 2025 15:05:45 +0530 Subject: [PATCH 29/49] use gcpstandardresource --- bin/lib/common.py | 1 + bin/run_test_server_cloud_run.py | 3 +- eshita_logs.txt | 387 ++++++++++++++++++++++ framework/infrastructure/gcp/api.py | 39 ++- framework/infrastructure/gcp/cloud_run.py | 30 +- tests/cloud_run_csm_inbound_test.py | 6 +- 6 files changed, 437 insertions(+), 29 deletions(-) create mode 100644 eshita_logs.txt diff --git a/bin/lib/common.py b/bin/lib/common.py index 5f213014..08ae9158 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -170,6 +170,7 @@ def make_cloud_run_server_runner() -> CloudRunServerRunner: image_name=xds_k8s_flags.SERVER_IMAGE.value, network=xds_flags.NETWORK.value, region=xds_flags.CLOUD_RUN_REGION.value, + gcp_api_manager=gcp.api.GcpApiManager(), ) server_runner = CloudRunServerRunner return server_runner(**runner_kwargs) diff --git a/bin/run_test_server_cloud_run.py b/bin/run_test_server_cloud_run.py index 5d9269a3..0bbb8581 100644 --- a/bin/run_test_server_cloud_run.py +++ b/bin/run_test_server_cloud_run.py @@ -42,9 +42,8 @@ def main(argv): xds_flags.set_socket_default_timeout_from_flag() - run_kwargs = dict() server_runner = common.make_cloud_run_server_runner() - server_runner.run(**run_kwargs) + server_runner.run() if __name__ == "__main__": diff --git a/eshita_logs.txt b/eshita_logs.txt new file mode 100644 index 00000000..b8458a67 --- /dev/null +++ b/eshita_logs.txt @@ -0,0 +1,387 @@ +Saving framework log to /Users/emchandwani/psm-interop/out/run/20250403-145431-cloud_run_csm_inbound_test.log ++ [14:54:31 IST] exec python tests/cloud_run_csm_inbound_test.py --flagfile=/Users/emchandwani/psm-interop/config/local-dev.cfg +Running tests under Python 3.10.16: /Users/emchandwani/psm-interop/venv/bin/python +I0403 14:54:32.686208 8252002176 xds_k8s_testcase.py:177] ----- Testing CloudRunCsmInboundTest ----- +I0403 14:54:32.686257 8252002176 xds_k8s_testcase.py:178] Logs timezone: IST +I0403 14:54:32.686340 8252002176 xds_k8s_testcase.py:106] Detected language and version: TestConfig(client_lang='cpp', server_lang='cpp', version=None) +I0403 14:54:34.163246 8252002176 k8s.py:264] Using kubernetes context "gke_emchandwani-default_us-central1-a_xds-k8s-interop-tests-cluster5", active host: https://34.55.128.246 +[ RUN ] CloudRunCsmInboundTest.test_gke_to_cloud_run +I0403 14:54:35.881356 8252002176 xds_k8s_testcase.py:865] Test run resource prefix: emchandwani-psm, suffix: 20250403-0924-5i0rd +I0403 14:54:41.003833 8252002176 xds_k8s_testcase.py:860] Found unused xds port: 11198 +I0403 14:54:41.005290 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.0_create_mesh --- +I0403 14:54:41.005562 8252002176 traffic_director.py:947] Creating Mesh emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +I0403 14:54:41.009238 8252002176 api.py:543] Creating networkservices resource: +--- {} +... + +I0403 14:54:42.954442 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672282522-631dc57012215-2760f929-becd3af7', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:24:42.694884164Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd', 'verb': 'create', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} +I0403 14:54:42.954872 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672282522-631dc57012215-2760f929-becd3af7 +I0403 14:54:42.958230 8252002176 api.py:612] eshita Waiting for operation: +I0403 14:54:49.019292 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd: +--- +createTime: '2025-04-03T09:24:42.674442384Z' +name: projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +selfLink: https://networkservices.googleapis.com/v1alpha1/projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +updateTime: '2025-04-03T09:24:45.928312100Z' +... + +I0403 14:54:49.019686 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.0_create_mesh --- +I0403 14:54:49.019819 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.1_start_cloud_run_test_server --- +I0403 14:54:49.746824 8252002176 cloud_run_xds_server_runner.py:62] Starting cloud run server with service emchandwani-psm-server-20250403-0924-5i0rd and image us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x +I0403 14:54:49.746951 8252002176 cloud_run.py:134] Deploying Cloud Run service 'emchandwani-psm-server-20250403-0924-5i0rd' +I0403 14:54:49.748914 8252002176 api.py:543] Creating run resource: +--- +launch_stage: alpha +template: + containers: + - image: us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x + ports: + - containerPort: 8080 + name: h2c +... + +I0403 14:54:52.626143 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/5dd1d9a3-001c-4924-a43a-d6d0abfe82ed', 'metadata': {'@type': 'type.googleapis.com/google.cloud.run.v2.Service', 'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd', 'uid': 'b0977056-29b5-45b8-b626-8c526e436119', 'generation': '1', 'createTime': '2025-04-03T09:24:52.130328Z', 'updateTime': '2025-04-03T09:24:52.130328Z', 'creator': 'emchandwani@google.com', 'lastModifier': 'emchandwani@google.com', 'launchStage': 'GA', 'template': {'scaling': {'maxInstanceCount': 100}, 'timeout': '300s', 'serviceAccount': '921384807982-compute@developer.gserviceaccount.com', 'containers': [{'image': 'us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x', 'resources': {'limits': {'memory': '512Mi', 'cpu': '1000m'}, 'cpuIdle': True}, 'ports': [{'name': 'h2c', 'containerPort': 8080}], 'startupProbe': {'timeoutSeconds': 240, 'periodSeconds': 240, 'failureThreshold': 1, 'tcpSocket': {'port': 8080}}}], 'maxInstanceRequestConcurrency': 80}, 'traffic': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'urls': ['https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app'], 'terminalCondition': {'state': 'CONDITION_PENDING'}, 'reconciling': True, 'etag': '"COSnub8GEMDLkj4/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"'}} +I0403 14:54:52.626596 8252002176 api.py:602] Waiting 600 sec for run operation id: projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/5dd1d9a3-001c-4924-a43a-d6d0abfe82ed +I0403 14:54:52.628556 8252002176 api.py:612] eshita Waiting for operation: +I0403 14:55:02.208549 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd: +--- +conditions: +- lastTransitionTime: '2025-04-03T09:24:58.430396Z' + state: CONDITION_SUCCEEDED + type: RoutesReady +- lastTransitionTime: '2025-04-03T09:24:57.419395Z' + state: CONDITION_SUCCEEDED + type: ConfigurationsReady +createTime: '2025-04-03T09:24:52.130328Z' +creator: emchandwani@google.com +etag: '"COSnub8GEMDLkj4/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"' +generation: '1' +ingress: INGRESS_TRAFFIC_ALL +lastModifier: emchandwani@google.com +latestCreatedRevision: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l +latestReadyRevision: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l +launchStage: GA +name: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd +observedGeneration: '1' +template: + containers: + - image: us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x + ports: + - containerPort: 8080 + name: h2c + resources: + cpuIdle: true + limits: + cpu: 1000m + memory: 512Mi + startupProbe: + failureThreshold: 1 + periodSeconds: 240 + tcpSocket: + port: 8080 + timeoutSeconds: 240 + maxInstanceRequestConcurrency: 80 + scaling: + maxInstanceCount: 100 + serviceAccount: 921384807982-compute@developer.gserviceaccount.com + timeout: 300s +terminalCondition: + lastTransitionTime: '2025-04-03T09:24:58.473882Z' + state: CONDITION_SUCCEEDED + type: Ready +traffic: +- percent: 100 + type: TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST +trafficStatuses: +- percent: 100 + type: TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST +uid: b0977056-29b5-45b8-b626-8c526e436119 +updateTime: '2025-04-03T09:24:52.130328Z' +uri: https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app +urls: +- https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app +- https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app +... + +I0403 14:55:02.209048 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.1_start_cloud_run_test_server --- +I0403 14:55:02.209176 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.2_create_serverless_neg --- +I0403 14:55:02.209264 8252002176 xds_k8s_testcase.py:1504] Creating serverless NEG +I0403 14:55:02.209345 8252002176 compute.py:590] Creating serverless NEG emchandwani-psm-neg-20250403-0924-5i0rd in us-east1 +I0403 14:55:05.450768 8252002176 compute.py:595] Created serverless neg {'kind': 'compute#networkEndpointGroup', 'id': '2166457725700271360', 'creationTimestamp': '2025-04-03T02:25:03.766-07:00', 'selfLink': 'https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd', 'name': 'emchandwani-psm-neg-20250403-0924-5i0rd', 'networkEndpointType': 'SERVERLESS', 'size': 0, 'region': 'https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1', 'cloudRun': {'service': 'emchandwani-psm-server-20250403-0924-5i0rd'}} +I0403 14:55:05.451114 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.2_create_serverless_neg --- +I0403 14:55:05.451211 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.3_create_backend_service --- +I0403 14:55:05.451360 8252002176 traffic_director.py:253] Creating HTTP2 Backend Service "emchandwani-psm-backend-service-20250403-0924-5i0rd" +I0403 14:55:05.465883 8252002176 compute.py:662] Creating compute resource: +--- +loadBalancingScheme: INTERNAL_SELF_MANAGED +name: emchandwani-psm-backend-service-20250403-0924-5i0rd +protocol: HTTP2 +... + +I0403 14:55:06.219388 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672305646-631dc5861fd32-6386d9d6-95223583 +I0403 14:55:25.550782 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.3_create_backend_service --- +I0403 14:55:25.550887 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.4_add_server_backends_to_backend_service --- +I0403 14:55:25.550961 8252002176 traffic_director_cloud_run.py:77] Adding backends to Backend Service emchandwani-psm-backend-service-20250403-0924-5i0rd: [{'group': ComputeV1.GcpResource(name='emchandwani-psm-neg-20250403-0924-5i0rd', url='https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd'), 'balancingMode': 'us-east1', 'maxRatePerEndpoint': None, 'capacityScaler': 1.0}] +I0403 14:55:25.554704 8252002176 compute.py:669] Patching compute resource: +--- +backends: +- group: https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd +... + +I0403 14:55:27.369940 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672325737-631dc59948d81-608bf3ee-0cf3eed7 +I0403 14:56:29.219877 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.4_add_server_backends_to_backend_service --- +I0403 14:56:29.220246 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.5_create_grpc_route --- +I0403 14:56:29.220464 8252002176 traffic_director.py:978] Creating GrpcRoute emchandwani-psm-grpc-route-20250403-0924-5i0rd +I0403 14:56:29.225157 8252002176 api.py:543] Creating networkservices resource: +--- +hostnames: psm-grpc-server:11198 +meshes: +- projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +rules: +- action: + destinations: + - serviceName: projects/bct-gcestaging-guitar-grpc/locations/global/backendServices/emchandwani-psm-backend-service-20250403-0924-5i0rd +... + +I0403 14:56:30.814513 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672390512-631dc5d70f075-8b81644c-f50f08b6', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:26:30.686892159Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd', 'verb': 'create', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} +I0403 14:56:30.814918 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672390512-631dc5d70f075-8b81644c-f50f08b6 +I0403 14:56:30.817574 8252002176 api.py:612] eshita Waiting for operation: +I0403 14:56:39.134803 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd: +--- +createTime: '2025-04-03T09:26:30.665875084Z' +hostnames: +- psm-grpc-server:11198 +meshes: +- projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +name: projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd +rules: +- action: + destinations: + - serviceName: projects/bct-gcestaging-guitar-grpc/locations/global/backendServices/emchandwani-psm-backend-service-20250403-0924-5i0rd +selfLink: https://networkservices.googleapis.com/v1alpha1/projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd +updateTime: '2025-04-03T09:26:36.441076738Z' +... + +I0403 14:56:39.135174 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.5_create_grpc_route --- +I0403 14:56:39.135293 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.6_start_test_client --- +I0403 14:56:39.135482 8252002176 k8s_xds_client_runner.py:136] Deploying xDS test client "psm-grpc-client" to k8s namespace emchandwani-psm-client-20250403-0924-5i0rd: server_target=xds:///psm-grpc-server:11198 rpc=UnaryCall qps=25 metadata='' secure_mode=True print_response=False is_trusted_xds_server_experimental=True +I0403 14:56:39.135941 8252002176 k8s_base_runner.py:1171] GCP Logs Explorer link to psm-grpc-client: +https://console.cloud.google.com/logs/query;query=resource.type%3D%22k8s_container%22%0Aresource.labels.project_id%3D%22bct-gcestaging-guitar-grpc%22%0Aresource.labels.container_name%3D%22psm-grpc-client%22%0Aresource.labels.namespace_name%3D%22emchandwani-psm-client-20250403-0924-5i0rd%22;timeRange=2025-04-03T09:26:39.135656Z%2F2025-04-03T09:56:39.135656Z?project=bct-gcestaging-guitar-grpc +I0403 14:56:39.144119 8252002176 k8s_base_runner.py:351] Rendered template kubernetes-manifests/namespace.yaml: +--- +apiVersion: v1 +kind: Namespace +metadata: + name: emchandwani-psm-client-20250403-0924-5i0rd + labels: + name: emchandwani-psm-client-20250403-0924-5i0rd + owner: xds-k8s-interop-test +... + +W0403 14:57:54.148485 8252002176 connectionpool.py:823] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces +W0403 14:59:09.151218 8252002176 connectionpool.py:823] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces +W0403 15:00:24.153978 8252002176 connectionpool.py:823] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces +I0403 15:01:39.157646 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.6_start_test_client --- +I0403 15:01:39.157891 8252002176 xds_k8s_testcase.py:1526] ----- TestMethod CloudRunCsmInboundTest.test_gke_to_cloud_run teardown ----- +I0403 15:01:39.158314 8252002176 cloud_run_base_runner.py:196] Deleting Cloud Run service: emchandwani-psm-server-20250403-0924-5i0rd +I0403 15:01:41.273350 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/810c3f49-c231-499e-8713-40e0de5e68db', 'metadata': {'@type': 'type.googleapis.com/google.cloud.run.v2.Service', 'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd', 'uid': 'b0977056-29b5-45b8-b626-8c526e436119', 'generation': '2', 'createTime': '2025-04-03T09:24:52.130328Z', 'updateTime': '2025-04-03T09:31:41.167940Z', 'deleteTime': '2025-04-03T09:31:41.167940Z', 'expireTime': '2025-05-03T09:31:41.167940Z', 'creator': 'emchandwani@google.com', 'lastModifier': 'emchandwani@google.com', 'ingress': 'INGRESS_TRAFFIC_ALL', 'launchStage': 'GA', 'template': {'scaling': {'maxInstanceCount': 100}, 'timeout': '300s', 'serviceAccount': '921384807982-compute@developer.gserviceaccount.com', 'containers': [{'image': 'us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x', 'resources': {'limits': {'memory': '512Mi', 'cpu': '1000m'}, 'cpuIdle': True}, 'ports': [{'name': 'h2c', 'containerPort': 8080}], 'startupProbe': {'timeoutSeconds': 240, 'periodSeconds': 240, 'failureThreshold': 1, 'tcpSocket': {'port': 8080}}}], 'maxInstanceRequestConcurrency': 80}, 'traffic': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'urls': ['https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app', 'https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app'], 'observedGeneration': '1', 'terminalCondition': {'type': 'Ready', 'state': 'CONDITION_PENDING', 'lastTransitionTime': '2025-04-03T09:24:58.473882Z'}, 'conditions': [{'type': 'RoutesReady', 'state': 'CONDITION_SUCCEEDED', 'lastTransitionTime': '2025-04-03T09:24:58.430396Z'}, {'type': 'ConfigurationsReady', 'state': 'CONDITION_SUCCEEDED', 'lastTransitionTime': '2025-04-03T09:24:57.419395Z'}], 'latestReadyRevision': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l', 'latestCreatedRevision': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l', 'trafficStatuses': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'uri': 'https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app', 'reconciling': True, 'etag': '"CP2qub8GEKCfilA/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"'}} +I0403 15:01:41.273536 8252002176 api.py:602] Waiting 600 sec for run operation id: projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/810c3f49-c231-499e-8713-40e0de5e68db +I0403 15:01:41.274407 8252002176 api.py:612] eshita Waiting for operation: +I0403 15:01:42.965970 8252002176 cloud_run_base_runner.py:199] Deleted cloud run service: emchandwani-psm-server-20250403-0924-5i0rd +I0403 15:01:42.966464 8252002176 traffic_director.py:1007] Deleting GrpcRoute emchandwani-psm-grpc-route-20250403-0924-5i0rd +I0403 15:01:44.544538 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672704339-631dc70258d59-c4ac7305-8032a00b', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:31:44.409306336Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd', 'verb': 'delete', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} +I0403 15:01:44.544960 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672704339-631dc70258d59-c4ac7305-8032a00b +I0403 15:01:44.547852 8252002176 api.py:612] eshita Waiting for operation: +I0403 15:01:52.432933 8252002176 traffic_director.py:961] Deleting Mesh emchandwani-psm-grpc-mesh-20250403-0924-5i0rd +I0403 15:01:53.856508 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672713631-631dc70b35847-0776a9a5-e1e5f33d', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:31:53.684546316Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd', 'verb': 'delete', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} +I0403 15:01:53.856900 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672713631-631dc70b35847-0776a9a5-e1e5f33d +I0403 15:01:53.858998 8252002176 api.py:612] eshita Waiting for operation: +I0403 15:01:58.510071 8252002176 traffic_director.py:279] Deleting Backend Service "emchandwani-psm-backend-service-20250403-0924-5i0rd" +I0403 15:01:59.445646 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672718908-631dc7103daeb-52d70a92-fb70e80c +I0403 15:02:14.751783 8252002176 compute.py:606] Deleting serverless NEG emchandwani-psm-neg-20250403-0924-5i0rd in us-east1 +I0403 15:02:16.436823 8252002176 compute.py:612] Deleted serverless neg : emchandwani-psm-neg-20250403-0924-5i0rd +I0403 15:02:16.437970 8252002176 xds_k8s_testcase.py:1546] ----- Test client/server logs ----- +I0403 15:02:16.438210 8252002176 k8s_base_runner.py:1114] No completed deployments of psm-grpc-client +I0403 15:02:16.438795 8252002176 cloud_run_base_runner.py:172] GCP Logs Explorer link to emchandwani-psm-server-20250403-0924-5i0rd: +https://console.cloud.google.com/logs/query;query=resource.type%3D%22cloud_run_revision%22%0Aresource.labels.project_id%3D%22bct-gcestaging-guitar-grpc%22%0Aresource.labels.service_name%3D%22emchandwani-psm-server-20250403-0924-5i0rd%22%0Aresource.labels.location%3D%22us-east1%22;timeRange=2025-04-03T09:24:49.746934Z%2F2025-04-03T09:31:42.966335Z;cursorTimestamp=2025-04-03T09:25:02.208940Z?project=bct-gcestaging-guitar-grpc +[ FAILED ] CloudRunCsmInboundTest.test_gke_to_cloud_run +E0403 15:02:16.467079 8252002176 base_testcase.py:44] ----- PSM Test Case FAILED: CloudRunCsmInboundTest.test_gke_to_cloud_run ----- +E0403 15:02:16.467297 8252002176 base_testcase.py:145] (ERROR) PSM Interop Test Failed: CloudRunCsmInboundTest.test_gke_to_cloud_run +^^^^^ +# [CloudRunCsmInboundTest.test_gke_to_cloud_run] PSM Failed Test Traceback BEGIN +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn + conn = connection.create_connection( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection + raise err + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + sock.connect(sa) +TimeoutError: [Errno 60] Operation timed out + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen + httplib_response = self._make_request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request + self._validate_conn(conn) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn + conn.connect() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect + self.sock = conn = self._new_conn() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 179, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to 34.55.128.246 timed out. (connect timeout=None)') + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 55, in test_gke_to_cloud_run + test_client: _XdsTestClient = self.startSecureTestClient( + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1177, in startSecureTestClient + return self._start_test_client( + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 951, in _start_test_client + test_client = self.client_runner.run( + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_xds_client_runner.py", line 152, in run + super().run() + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 196, in run + self.namespace = self._create_namespace( + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 420, in _create_namespace + namespace = self._create_from_template(template, **kwargs) + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 366, in _create_from_template + k8s_object = self.k8s_namespace.create_single_resource( + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 589, in create_single_resource + return self._execute(self._apply_manifest_single, manifest) + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 453, in _execute + return method(*args, **kwargs) + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 390, in _apply_manifest_single + k8s_objects = utils.create_from_dict( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 221, in create_from_dict + created = create_from_yaml_single_item( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 263, in create_from_yaml_single_item + resp = getattr(k8s_api, "create_{0}".format(kind))( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6363, in create_namespace + return self.create_namespace_with_http_info(body, **kwargs) # noqa: E501 + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6454, in create_namespace_with_http_info + return self.api_client.call_api( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 391, in request + return self.rest_client.POST(url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 279, in POST + return self.request("POST", url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 172, in request + r = self.pool_manager.request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 78, in request + return self.request_encode_body( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 170, in request_encode_body + return self.urlopen(method, url, **extra_kw) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/poolmanager.py", line 376, in urlopen + response = conn.urlopen(method, u.request_uri, **kw) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen + retries = retries.increment( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment + raise MaxRetryError(_pool, url, error or ResponseError(cause)) +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='34.55.128.246', port=443): Max retries exceeded with url: /api/v1/namespaces (Caused by ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')) +# [CloudRunCsmInboundTest.test_gke_to_cloud_run] PSM Failed Test Traceback END + +I0403 15:02:16.467360 8252002176 base_testcase.py:49] ----- PSM Test Case CloudRunCsmInboundTest.test_gke_to_cloud_run Error Count: 1 ----- +====================================================================== +ERROR: test_gke_to_cloud_run (__main__.CloudRunCsmInboundTest) +CloudRunCsmInboundTest.test_gke_to_cloud_run +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn + conn = connection.create_connection( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection + raise err + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + sock.connect(sa) +TimeoutError: [Errno 60] Operation timed out + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen + httplib_response = self._make_request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request + self._validate_conn(conn) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn + conn.connect() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect + self.sock = conn = self._new_conn() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 179, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to 34.55.128.246 timed out. (connect timeout=None)') + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 55, in test_gke_to_cloud_run + test_client: _XdsTestClient = self.startSecureTestClient( + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1177, in startSecureTestClient + return self._start_test_client( + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 951, in _start_test_client + test_client = self.client_runner.run( + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_xds_client_runner.py", line 152, in run + super().run() + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 196, in run + self.namespace = self._create_namespace( + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 420, in _create_namespace + namespace = self._create_from_template(template, **kwargs) + File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 366, in _create_from_template + k8s_object = self.k8s_namespace.create_single_resource( + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 589, in create_single_resource + return self._execute(self._apply_manifest_single, manifest) + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 453, in _execute + return method(*args, **kwargs) + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 390, in _apply_manifest_single + k8s_objects = utils.create_from_dict( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 221, in create_from_dict + created = create_from_yaml_single_item( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 263, in create_from_yaml_single_item + resp = getattr(k8s_api, "create_{0}".format(kind))( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6363, in create_namespace + return self.create_namespace_with_http_info(body, **kwargs) # noqa: E501 + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6454, in create_namespace_with_http_info + return self.api_client.call_api( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 391, in request + return self.rest_client.POST(url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 279, in POST + return self.request("POST", url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 172, in request + r = self.pool_manager.request( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 78, in request + return self.request_encode_body( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 170, in request_encode_body + return self.urlopen(method, url, **extra_kw) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/poolmanager.py", line 376, in urlopen + response = conn.urlopen(method, u.request_uri, **kw) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen + return self.urlopen( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen + retries = retries.increment( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment + raise MaxRetryError(_pool, url, error or ResponseError(cause)) +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='34.55.128.246', port=443): Max retries exceeded with url: /api/v1/namespaces (Caused by ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')) + +---------------------------------------------------------------------- +Ran 1 test in 463.787s + +FAILED (errors=1) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index ff78829c..e394fc38 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -537,6 +537,7 @@ def _create_resource( collection: discovery.Resource, body: dict, location: Optional[str] = None, + is_cloudrun: bool = False, **kwargs, ): logger.info( @@ -547,7 +548,8 @@ def _create_resource( create_req = collection.create( parent=self.parent(location), body=body, **kwargs ) - self._execute(create_req) + + self._execute(create_req, is_cloudrun=is_cloudrun) @property @abc.abstractmethod @@ -567,11 +569,16 @@ def _get_resource(self, collection: discovery.Resource, full_name): return resource def _delete_resource( - self, collection: discovery.Resource, full_name: str + self, + collection: discovery.Resource, + full_name: str, + is_cloudrun: bool = False, ) -> bool: logger.debug("Deleting %s", full_name) try: - self._execute(collection.delete(name=full_name)) + self._execute( + collection.delete(name=full_name), is_cloudrun=is_cloudrun + ) return True except _HttpError as error: if error.resp and error.resp.status == 404: @@ -585,15 +592,17 @@ def _execute( # pylint: disable=arguments-differ self, request: HttpRequest, timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, + is_cloudrun: bool = False, ): operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.debug("Operation %s", operation) - self._wait(operation["name"], timeout_sec) + logger.info("Operation %s", operation) + self._wait(operation["name"], timeout_sec, is_cloudrun) def _wait( self, operation_id: str, timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, + is_cloudrun: bool = False, ): logger.info( "Waiting %s sec for %s operation id: %s", @@ -605,11 +614,21 @@ def _wait( op_request = ( self.api.projects().locations().operations().get(name=operation_id) ) - operation = self.wait_for_operation( - operation_request=op_request, - test_success_fn=lambda result: result["done"], - timeout_sec=timeout_sec, - ) + if is_cloudrun: + operation = self.wait_for_operation( + operation_request=op_request, + test_success_fn=lambda result: result["metadata"][ + "terminalCondition" + ]["state"] + == "CONDITION_SUCCEEDED", + timeout_sec=timeout_sec, + ) + else: + operation = self.wait_for_operation( + operation_request=op_request, + test_success_fn=lambda result: result["done"], + timeout_sec=timeout_sec, + ) logger.debug("Completed operation: %s", operation) if "error" in operation: diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 2426ca6a..e4990eba 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -77,9 +77,13 @@ def api_version(self) -> str: def create_service( self, service: discovery.Resource, service_name: str, body: dict ) -> GcpResource: - service.projects().locations().services().create( - parent=self._parent, serviceId=service_name, body=body - ).execute() + return self._create_resource( + collection=service.projects().locations().services(), + location=self.region, + serviceId=service_name, + body=body, + is_cloudrun=True, + ) def get_service( self, service: discovery.Resource, service_name: str @@ -99,10 +103,15 @@ def get_service_uri(self, service_name: str) -> str: response = self.get_service(self.service, service_name) return response.url - def _delete_service(self, service: discovery.Resource, service_name: str): - service.projects().locations().services().delete( - name=self.resource_full_name(service_name, "services", self.region) - ).execute() + def delete_service(self, service_name: str): + # pylint: disable=no-member + self._delete_resource( + self.service.projects().locations().services(), + full_name=self.resource_full_name( + service_name, "services", self.region + ), + is_cloudrun=True, + ) def deploy_service( self, @@ -138,10 +147,3 @@ def deploy_service( except Exception as e: # noqa pylint: disable=broad-except logger.exception("Error deploying Cloud Run service: %s", e) raise - - def delete_service(self, service_name: str): - try: - self._delete_service(self.service, service_name) - except Exception as e: - logger.exception("Error deleting service: %s", e) - raise diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index f16067d5..7972abc5 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -51,17 +51,17 @@ def test_gke_to_cloud_run(self): self.server_xds_host, self.server_xds_port ) - with self.subTest("7_start_test_client"): + with self.subTest("6_start_test_client"): test_client: _XdsTestClient = self.startSecureTestClient( test_server, config_mesh=self.td.mesh.name, is_trusted_xds_server_experimental=True, ) - with self.subTest("8_test_client_xds_config_exists"): + with self.subTest("7_test_client_xds_config_exists"): self.assertXdsConfigExists(test_client) - with self.subTest("9_test_server_received_rpcs_from_test_client"): + with self.subTest("8_test_server_received_rpcs_from_test_client"): self.assertSuccessfulRpcs(test_client) From 5759ea62a42b58391818622daaca2ca7f378468d Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 8 Apr 2025 22:41:18 +0530 Subject: [PATCH 30/49] neg changes --- eshita_logs.txt | 445 ++++-------------------- framework/infrastructure/gcp/compute.py | 46 ++- tests/cloud_run_csm_inbound_test.py | 6 +- 3 files changed, 105 insertions(+), 392 deletions(-) diff --git a/eshita_logs.txt b/eshita_logs.txt index b8458a67..cfdc144c 100644 --- a/eshita_logs.txt +++ b/eshita_logs.txt @@ -1,315 +1,86 @@ -Saving framework log to /Users/emchandwani/psm-interop/out/run/20250403-145431-cloud_run_csm_inbound_test.log -+ [14:54:31 IST] exec python tests/cloud_run_csm_inbound_test.py --flagfile=/Users/emchandwani/psm-interop/config/local-dev.cfg +Saving framework log to /Users/emchandwani/psm-interop/out/run/20250408-112536-cloud_run_csm_inbound_test.log ++ [11:25:36 IST] exec python tests/cloud_run_csm_inbound_test.py --flagfile=/Users/emchandwani/psm-interop/config/local-dev.cfg Running tests under Python 3.10.16: /Users/emchandwani/psm-interop/venv/bin/python -I0403 14:54:32.686208 8252002176 xds_k8s_testcase.py:177] ----- Testing CloudRunCsmInboundTest ----- -I0403 14:54:32.686257 8252002176 xds_k8s_testcase.py:178] Logs timezone: IST -I0403 14:54:32.686340 8252002176 xds_k8s_testcase.py:106] Detected language and version: TestConfig(client_lang='cpp', server_lang='cpp', version=None) -I0403 14:54:34.163246 8252002176 k8s.py:264] Using kubernetes context "gke_emchandwani-default_us-central1-a_xds-k8s-interop-tests-cluster5", active host: https://34.55.128.246 -[ RUN ] CloudRunCsmInboundTest.test_gke_to_cloud_run -I0403 14:54:35.881356 8252002176 xds_k8s_testcase.py:865] Test run resource prefix: emchandwani-psm, suffix: 20250403-0924-5i0rd -I0403 14:54:41.003833 8252002176 xds_k8s_testcase.py:860] Found unused xds port: 11198 -I0403 14:54:41.005290 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.0_create_mesh --- -I0403 14:54:41.005562 8252002176 traffic_director.py:947] Creating Mesh emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -I0403 14:54:41.009238 8252002176 api.py:543] Creating networkservices resource: ---- {} -... - -I0403 14:54:42.954442 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672282522-631dc57012215-2760f929-becd3af7', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:24:42.694884164Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd', 'verb': 'create', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} -I0403 14:54:42.954872 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672282522-631dc57012215-2760f929-becd3af7 -I0403 14:54:42.958230 8252002176 api.py:612] eshita Waiting for operation: -I0403 14:54:49.019292 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd: ---- -createTime: '2025-04-03T09:24:42.674442384Z' -name: projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -selfLink: https://networkservices.googleapis.com/v1alpha1/projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -updateTime: '2025-04-03T09:24:45.928312100Z' -... - -I0403 14:54:49.019686 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.0_create_mesh --- -I0403 14:54:49.019819 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.1_start_cloud_run_test_server --- -I0403 14:54:49.746824 8252002176 cloud_run_xds_server_runner.py:62] Starting cloud run server with service emchandwani-psm-server-20250403-0924-5i0rd and image us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x -I0403 14:54:49.746951 8252002176 cloud_run.py:134] Deploying Cloud Run service 'emchandwani-psm-server-20250403-0924-5i0rd' -I0403 14:54:49.748914 8252002176 api.py:543] Creating run resource: ---- -launch_stage: alpha -template: - containers: - - image: us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x - ports: - - containerPort: 8080 - name: h2c -... - -I0403 14:54:52.626143 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/5dd1d9a3-001c-4924-a43a-d6d0abfe82ed', 'metadata': {'@type': 'type.googleapis.com/google.cloud.run.v2.Service', 'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd', 'uid': 'b0977056-29b5-45b8-b626-8c526e436119', 'generation': '1', 'createTime': '2025-04-03T09:24:52.130328Z', 'updateTime': '2025-04-03T09:24:52.130328Z', 'creator': 'emchandwani@google.com', 'lastModifier': 'emchandwani@google.com', 'launchStage': 'GA', 'template': {'scaling': {'maxInstanceCount': 100}, 'timeout': '300s', 'serviceAccount': '921384807982-compute@developer.gserviceaccount.com', 'containers': [{'image': 'us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x', 'resources': {'limits': {'memory': '512Mi', 'cpu': '1000m'}, 'cpuIdle': True}, 'ports': [{'name': 'h2c', 'containerPort': 8080}], 'startupProbe': {'timeoutSeconds': 240, 'periodSeconds': 240, 'failureThreshold': 1, 'tcpSocket': {'port': 8080}}}], 'maxInstanceRequestConcurrency': 80}, 'traffic': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'urls': ['https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app'], 'terminalCondition': {'state': 'CONDITION_PENDING'}, 'reconciling': True, 'etag': '"COSnub8GEMDLkj4/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"'}} -I0403 14:54:52.626596 8252002176 api.py:602] Waiting 600 sec for run operation id: projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/5dd1d9a3-001c-4924-a43a-d6d0abfe82ed -I0403 14:54:52.628556 8252002176 api.py:612] eshita Waiting for operation: -I0403 14:55:02.208549 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd: ---- -conditions: -- lastTransitionTime: '2025-04-03T09:24:58.430396Z' - state: CONDITION_SUCCEEDED - type: RoutesReady -- lastTransitionTime: '2025-04-03T09:24:57.419395Z' - state: CONDITION_SUCCEEDED - type: ConfigurationsReady -createTime: '2025-04-03T09:24:52.130328Z' -creator: emchandwani@google.com -etag: '"COSnub8GEMDLkj4/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"' -generation: '1' -ingress: INGRESS_TRAFFIC_ALL -lastModifier: emchandwani@google.com -latestCreatedRevision: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l -latestReadyRevision: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l -launchStage: GA -name: projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd -observedGeneration: '1' -template: - containers: - - image: us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x - ports: - - containerPort: 8080 - name: h2c - resources: - cpuIdle: true - limits: - cpu: 1000m - memory: 512Mi - startupProbe: - failureThreshold: 1 - periodSeconds: 240 - tcpSocket: - port: 8080 - timeoutSeconds: 240 - maxInstanceRequestConcurrency: 80 - scaling: - maxInstanceCount: 100 - serviceAccount: 921384807982-compute@developer.gserviceaccount.com - timeout: 300s -terminalCondition: - lastTransitionTime: '2025-04-03T09:24:58.473882Z' - state: CONDITION_SUCCEEDED - type: Ready -traffic: -- percent: 100 - type: TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST -trafficStatuses: -- percent: 100 - type: TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST -uid: b0977056-29b5-45b8-b626-8c526e436119 -updateTime: '2025-04-03T09:24:52.130328Z' -uri: https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app -urls: -- https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app -- https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app -... - -I0403 14:55:02.209048 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.1_start_cloud_run_test_server --- -I0403 14:55:02.209176 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.2_create_serverless_neg --- -I0403 14:55:02.209264 8252002176 xds_k8s_testcase.py:1504] Creating serverless NEG -I0403 14:55:02.209345 8252002176 compute.py:590] Creating serverless NEG emchandwani-psm-neg-20250403-0924-5i0rd in us-east1 -I0403 14:55:05.450768 8252002176 compute.py:595] Created serverless neg {'kind': 'compute#networkEndpointGroup', 'id': '2166457725700271360', 'creationTimestamp': '2025-04-03T02:25:03.766-07:00', 'selfLink': 'https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd', 'name': 'emchandwani-psm-neg-20250403-0924-5i0rd', 'networkEndpointType': 'SERVERLESS', 'size': 0, 'region': 'https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1', 'cloudRun': {'service': 'emchandwani-psm-server-20250403-0924-5i0rd'}} -I0403 14:55:05.451114 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.2_create_serverless_neg --- -I0403 14:55:05.451211 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.3_create_backend_service --- -I0403 14:55:05.451360 8252002176 traffic_director.py:253] Creating HTTP2 Backend Service "emchandwani-psm-backend-service-20250403-0924-5i0rd" -I0403 14:55:05.465883 8252002176 compute.py:662] Creating compute resource: ---- -loadBalancingScheme: INTERNAL_SELF_MANAGED -name: emchandwani-psm-backend-service-20250403-0924-5i0rd -protocol: HTTP2 -... - -I0403 14:55:06.219388 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672305646-631dc5861fd32-6386d9d6-95223583 -I0403 14:55:25.550782 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.3_create_backend_service --- -I0403 14:55:25.550887 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.4_add_server_backends_to_backend_service --- -I0403 14:55:25.550961 8252002176 traffic_director_cloud_run.py:77] Adding backends to Backend Service emchandwani-psm-backend-service-20250403-0924-5i0rd: [{'group': ComputeV1.GcpResource(name='emchandwani-psm-neg-20250403-0924-5i0rd', url='https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd'), 'balancingMode': 'us-east1', 'maxRatePerEndpoint': None, 'capacityScaler': 1.0}] -I0403 14:55:25.554704 8252002176 compute.py:669] Patching compute resource: ---- -backends: -- group: https://www.googleapis.com/compute/v1/projects/bct-gcestaging-guitar-grpc/regions/us-east1/networkEndpointGroups/emchandwani-psm-neg-20250403-0924-5i0rd -... - -I0403 14:55:27.369940 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672325737-631dc59948d81-608bf3ee-0cf3eed7 -I0403 14:56:29.219877 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.4_add_server_backends_to_backend_service --- -I0403 14:56:29.220246 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.5_create_grpc_route --- -I0403 14:56:29.220464 8252002176 traffic_director.py:978] Creating GrpcRoute emchandwani-psm-grpc-route-20250403-0924-5i0rd -I0403 14:56:29.225157 8252002176 api.py:543] Creating networkservices resource: ---- -hostnames: psm-grpc-server:11198 -meshes: -- projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -rules: -- action: - destinations: - - serviceName: projects/bct-gcestaging-guitar-grpc/locations/global/backendServices/emchandwani-psm-backend-service-20250403-0924-5i0rd -... - -I0403 14:56:30.814513 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672390512-631dc5d70f075-8b81644c-f50f08b6', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:26:30.686892159Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd', 'verb': 'create', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} -I0403 14:56:30.814918 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672390512-631dc5d70f075-8b81644c-f50f08b6 -I0403 14:56:30.817574 8252002176 api.py:612] eshita Waiting for operation: -I0403 14:56:39.134803 8252002176 api.py:566] Loaded projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd: ---- -createTime: '2025-04-03T09:26:30.665875084Z' -hostnames: -- psm-grpc-server:11198 -meshes: -- projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -name: projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd -rules: -- action: - destinations: - - serviceName: projects/bct-gcestaging-guitar-grpc/locations/global/backendServices/emchandwani-psm-backend-service-20250403-0924-5i0rd -selfLink: https://networkservices.googleapis.com/v1alpha1/projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd -updateTime: '2025-04-03T09:26:36.441076738Z' -... - -I0403 14:56:39.135174 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.5_create_grpc_route --- -I0403 14:56:39.135293 8252002176 xds_k8s_testcase.py:340] --- Starting subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.6_start_test_client --- -I0403 14:56:39.135482 8252002176 k8s_xds_client_runner.py:136] Deploying xDS test client "psm-grpc-client" to k8s namespace emchandwani-psm-client-20250403-0924-5i0rd: server_target=xds:///psm-grpc-server:11198 rpc=UnaryCall qps=25 metadata='' secure_mode=True print_response=False is_trusted_xds_server_experimental=True -I0403 14:56:39.135941 8252002176 k8s_base_runner.py:1171] GCP Logs Explorer link to psm-grpc-client: -https://console.cloud.google.com/logs/query;query=resource.type%3D%22k8s_container%22%0Aresource.labels.project_id%3D%22bct-gcestaging-guitar-grpc%22%0Aresource.labels.container_name%3D%22psm-grpc-client%22%0Aresource.labels.namespace_name%3D%22emchandwani-psm-client-20250403-0924-5i0rd%22;timeRange=2025-04-03T09:26:39.135656Z%2F2025-04-03T09:56:39.135656Z?project=bct-gcestaging-guitar-grpc -I0403 14:56:39.144119 8252002176 k8s_base_runner.py:351] Rendered template kubernetes-manifests/namespace.yaml: ---- -apiVersion: v1 -kind: Namespace -metadata: - name: emchandwani-psm-client-20250403-0924-5i0rd - labels: - name: emchandwani-psm-client-20250403-0924-5i0rd - owner: xds-k8s-interop-test -... - -W0403 14:57:54.148485 8252002176 connectionpool.py:823] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces -W0403 14:59:09.151218 8252002176 connectionpool.py:823] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces -W0403 15:00:24.153978 8252002176 connectionpool.py:823] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /api/v1/namespaces -I0403 15:01:39.157646 8252002176 xds_k8s_testcase.py:345] --- Finished subTest CloudRunCsmInboundTest.test_gke_to_cloud_run.6_start_test_client --- -I0403 15:01:39.157891 8252002176 xds_k8s_testcase.py:1526] ----- TestMethod CloudRunCsmInboundTest.test_gke_to_cloud_run teardown ----- -I0403 15:01:39.158314 8252002176 cloud_run_base_runner.py:196] Deleting Cloud Run service: emchandwani-psm-server-20250403-0924-5i0rd -I0403 15:01:41.273350 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/810c3f49-c231-499e-8713-40e0de5e68db', 'metadata': {'@type': 'type.googleapis.com/google.cloud.run.v2.Service', 'name': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd', 'uid': 'b0977056-29b5-45b8-b626-8c526e436119', 'generation': '2', 'createTime': '2025-04-03T09:24:52.130328Z', 'updateTime': '2025-04-03T09:31:41.167940Z', 'deleteTime': '2025-04-03T09:31:41.167940Z', 'expireTime': '2025-05-03T09:31:41.167940Z', 'creator': 'emchandwani@google.com', 'lastModifier': 'emchandwani@google.com', 'ingress': 'INGRESS_TRAFFIC_ALL', 'launchStage': 'GA', 'template': {'scaling': {'maxInstanceCount': 100}, 'timeout': '300s', 'serviceAccount': '921384807982-compute@developer.gserviceaccount.com', 'containers': [{'image': 'us-docker.pkg.dev/grpc-testing/psm-interop/cpp-server:v1.71.x', 'resources': {'limits': {'memory': '512Mi', 'cpu': '1000m'}, 'cpuIdle': True}, 'ports': [{'name': 'h2c', 'containerPort': 8080}], 'startupProbe': {'timeoutSeconds': 240, 'periodSeconds': 240, 'failureThreshold': 1, 'tcpSocket': {'port': 8080}}}], 'maxInstanceRequestConcurrency': 80}, 'traffic': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'urls': ['https://emchandwani-psm-server-20250403-0924-5i0rd-921384807982.us-east1.run.app', 'https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app'], 'observedGeneration': '1', 'terminalCondition': {'type': 'Ready', 'state': 'CONDITION_PENDING', 'lastTransitionTime': '2025-04-03T09:24:58.473882Z'}, 'conditions': [{'type': 'RoutesReady', 'state': 'CONDITION_SUCCEEDED', 'lastTransitionTime': '2025-04-03T09:24:58.430396Z'}, {'type': 'ConfigurationsReady', 'state': 'CONDITION_SUCCEEDED', 'lastTransitionTime': '2025-04-03T09:24:57.419395Z'}], 'latestReadyRevision': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l', 'latestCreatedRevision': 'projects/bct-gcestaging-guitar-grpc/locations/us-east1/services/emchandwani-psm-server-20250403-0924-5i0rd/revisions/emchandwani-psm-server-20250403-0924-5i0rd-00001-p6l', 'trafficStatuses': [{'type': 'TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST', 'percent': 100}], 'uri': 'https://emchandwani-psm-server-20250403-0924-5i0rd-ouuwihx2qq-ue.a.run.app', 'reconciling': True, 'etag': '"CP2qub8GEKCfilA/cHJvamVjdHMvYmN0LWdjZXN0YWdpbmctZ3VpdGFyLWdycGMvbG9jYXRpb25zL3VzLWVhc3QxL3NlcnZpY2VzL2VtY2hhbmR3YW5pLXBzbS1zZXJ2ZXItMjAyNTA0MDMtMDkyNC01aTByZA"'}} -I0403 15:01:41.273536 8252002176 api.py:602] Waiting 600 sec for run operation id: projects/bct-gcestaging-guitar-grpc/locations/us-east1/operations/810c3f49-c231-499e-8713-40e0de5e68db -I0403 15:01:41.274407 8252002176 api.py:612] eshita Waiting for operation: -I0403 15:01:42.965970 8252002176 cloud_run_base_runner.py:199] Deleted cloud run service: emchandwani-psm-server-20250403-0924-5i0rd -I0403 15:01:42.966464 8252002176 traffic_director.py:1007] Deleting GrpcRoute emchandwani-psm-grpc-route-20250403-0924-5i0rd -I0403 15:01:44.544538 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672704339-631dc70258d59-c4ac7305-8032a00b', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:31:44.409306336Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/grpcRoutes/emchandwani-psm-grpc-route-20250403-0924-5i0rd', 'verb': 'delete', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} -I0403 15:01:44.544960 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672704339-631dc70258d59-c4ac7305-8032a00b -I0403 15:01:44.547852 8252002176 api.py:612] eshita Waiting for operation: -I0403 15:01:52.432933 8252002176 traffic_director.py:961] Deleting Mesh emchandwani-psm-grpc-mesh-20250403-0924-5i0rd -I0403 15:01:53.856508 8252002176 api.py:593] Operation {'name': 'projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672713631-631dc70b35847-0776a9a5-e1e5f33d', 'metadata': {'@type': 'type.googleapis.com/google.cloud.networkservices.v1.OperationMetadata', 'createTime': '2025-04-03T09:31:53.684546316Z', 'target': 'projects/bct-gcestaging-guitar-grpc/locations/global/meshes/emchandwani-psm-grpc-mesh-20250403-0924-5i0rd', 'verb': 'delete', 'requestedCancellation': False, 'apiVersion': 'v1'}, 'done': False} -I0403 15:01:53.856900 8252002176 api.py:602] Waiting 600 sec for networkservices operation id: projects/bct-gcestaging-guitar-grpc/locations/global/operations/operation-1743672713631-631dc70b35847-0776a9a5-e1e5f33d -I0403 15:01:53.858998 8252002176 api.py:612] eshita Waiting for operation: -I0403 15:01:58.510071 8252002176 traffic_director.py:279] Deleting Backend Service "emchandwani-psm-backend-service-20250403-0924-5i0rd" -I0403 15:01:59.445646 8252002176 compute.py:739] Waiting 600 sec for compute operation id: operation-1743672718908-631dc7103daeb-52d70a92-fb70e80c -I0403 15:02:14.751783 8252002176 compute.py:606] Deleting serverless NEG emchandwani-psm-neg-20250403-0924-5i0rd in us-east1 -I0403 15:02:16.436823 8252002176 compute.py:612] Deleted serverless neg : emchandwani-psm-neg-20250403-0924-5i0rd -I0403 15:02:16.437970 8252002176 xds_k8s_testcase.py:1546] ----- Test client/server logs ----- -I0403 15:02:16.438210 8252002176 k8s_base_runner.py:1114] No completed deployments of psm-grpc-client -I0403 15:02:16.438795 8252002176 cloud_run_base_runner.py:172] GCP Logs Explorer link to emchandwani-psm-server-20250403-0924-5i0rd: -https://console.cloud.google.com/logs/query;query=resource.type%3D%22cloud_run_revision%22%0Aresource.labels.project_id%3D%22bct-gcestaging-guitar-grpc%22%0Aresource.labels.service_name%3D%22emchandwani-psm-server-20250403-0924-5i0rd%22%0Aresource.labels.location%3D%22us-east1%22;timeRange=2025-04-03T09:24:49.746934Z%2F2025-04-03T09:31:42.966335Z;cursorTimestamp=2025-04-03T09:25:02.208940Z?project=bct-gcestaging-guitar-grpc -[ FAILED ] CloudRunCsmInboundTest.test_gke_to_cloud_run -E0403 15:02:16.467079 8252002176 base_testcase.py:44] ----- PSM Test Case FAILED: CloudRunCsmInboundTest.test_gke_to_cloud_run ----- -E0403 15:02:16.467297 8252002176 base_testcase.py:145] (ERROR) PSM Interop Test Failed: CloudRunCsmInboundTest.test_gke_to_cloud_run -^^^^^ -# [CloudRunCsmInboundTest.test_gke_to_cloud_run] PSM Failed Test Traceback BEGIN +I0408 11:25:37.223489 8473956224 xds_k8s_testcase.py:177] ----- Testing CloudRunCsmInboundTest ----- +I0408 11:25:37.223536 8473956224 xds_k8s_testcase.py:178] Logs timezone: IST +I0408 11:25:37.223610 8473956224 xds_k8s_testcase.py:106] Detected language and version: TestConfig(client_lang='cpp', server_lang='cpp', version=None) +I0408 11:25:39.002089 8473956224 k8s.py:264] Using kubernetes context "gke_emchandwani-default_us-central1-a_xds-k8s-interop-tests-cluster5", active host: https://34.55.128.246 +W0408 11:26:54.009544 8473956224 connectionpool.py:823] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /version Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn - conn = connection.create_connection( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection - raise err - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection - sock.connect(sa) -TimeoutError: [Errno 60] Operation timed out - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen - httplib_response = self._make_request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request - self._validate_conn(conn) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn - conn.connect() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect - self.sock = conn = self._new_conn() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 179, in _new_conn - raise ConnectTimeoutError( -urllib3.exceptions.ConnectTimeoutError: (, 'Connection to 34.55.128.246 timed out. (connect timeout=None)') - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 55, in test_gke_to_cloud_run - test_client: _XdsTestClient = self.startSecureTestClient( - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1177, in startSecureTestClient - return self._start_test_client( - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 951, in _start_test_client - test_client = self.client_runner.run( - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_xds_client_runner.py", line 152, in run - super().run() - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 196, in run - self.namespace = self._create_namespace( - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 420, in _create_namespace - namespace = self._create_from_template(template, **kwargs) - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 366, in _create_from_template - k8s_object = self.k8s_namespace.create_single_resource( - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 589, in create_single_resource - return self._execute(self._apply_manifest_single, manifest) - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 453, in _execute - return method(*args, **kwargs) - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 390, in _apply_manifest_single - k8s_objects = utils.create_from_dict( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 221, in create_from_dict - created = create_from_yaml_single_item( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 263, in create_from_yaml_single_item - resp = getattr(k8s_api, "create_{0}".format(kind))( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6363, in create_namespace - return self.create_namespace_with_http_info(body, **kwargs) # noqa: E501 - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6454, in create_namespace_with_http_info - return self.api_client.call_api( + File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 69, in + absltest.main(failfast=True) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2131, in main + _run_in_app(run_tests, args, kwargs) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2236, in _run_in_app + app.run(main=main_function) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/app.py", line 308, in run + _run_main(main, args) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/app.py", line 254, in _run_main + sys.exit(main(argv)) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2234, in main_function + function(argv, args, kwargs) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2689, in run_tests + result, fail_when_no_tests_ran = _run_and_get_tests_result( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2653, in _run_and_get_tests_result + test_program = unittest.TestProgram(*args, **kwargs) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/main.py", line 101, in __init__ + self.runTests() + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/main.py", line 271, in runTests + self.result = testRunner.run(self.test) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/_pretty_print_reporter.py", line 82, in run + return super(TextTestRunner, self).run(test) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/runner.py", line 184, in run + test(result) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 84, in __call__ + return self.run(*args, **kwds) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 122, in run + test(result) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 84, in __call__ + return self.run(*args, **kwds) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 114, in run + self._handleClassSetUp(test, result) + File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 166, in _handleClassSetUp + setUpClass() + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1428, in setUpClass + super().setUpClass() + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1082, in setUpClass + super().setUpClass() + File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 247, in setUpClass + cls.k8s_api_manager = k8s.KubernetesApiManager( + File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 145, in __init__ + self._dynamic_client = dynamic.DynamicClient(self._client) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 84, in __init__ + self.__discoverer = discoverer(self, cache_file) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 228, in __init__ + Discoverer.__init__(self, client, cache_file) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 54, in __init__ + self.__init_cache() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 70, in __init_cache + self._load_server_info() + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 145, in _load_server_info + 'kubernetes': self.client.request('get', '/version', serializer=just_json) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 55, in inner + resp = func(self, *args, **kwargs) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 273, in request + api_response = self.client.call_api( File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api return self.__call_api(resource_path, method, File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api response_data = self.request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 391, in request - return self.rest_client.POST(url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 279, in POST - return self.request("POST", url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 172, in request - r = self.pool_manager.request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 78, in request - return self.request_encode_body( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 170, in request_encode_body + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 217, in request + r = self.pool_manager.request(method, url, + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 74, in request + return self.request_encode_url( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 96, in request_encode_url return self.urlopen(method, url, **extra_kw) File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/poolmanager.py", line 376, in urlopen response = conn.urlopen(method, u.request_uri, **kw) File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen - retries = retries.increment( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment - raise MaxRetryError(_pool, url, error or ResponseError(cause)) -urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='34.55.128.246', port=443): Max retries exceeded with url: /api/v1/namespaces (Caused by ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')) -# [CloudRunCsmInboundTest.test_gke_to_cloud_run] PSM Failed Test Traceback END - -I0403 15:02:16.467360 8252002176 base_testcase.py:49] ----- PSM Test Case CloudRunCsmInboundTest.test_gke_to_cloud_run Error Count: 1 ----- -====================================================================== -ERROR: test_gke_to_cloud_run (__main__.CloudRunCsmInboundTest) -CloudRunCsmInboundTest.test_gke_to_cloud_run ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn - conn = connection.create_connection( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection - raise err - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection - sock.connect(sa) -TimeoutError: [Errno 60] Operation timed out - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen httplib_response = self._make_request( File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request @@ -318,70 +89,8 @@ Traceback (most recent call last): conn.connect() File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect self.sock = conn = self._new_conn() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 179, in _new_conn - raise ConnectTimeoutError( -urllib3.exceptions.ConnectTimeoutError: (, 'Connection to 34.55.128.246 timed out. (connect timeout=None)') - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 55, in test_gke_to_cloud_run - test_client: _XdsTestClient = self.startSecureTestClient( - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1177, in startSecureTestClient - return self._start_test_client( - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 951, in _start_test_client - test_client = self.client_runner.run( - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_xds_client_runner.py", line 152, in run - super().run() - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 196, in run - self.namespace = self._create_namespace( - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 420, in _create_namespace - namespace = self._create_from_template(template, **kwargs) - File "/Users/emchandwani/psm-interop/framework/test_app/runners/k8s/k8s_base_runner.py", line 366, in _create_from_template - k8s_object = self.k8s_namespace.create_single_resource( - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 589, in create_single_resource - return self._execute(self._apply_manifest_single, manifest) - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 453, in _execute - return method(*args, **kwargs) - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 390, in _apply_manifest_single - k8s_objects = utils.create_from_dict( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 221, in create_from_dict - created = create_from_yaml_single_item( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/utils/create_from_yaml.py", line 263, in create_from_yaml_single_item - resp = getattr(k8s_api, "create_{0}".format(kind))( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6363, in create_namespace - return self.create_namespace_with_http_info(body, **kwargs) # noqa: E501 - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api/core_v1_api.py", line 6454, in create_namespace_with_http_info - return self.api_client.call_api( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 391, in request - return self.rest_client.POST(url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 279, in POST - return self.request("POST", url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 172, in request - r = self.pool_manager.request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 78, in request - return self.request_encode_body( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 170, in request_encode_body - return self.urlopen(method, url, **extra_kw) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/poolmanager.py", line 376, in urlopen - response = conn.urlopen(method, u.request_uri, **kw) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen - retries = retries.increment( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment - raise MaxRetryError(_pool, url, error or ResponseError(cause)) -urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='34.55.128.246', port=443): Max retries exceeded with url: /api/v1/namespaces (Caused by ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')) - ----------------------------------------------------------------------- -Ran 1 test in 463.787s - -FAILED (errors=1) + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn + conn = connection.create_connection( + File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + sock.connect(sa) +KeyboardInterrupt diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index c12a9a87..0ff05141 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -588,9 +588,12 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): "cloudRun": {"service": service_name}, } logger.info("Creating serverless NEG %s in %s", name, region) - self.api.regionNetworkEndpointGroups().insert( - project=self.project, region=region, body=neg_body - ).execute() + self._insert_resource( + self.api.regionNetworkEndpointGroups() , + neg_body, + region=region, + ) + neg = self.get_serverless_network_endpoint_group(name, region) logger.info("Created serverless neg %s ", neg) return neg @@ -604,6 +607,7 @@ def delete_serverless_neg(self, name: str, region: str): """ try: logger.info("Deleting serverless NEG %s in %s", name, region) + self._delete_resource(self.api.regionNetworkEndpointGroups(),resource_type="networkEndpointGroup",resource_name=name,region=region) self.api.regionNetworkEndpointGroups().delete( project=self.project, region=region, @@ -625,15 +629,8 @@ def delete_serverless_neg(self, name: str, region: str): raise def get_serverless_network_endpoint_group(self, name, region): - neg = ( - self.api.regionNetworkEndpointGroups() - .get( - project=self.project, - networkEndpointGroup=name, - region=region, - ) - .execute() - ) + neg=self._get_resource(self.api.regionNetworkEndpointGroups(),networkEndpointGroup=name, + region=region) return neg def _get_resource( @@ -657,12 +654,15 @@ def _exists_resource( return "items" in resp and resp["items"] def _insert_resource( - self, collection: discovery.Resource, body: Dict[str, Any] + self, collection: discovery.Resource, body: Dict[str, Any],region:str = None ) -> "GcpResource": logger.info( "Creating compute resource:\n%s", self.resource_pretty_format(body) ) - resp = self._execute(collection.insert(project=self.project, body=body)) + if region: + resp=self._execute(collection.insert(project=self.project, region=region,body=body),region=region) + else: + resp = self._execute(collection.insert(project=self.project, body=body)) return self.GcpResource(body["name"], resp["targetLink"]) def _patch_resource(self, collection, body, **kwargs): @@ -683,9 +683,12 @@ def _delete_resource( collection: discovery.Resource, resource_type: str, resource_name: str, + region: str = None, ) -> bool: try: params = {"project": self.project, resource_type: resource_name} + if region: + params["region"] = region self._execute(collection.delete(**params)) return True except googleapiclient.errors.HttpError as error: @@ -721,7 +724,7 @@ def _log_debug_header(resp: httplib2.Response): ) def _execute( # pylint: disable=arguments-differ - self, request, *, timeout_sec=_WAIT_FOR_OPERATION_SEC + self, request, *, timeout_sec=_WAIT_FOR_OPERATION_SEC,region:str=None ): if self.gfe_debug_header: logger.debug( @@ -730,11 +733,11 @@ def _execute( # pylint: disable=arguments-differ request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header request.add_response_callback(self._log_debug_header) operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.debug("Operation %s", operation) - return self._wait(operation["name"], timeout_sec) + logger.info("Operation %s", operation) + return self._wait(operation["name"], timeout_sec,region) def _wait( - self, operation_id: str, timeout_sec: int = _WAIT_FOR_OPERATION_SEC + self, operation_id: str, timeout_sec: int = _WAIT_FOR_OPERATION_SEC,region:str=None ) -> dict: logger.info( "Waiting %s sec for compute operation id: %s", @@ -744,7 +747,12 @@ def _wait( # TODO(sergiitk) try using wait() here # https://googleapis.github.io/google-api-python-client/docs/dyn/compute_v1.globalOperations.html#wait - op_request = self.api.globalOperations().get( + if region: + op_request = self.api.regionOperations().get( + project=self.project, operation=operation_id,region=region + ) + else: + op_request = self.api.globalOperations().get( project=self.project, operation=operation_id ) operation = self.wait_for_operation( diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 7972abc5..231400ff 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -40,11 +40,7 @@ def test_gke_to_cloud_run(self): ) with self.subTest("4_add_server_backends_to_backend_service"): - neg_resource = self.compute_v1.GcpResource( - name=neg["name"], - url=neg["selfLink"], - ) - self.td.backend_service_add_backends([neg_resource], self.region) + self.td.backend_service_add_backends([neg], self.region) with self.subTest("5_create_grpc_route"): self.td.create_grpc_route( From 8d0f40ed7594775fa9e997c6058f1d6d4f784675 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 9 Apr 2025 11:27:52 +0530 Subject: [PATCH 31/49] correct keyerror --- bin/lib/common.py | 1 + eshita_logs.txt | 96 ------------------- framework/infrastructure/gcp/api.py | 36 +++---- framework/infrastructure/gcp/cloud_run.py | 2 - framework/infrastructure/gcp/compute.py | 79 ++++++++------- framework/xds_k8s_testcase.py | 2 +- .../client-secure.deployment.yaml | 13 ++- kubernetes-manifests/client.deployment.yaml | 83 +--------------- 8 files changed, 73 insertions(+), 239 deletions(-) delete mode 100644 eshita_logs.txt diff --git a/bin/lib/common.py b/bin/lib/common.py index 08ae9158..eb9d3b05 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -125,6 +125,7 @@ def make_server_namespace( ) return k8s.KubernetesNamespace(k8s_api_manager(), namespace_name) + def make_server_runner( namespace: k8s.KubernetesNamespace, *, diff --git a/eshita_logs.txt b/eshita_logs.txt deleted file mode 100644 index cfdc144c..00000000 --- a/eshita_logs.txt +++ /dev/null @@ -1,96 +0,0 @@ -Saving framework log to /Users/emchandwani/psm-interop/out/run/20250408-112536-cloud_run_csm_inbound_test.log -+ [11:25:36 IST] exec python tests/cloud_run_csm_inbound_test.py --flagfile=/Users/emchandwani/psm-interop/config/local-dev.cfg -Running tests under Python 3.10.16: /Users/emchandwani/psm-interop/venv/bin/python -I0408 11:25:37.223489 8473956224 xds_k8s_testcase.py:177] ----- Testing CloudRunCsmInboundTest ----- -I0408 11:25:37.223536 8473956224 xds_k8s_testcase.py:178] Logs timezone: IST -I0408 11:25:37.223610 8473956224 xds_k8s_testcase.py:106] Detected language and version: TestConfig(client_lang='cpp', server_lang='cpp', version=None) -I0408 11:25:39.002089 8473956224 k8s.py:264] Using kubernetes context "gke_emchandwani-default_us-central1-a_xds-k8s-interop-tests-cluster5", active host: https://34.55.128.246 -W0408 11:26:54.009544 8473956224 connectionpool.py:823] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(, 'Connection to 34.55.128.246 timed out. (connect timeout=None)')': /version -Traceback (most recent call last): - File "/Users/emchandwani/psm-interop/tests/cloud_run_csm_inbound_test.py", line 69, in - absltest.main(failfast=True) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2131, in main - _run_in_app(run_tests, args, kwargs) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2236, in _run_in_app - app.run(main=main_function) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/app.py", line 308, in run - _run_main(main, args) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/app.py", line 254, in _run_main - sys.exit(main(argv)) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2234, in main_function - function(argv, args, kwargs) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2689, in run_tests - result, fail_when_no_tests_ran = _run_and_get_tests_result( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/absltest.py", line 2653, in _run_and_get_tests_result - test_program = unittest.TestProgram(*args, **kwargs) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/main.py", line 101, in __init__ - self.runTests() - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/main.py", line 271, in runTests - self.result = testRunner.run(self.test) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/absl/testing/_pretty_print_reporter.py", line 82, in run - return super(TextTestRunner, self).run(test) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/runner.py", line 184, in run - test(result) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 84, in __call__ - return self.run(*args, **kwds) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 122, in run - test(result) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 84, in __call__ - return self.run(*args, **kwds) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 114, in run - self._handleClassSetUp(test, result) - File "/opt/homebrew/Cellar/python@3.10/3.10.16/Frameworks/Python.framework/Versions/3.10/lib/python3.10/unittest/suite.py", line 166, in _handleClassSetUp - setUpClass() - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1428, in setUpClass - super().setUpClass() - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 1082, in setUpClass - super().setUpClass() - File "/Users/emchandwani/psm-interop/framework/xds_k8s_testcase.py", line 247, in setUpClass - cls.k8s_api_manager = k8s.KubernetesApiManager( - File "/Users/emchandwani/psm-interop/framework/infrastructure/k8s.py", line 145, in __init__ - self._dynamic_client = dynamic.DynamicClient(self._client) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 84, in __init__ - self.__discoverer = discoverer(self, cache_file) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 228, in __init__ - Discoverer.__init__(self, client, cache_file) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 54, in __init__ - self.__init_cache() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 70, in __init_cache - self._load_server_info() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/discovery.py", line 145, in _load_server_info - 'kubernetes': self.client.request('get', '/version', serializer=just_json) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 55, in inner - resp = func(self, *args, **kwargs) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/dynamic/client.py", line 273, in request - api_response = self.client.call_api( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/kubernetes/client/rest.py", line 217, in request - r = self.pool_manager.request(method, url, - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 74, in request - return self.request_encode_url( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/request.py", line 96, in request_encode_url - return self.urlopen(method, url, **extra_kw) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/poolmanager.py", line 376, in urlopen - response = conn.urlopen(method, u.request_uri, **kw) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 826, in urlopen - return self.urlopen( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen - httplib_response = self._make_request( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request - self._validate_conn(conn) - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn - conn.connect() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect - self.sock = conn = self._new_conn() - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn - conn = connection.create_connection( - File "/Users/emchandwani/psm-interop/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection - sock.connect(sa) -KeyboardInterrupt diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index e394fc38..f6109baf 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -537,7 +537,6 @@ def _create_resource( collection: discovery.Resource, body: dict, location: Optional[str] = None, - is_cloudrun: bool = False, **kwargs, ): logger.info( @@ -549,7 +548,7 @@ def _create_resource( parent=self.parent(location), body=body, **kwargs ) - self._execute(create_req, is_cloudrun=is_cloudrun) + self._execute(create_req) @property @abc.abstractmethod @@ -572,13 +571,10 @@ def _delete_resource( self, collection: discovery.Resource, full_name: str, - is_cloudrun: bool = False, ) -> bool: logger.debug("Deleting %s", full_name) try: - self._execute( - collection.delete(name=full_name), is_cloudrun=is_cloudrun - ) + self._execute(collection.delete(name=full_name)) return True except _HttpError as error: if error.resp and error.resp.status == 404: @@ -592,17 +588,15 @@ def _execute( # pylint: disable=arguments-differ self, request: HttpRequest, timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, - is_cloudrun: bool = False, ): operation = request.execute(num_retries=self._GCP_API_RETRIES) logger.info("Operation %s", operation) - self._wait(operation["name"], timeout_sec, is_cloudrun) + self._wait(operation["name"], timeout_sec) def _wait( self, operation_id: str, timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, - is_cloudrun: bool = False, ): logger.info( "Waiting %s sec for %s operation id: %s", @@ -614,22 +608,16 @@ def _wait( op_request = ( self.api.projects().locations().operations().get(name=operation_id) ) - if is_cloudrun: - operation = self.wait_for_operation( - operation_request=op_request, - test_success_fn=lambda result: result["metadata"][ - "terminalCondition" - ]["state"] - == "CONDITION_SUCCEEDED", - timeout_sec=timeout_sec, - ) - else: - operation = self.wait_for_operation( - operation_request=op_request, - test_success_fn=lambda result: result["done"], - timeout_sec=timeout_sec, - ) + operation = self.wait_for_operation( + operation_request=op_request, + test_success_fn=self._operation_status_done, + timeout_sec=timeout_sec, + ) logger.debug("Completed operation: %s", operation) if "error" in operation: raise OperationError(self.api_name, operation) + + @staticmethod + def _operation_status_done(operation: dict[str, Any]) -> bool: + return operation.get("done") diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index e4990eba..1381ba8d 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -82,7 +82,6 @@ def create_service( location=self.region, serviceId=service_name, body=body, - is_cloudrun=True, ) def get_service( @@ -110,7 +109,6 @@ def delete_service(self, service_name: str): full_name=self.resource_full_name( service_name, "services", self.region ), - is_cloudrun=True, ) def deploy_service( diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 0ff05141..5fdd0989 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -580,16 +580,16 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): The NEG selfLink URL """ - # Note: Cloud Run is not the only service that uses serverless NEGs. - # See: https://cloud.google.com/load-balancing/docs/negs/serverless-neg-concepts neg_body = { "name": name, "networkEndpointType": "SERVERLESS", + # "Note: Cloud Run is not the only service that uses serverless NEGs + # but we are only supporting Cloud Run right now." "cloudRun": {"service": service_name}, } logger.info("Creating serverless NEG %s in %s", name, region) self._insert_resource( - self.api.regionNetworkEndpointGroups() , + self.api.regionNetworkEndpointGroups(), neg_body, region=region, ) @@ -605,32 +605,20 @@ def delete_serverless_neg(self, name: str, region: str): name: The name of the NEG to delete. zone: The zone of the NEG. """ - try: - logger.info("Deleting serverless NEG %s in %s", name, region) - self._delete_resource(self.api.regionNetworkEndpointGroups(),resource_type="networkEndpointGroup",resource_name=name,region=region) - self.api.regionNetworkEndpointGroups().delete( - project=self.project, - region=region, - networkEndpointGroup=name, - ).execute() - logger.info("Deleted serverless neg : %s", name) - except googleapiclient.errors.HttpError as error: - if error.resp.status == 404: # NEG not found - logger.debug( - "NEG %s not found in zone %s. Skipping deletion.", - name, - region, - ) - return - logger.exception("Error deleting serverless NEG: %s", error) - raise - except Exception as e: - logger.exception("Error deleting serverless NEG: %s", e) - raise + logger.info("Deleting serverless NEG %s in %s", name, region) + self._delete_resource( + self.api.regionNetworkEndpointGroups(), + resource_type="networkEndpointGroup", + resource_name=name, + region=region, + ) def get_serverless_network_endpoint_group(self, name, region): - neg=self._get_resource(self.api.regionNetworkEndpointGroups(),networkEndpointGroup=name, - region=region) + neg = self._get_resource( + self.api.regionNetworkEndpointGroups(), + networkEndpointGroup=name, + region=region, + ) return neg def _get_resource( @@ -654,15 +642,25 @@ def _exists_resource( return "items" in resp and resp["items"] def _insert_resource( - self, collection: discovery.Resource, body: Dict[str, Any],region:str = None + self, + collection: discovery.Resource, + body: Dict[str, Any], + region: str = None, ) -> "GcpResource": logger.info( "Creating compute resource:\n%s", self.resource_pretty_format(body) ) if region: - resp=self._execute(collection.insert(project=self.project, region=region,body=body),region=region) + resp = self._execute( + collection.insert( + project=self.project, region=region, body=body + ), + region=region, + ) else: - resp = self._execute(collection.insert(project=self.project, body=body)) + resp = self._execute( + collection.insert(project=self.project, body=body) + ) return self.GcpResource(body["name"], resp["targetLink"]) def _patch_resource(self, collection, body, **kwargs): @@ -724,7 +722,11 @@ def _log_debug_header(resp: httplib2.Response): ) def _execute( # pylint: disable=arguments-differ - self, request, *, timeout_sec=_WAIT_FOR_OPERATION_SEC,region:str=None + self, + request, + *, + timeout_sec=_WAIT_FOR_OPERATION_SEC, + region: str = None, ): if self.gfe_debug_header: logger.debug( @@ -734,10 +736,13 @@ def _execute( # pylint: disable=arguments-differ request.add_response_callback(self._log_debug_header) operation = request.execute(num_retries=self._GCP_API_RETRIES) logger.info("Operation %s", operation) - return self._wait(operation["name"], timeout_sec,region) + return self._wait(operation["name"], timeout_sec, region) def _wait( - self, operation_id: str, timeout_sec: int = _WAIT_FOR_OPERATION_SEC,region:str=None + self, + operation_id: str, + timeout_sec: int = _WAIT_FOR_OPERATION_SEC, + region: str = None, ) -> dict: logger.info( "Waiting %s sec for compute operation id: %s", @@ -749,12 +754,12 @@ def _wait( # https://googleapis.github.io/google-api-python-client/docs/dyn/compute_v1.globalOperations.html#wait if region: op_request = self.api.regionOperations().get( - project=self.project, operation=operation_id,region=region - ) + project=self.project, operation=operation_id, region=region + ) else: op_request = self.api.globalOperations().get( - project=self.project, operation=operation_id - ) + project=self.project, operation=operation_id + ) operation = self.wait_for_operation( operation_request=op_request, test_success_fn=self._operation_status_done, diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index d26fdf86..388e63eb 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1458,7 +1458,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: network=self.network, debug_use_port_forwarding=self.debug_use_port_forwarding, enable_workload_identity=self.enable_workload_identity, - deployment_template="client.deployment.yaml", + deployment_template="client-secure.deployment.yaml", stats_port=self.client_port, reuse_namespace=self.server_namespace == self.client_namespace, **kwargs, diff --git a/kubernetes-manifests/client-secure.deployment.yaml b/kubernetes-manifests/client-secure.deployment.yaml index 3842fd59..d09c3282 100644 --- a/kubernetes-manifests/client-secure.deployment.yaml +++ b/kubernetes-manifests/client-secure.deployment.yaml @@ -53,6 +53,14 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST value: "true" + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER + value: "true" + - name: GRPC_TRACE + value: "xds_client" volumeMounts: - mountPath: /tmp/grpc-xds/ name: grpc-td-conf @@ -75,7 +83,10 @@ spec: - "--xds-server-uri=${xds_server_uri}" % endif % if config_mesh: - - "--config-mesh-experimental=${config_mesh}" + - "--config_mesh=${config_mesh}" + % endif + % if is_trusted_xds_server_experimental: + - "--is-trusted-xds-server-experimental=true" % endif resources: limits: diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index b5cfa2b9..724238a7 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -41,9 +41,6 @@ spec: - "--qps=${qps}" - "--rpc=${rpc}" - "--metadata=${metadata}" - % if secure_mode: - - "--secure_mode=${secure_mode}" - % endif % if request_payload_size > 0: - "--request_payload_size=${request_payload_size}" % endif @@ -54,44 +51,9 @@ spec: % if enable_csm_observability: - "--enable_csm_observability=true" % endif - ports: - - containerPort: ${stats_port} + ## #################################################################### + ## TODO(cl/698639274): Remove the these env once this CL is released. env: - - name: GRPC_TRACE - value: "xds_client" - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY - value: "true" - - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - % if enable_dualstack: - - name: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS - value: "true" - % endif - % if csm_workload_name: - - name: CSM_WORKLOAD_NAME - value: ${csm_workload_name} - % endif - % if csm_canonical_service_name: - - name: CSM_CANONICAL_SERVICE_NAME - value: ${csm_canonical_service_name} - % endif - % if enable_csm_observability: - name: POD_NAME valueFrom: fieldRef: @@ -102,11 +64,9 @@ spec: fieldPath: metadata.namespace - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME) - % endif - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true + ## ##################################################################### + ports: + - containerPort: ${stats_port} resources: limits: cpu: 800m @@ -114,37 +74,4 @@ spec: requests: cpu: 100m memory: 512Mi - initContainers: - - name: grpc-td-init - image: ${td_bootstrap_image} - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - % if config_mesh: - - "--config-mesh=${config_mesh}" - % endif - % if generate_mesh_id: - - "--generate-mesh-id-experimental" - % endif - % if is_trusted_xds_server_experimental: - - "--is-trusted-xds-server-experimental=true" - % endif - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory ... From 1ad1b42409d45c8ffc829c991307513edb5e55ed Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 9 Apr 2025 13:27:10 +0530 Subject: [PATCH 32/49] change yaml --- framework/xds_k8s_testcase.py | 2 +- .../client-secure.deployment.yaml | 13 +-- kubernetes-manifests/client.deployment.yaml | 83 +++++++++++++++++-- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 388e63eb..d26fdf86 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -1458,7 +1458,7 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: network=self.network, debug_use_port_forwarding=self.debug_use_port_forwarding, enable_workload_identity=self.enable_workload_identity, - deployment_template="client-secure.deployment.yaml", + deployment_template="client.deployment.yaml", stats_port=self.client_port, reuse_namespace=self.server_namespace == self.client_namespace, **kwargs, diff --git a/kubernetes-manifests/client-secure.deployment.yaml b/kubernetes-manifests/client-secure.deployment.yaml index d09c3282..3842fd59 100644 --- a/kubernetes-manifests/client-secure.deployment.yaml +++ b/kubernetes-manifests/client-secure.deployment.yaml @@ -53,14 +53,6 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST value: "true" - - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER - value: "true" - - name: GRPC_TRACE - value: "xds_client" volumeMounts: - mountPath: /tmp/grpc-xds/ name: grpc-td-conf @@ -83,10 +75,7 @@ spec: - "--xds-server-uri=${xds_server_uri}" % endif % if config_mesh: - - "--config_mesh=${config_mesh}" - % endif - % if is_trusted_xds_server_experimental: - - "--is-trusted-xds-server-experimental=true" + - "--config-mesh-experimental=${config_mesh}" % endif resources: limits: diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 724238a7..b5cfa2b9 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -41,6 +41,9 @@ spec: - "--qps=${qps}" - "--rpc=${rpc}" - "--metadata=${metadata}" + % if secure_mode: + - "--secure_mode=${secure_mode}" + % endif % if request_payload_size > 0: - "--request_payload_size=${request_payload_size}" % endif @@ -51,9 +54,44 @@ spec: % if enable_csm_observability: - "--enable_csm_observability=true" % endif - ## #################################################################### - ## TODO(cl/698639274): Remove the these env once this CL is released. + ports: + - containerPort: ${stats_port} env: + - name: GRPC_TRACE + value: "xds_client" + - name: GRPC_XDS_BOOTSTRAP + value: "/tmp/grpc-xds/td-grpc-bootstrap.json" + - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER + value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH + value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY + value: "true" + - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST + value: "true" + % if enable_dualstack: + - name: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS + value: "true" + % endif + % if csm_workload_name: + - name: CSM_WORKLOAD_NAME + value: ${csm_workload_name} + % endif + % if csm_canonical_service_name: + - name: CSM_CANONICAL_SERVICE_NAME + value: ${csm_canonical_service_name} + % endif + % if enable_csm_observability: - name: POD_NAME valueFrom: fieldRef: @@ -64,9 +102,11 @@ spec: fieldPath: metadata.namespace - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME) - ## ##################################################################### - ports: - - containerPort: ${stats_port} + % endif + volumeMounts: + - mountPath: /tmp/grpc-xds/ + name: grpc-td-conf + readOnly: true resources: limits: cpu: 800m @@ -74,4 +114,37 @@ spec: requests: cpu: 100m memory: 512Mi + initContainers: + - name: grpc-td-init + image: ${td_bootstrap_image} + imagePullPolicy: Always + args: + - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" + - "--vpc-network-name=${network}" + % if xds_server_uri: + - "--xds-server-uri=${xds_server_uri}" + % endif + % if config_mesh: + - "--config-mesh=${config_mesh}" + % endif + % if generate_mesh_id: + - "--generate-mesh-id-experimental" + % endif + % if is_trusted_xds_server_experimental: + - "--is-trusted-xds-server-experimental=true" + % endif + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 100Mi + volumeMounts: + - mountPath: /tmp/bootstrap/ + name: grpc-td-conf + volumes: + - name: grpc-td-conf + emptyDir: + medium: Memory ... From d25b162b04d58ac574c35b26bf6d34e39fb3f1ca Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Wed, 9 Apr 2025 22:54:18 +0530 Subject: [PATCH 33/49] comments --- framework/infrastructure/gcp/api.py | 2 +- framework/infrastructure/gcp/cloud_run.py | 96 ++++++++----------- framework/infrastructure/gcp/compute.py | 4 +- .../cloud_run/cloud_run_base_runner.py | 25 +++-- .../cloud_run/cloud_run_xds_server_runner.py | 4 +- 5 files changed, 56 insertions(+), 75 deletions(-) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index f6109baf..211f0e2e 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -590,7 +590,7 @@ def _execute( # pylint: disable=arguments-differ timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, ): operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.info("Operation %s", operation) + logger.debug("Operation %s", operation) self._wait(operation["name"], timeout_sec) def _wait( diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 1381ba8d..8a6b7086 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -14,9 +14,7 @@ import abc import dataclasses import logging -from typing import Any, Dict, Final - -from googleapiclient import discovery +from typing import Any, Final from framework.infrastructure import gcp @@ -30,84 +28,75 @@ @dataclasses.dataclass(frozen=True) -class CloudRun: +class CloudRunService: service_name: str url: str @classmethod - def from_response(cls, name: str, d: Dict[str, Any]) -> "CloudRun": + def from_response( + cls, name: str, response: dict[str, Any] + ) -> "CloudRunService": return cls( service_name=name, - url=d["urls"], + url=response["urls"], ) class CloudRunApiManager( gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta ): - project: str region: str - _parent: str - service: discovery.Resource - api_manager: gcp.api.GcpApiManager - def __init__(self, project: str, region: str): + def __init__( + self, project: str, region: str, api_manager: gcp.api.GcpApiManager + ): if not project: raise ValueError("Project ID cannot be empty or None.") if not region: raise ValueError("Region cannot be empty or None.") - self.api_manager = gcp.api.GcpApiManager(v2_discovery_uri=DISCOVERY_URI) - self.project = project + # api_manager = gcp.api.GcpApiManager(v2_discovery_uri=DISCOVERY_URI) self.region = region - service: discovery.Resource = self.api_manager.cloudrun("v2") - self.service = service - self._parent = f"projects/{self.project}/locations/{self.region}" - super().__init__(self.service, project) + super().__init__(api_manager.cloudrun(self.api_version), project) + self._services_collection = self.api.projects().locations().services() + + SERVICES = "services" @property def api_name(self) -> str: """Returns the API name for Cloud Run.""" - return "run" + return "Cloud Run" @property def api_version(self) -> str: """Returns the API version for Cloud Run.""" return "v2" - def create_service( - self, service: discovery.Resource, service_name: str, body: dict - ) -> GcpResource: + def create_service(self, service_name: str, body: dict) -> GcpResource: return self._create_resource( - collection=service.projects().locations().services(), + collection=self._services_collection, location=self.region, serviceId=service_name, body=body, ) - def get_service( - self, service: discovery.Resource, service_name: str - ) -> CloudRun: + def get_service(self, service_name: str) -> CloudRunService: result = self._get_resource( - collection=service.projects().locations().services(), + collection=self._services_collection, full_name=self.resource_full_name( - service_name, "services", self.region + service_name, self.SERVICES, self.region ), ) - return CloudRun.from_response( - self.resource_full_name(service_name, "services", self.region), + return CloudRunService.from_response( + self.resource_full_name(service_name, self.SERVICES, self.region), result, ) - def get_service_uri(self, service_name: str) -> str: - response = self.get_service(self.service, service_name) - return response.url - def delete_service(self, service_name: str): # pylint: disable=no-member self._delete_resource( - self.service.projects().locations().services(), + self._services_collection, full_name=self.resource_full_name( - service_name, "services", self.region + service_name, self.SERVICES, self.region ), ) @@ -123,25 +112,18 @@ def deploy_service( if not image_name: raise ValueError("image_name cannot be empty or None") - try: - service_body = { - "launch_stage": "alpha", - "template": { - "containers": [ - { - "image": image_name, - "ports": [ - {"containerPort": test_port, "name": "h2c"} - ], - } - ], - }, - } - - logger.info("Deploying Cloud Run service '%s'", service_name) - self.create_service(self.service, service_name, service_body) - return self.get_service_uri(service_name) - - except Exception as e: # noqa pylint: disable=broad-except - logger.exception("Error deploying Cloud Run service: %s", e) - raise + service_body = { + "launch_stage": "alpha", + "template": { + "containers": [ + { + "image": image_name, + "ports": [{"containerPort": test_port, "name": "h2c"}], + } + ], + }, + } + + logger.info("Deploying Cloud Run service '%s'", service_name) + self.create_service(service_name, service_body) + return self.get_service(service_name) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 5fdd0989..970fe3b8 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -595,7 +595,7 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): ) neg = self.get_serverless_network_endpoint_group(name, region) - logger.info("Created serverless neg %s ", neg) + logger.info("Created serverless NEG %s ", neg) return neg def delete_serverless_neg(self, name: str, region: str): @@ -735,7 +735,7 @@ def _execute( # pylint: disable=arguments-differ request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header request.add_response_callback(self._log_debug_header) operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.info("Operation %s", operation) + logger.debug("Operation %s", operation) return self._wait(operation["name"], timeout_sec, region) def _wait( diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 3fb3ec58..8f92f16c 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -22,12 +22,12 @@ from typing import Optional import framework +from framework.infrastructure import gcp from framework.infrastructure.gcp import cloud_run from framework.test_app.runners import base_runner logger = logging.getLogger(__name__) -_RunnerError = base_runner.RunnerError _HighlighterYaml = framework.helpers.highlighter.HighlighterYaml _helper_datetime = framework.helpers.datetime _datetime = dt.datetime @@ -51,8 +51,9 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): network: Optional[str] = None tag: str = "latest" region: str = "us-central1" + gcp_api_manager: gcp.api.GcpApiManager current_revision: Optional[str] = None - gcp_ui_url: Optional[str] = None + cloud_run_resource: Optional[cloud_run.GcpResource] = None run_history: collections.deque[RunHistory] @@ -66,7 +67,7 @@ def __init__( service_name: str, image_name: str, region: str, - gcp_ui_url: str, + gcp_api_manager: gcp.api.GcpApiManager, network: Optional[str] = None, ) -> None: super().__init__() @@ -77,7 +78,8 @@ def __init__( self.network = network self.region = region self.current_revision = None - self.gcp_ui_url = gcp_ui_url + self.gcp_ui_url = gcp_api_manager.gcp_ui_url + self.gcp_api_manager = gcp_api_manager # Persistent across many runs. self.run_history = collections.deque() @@ -93,7 +95,9 @@ def __init__( def _initalize_cloud_run_api_manager(self): """Initializes the CloudRunApiManager.""" self.cloud_run_api_manager = cloud_run.CloudRunApiManager( - project=self.project, region=self.region + project=self.project, + region=self.region, + api_manager=self.gcp_api_manager, ) def run(self, **kwargs): @@ -110,10 +114,11 @@ def run(self, **kwargs): self._reset_state() self.time_start_requested = _datetime.now() - self.current_revision = self.cloud_run_api_manager.deploy_service( + self.cloud_run_resource = self.cloud_run_api_manager.deploy_service( self.service_name, self.image_name, ) + self.current_revision = self.cloud_run_resource.url def _start_completed(self): self.time_start_completed = _datetime.now() @@ -194,10 +199,4 @@ def logs_explorer_run_history_links(self): def stop(self): """Deletes Cloud Run Service""" logger.info("Deleting Cloud Run service: %s", self.service_name) - try: - self.cloud_run_api_manager.delete_service(self.service_name) - logger.info("Deleted cloud run service: %s", self.service_name) - except Exception as e: # noqa pylint: disable=broad-except - logger.warning( - "Cloud Run service %s deletion failed: %s", self.service_name, e - ) + self.cloud_run_api_manager.delete_service(self.service_name) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 2a8ef4ac..ae5501c0 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -44,7 +44,7 @@ def __init__( image_name, network=network, region=region, - gcp_ui_url=gcp_api_manager.gcp_ui_url, + gcp_api_manager=gcp_api_manager, ) # Mutable state associated with each run. self._reset_state() @@ -76,7 +76,7 @@ def run(self, **kwargs) -> List[XdsTestServer]: return servers def get_service_url(self): - return self.cloud_run_api_manager.get_service_uri(self.service_name) + return self.cloud_run_api_manager.get_service(self.service_name).url @override def cleanup(self, *, force=False): From 3b3ae6acd60d2f710b050ec6c070ac42dab9aa4c Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 10 Apr 2025 02:13:43 +0530 Subject: [PATCH 34/49] common.py --- bin/lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index eb9d3b05..2bd6607b 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -65,7 +65,7 @@ def gcp_api_manager(): @functools.cache def cloud_run_api_manager(): return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, - region=xds_flags.CLOUD_RUN_REGION.value) + region=xds_flags.CLOUD_RUN_REGION.value,api_manager=gcp_api_manager()) def td_attrs(): return dict( From 0917ffae29ec7ddc1fb85ad71ad312dfb1997b6f Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 10 Apr 2025 02:23:55 +0530 Subject: [PATCH 35/49] pylint --- bin/lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index 2bd6607b..c064bbb9 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -65,7 +65,7 @@ def gcp_api_manager(): @functools.cache def cloud_run_api_manager(): return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, - region=xds_flags.CLOUD_RUN_REGION.value,api_manager=gcp_api_manager()) + region=xds_flags.CLOUD_RUN_REGION.value, api_manager=gcp_api_manager()) def td_attrs(): return dict( From aca0c1569028f3ad20e737037b064f05fb0573e8 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 10 Apr 2025 02:26:19 +0530 Subject: [PATCH 36/49] pylint --- bin/lib/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/common.py b/bin/lib/common.py index c064bbb9..2523e018 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -65,7 +65,8 @@ def gcp_api_manager(): @functools.cache def cloud_run_api_manager(): return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, - region=xds_flags.CLOUD_RUN_REGION.value, api_manager=gcp_api_manager()) + region=xds_flags.CLOUD_RUN_REGION.value, + api_manager=gcp_api_manager()) def td_attrs(): return dict( From 4a7feb96991a27f871c159305993a1f5270f3919 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 11 Apr 2025 12:40:16 +0530 Subject: [PATCH 37/49] comments --- bin/lib/common.py | 2 +- framework/infrastructure/gcp/api.py | 6 +- framework/infrastructure/gcp/cloud_run.py | 17 +- framework/infrastructure/gcp/compute.py | 3 +- .../traffic_director_cloud_run.py | 42 ++++- .../cloud_run/cloud_run_base_runner.py | 10 +- framework/test_cases/cloud_run_testcase.py | 167 ++++++++++++++++++ framework/xds_k8s_testcase.py | 160 +---------------- tests/cloud_run_csm_inbound_test.py | 7 +- 9 files changed, 230 insertions(+), 184 deletions(-) create mode 100644 framework/test_cases/cloud_run_testcase.py diff --git a/bin/lib/common.py b/bin/lib/common.py index 2523e018..1e87bdce 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -64,7 +64,7 @@ def gcp_api_manager(): @functools.cache def cloud_run_api_manager(): - return cloud_run.CloudRunApiManager(project=xds_flags.PROJECT.value, + return cloud_run.CloudRunV2(project=xds_flags.PROJECT.value, region=xds_flags.CLOUD_RUN_REGION.value, api_manager=gcp_api_manager()) diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index 211f0e2e..9a73126f 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -195,7 +195,7 @@ def cloudrun(self, version): api_name = "run" if version == "v2": return self._build_from_discovery_v2(api_name, version) - raise NotImplementedError(f"Cloud run {version} not supported") + raise NotImplementedError(f"Cloud Run {version} not supported") @functools.lru_cache(None) def networkservices(self, version): @@ -568,9 +568,7 @@ def _get_resource(self, collection: discovery.Resource, full_name): return resource def _delete_resource( - self, - collection: discovery.Resource, - full_name: str, + self, collection: discovery.Resource, full_name: str ) -> bool: logger.debug("Deleting %s", full_name) try: diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 8a6b7086..c9aab6a5 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -20,11 +20,8 @@ logger = logging.getLogger(__name__) -# Type aliases -GcpResource = gcp.compute.ComputeV1.GcpResource DEFAULT_TEST_PORT: Final[int] = 8080 -DISCOVERY_URI: Final[str] = "https://run.googleapis.com/$discovery/rest?" @dataclasses.dataclass(frozen=True) @@ -42,9 +39,11 @@ def from_response( ) -class CloudRunApiManager( - gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta -): +class CloudRunV2(gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta): + """Cloud Run API v2.""" + + SERVICES: Final[str] = "services" + region: str def __init__( @@ -54,13 +53,10 @@ def __init__( raise ValueError("Project ID cannot be empty or None.") if not region: raise ValueError("Region cannot be empty or None.") - # api_manager = gcp.api.GcpApiManager(v2_discovery_uri=DISCOVERY_URI) self.region = region super().__init__(api_manager.cloudrun(self.api_version), project) self._services_collection = self.api.projects().locations().services() - SERVICES = "services" - @property def api_name(self) -> str: """Returns the API name for Cloud Run.""" @@ -71,7 +67,7 @@ def api_version(self) -> str: """Returns the API version for Cloud Run.""" return "v2" - def create_service(self, service_name: str, body: dict) -> GcpResource: + def create_service(self, service_name: str, body: dict) -> None: return self._create_resource( collection=self._services_collection, location=self.region, @@ -92,7 +88,6 @@ def get_service(self, service_name: str) -> CloudRunService: ) def delete_service(self, service_name: str): - # pylint: disable=no-member self._delete_resource( self._services_collection, full_name=self.resource_full_name( diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 970fe3b8..24219df4 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -160,8 +160,7 @@ def create_backend_service_traffic_director( "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director "protocol": protocol.name, } - # If it is not for cloud run, add heath check since cloud run does not - # support health check. + if health_check: body["healthChecks"] = [health_check.url] diff --git a/framework/infrastructure/traffic_director_cloud_run.py b/framework/infrastructure/traffic_director_cloud_run.py index fca17384..b3cdfe0e 100644 --- a/framework/infrastructure/traffic_director_cloud_run.py +++ b/framework/infrastructure/traffic_director_cloud_run.py @@ -12,19 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Optional, Sequence +from typing import Final, Optional, Sequence from framework.infrastructure import gcp -from framework.infrastructure.traffic_director import ( - TrafficDirectorAppNetManager, -) +import framework.infrastructure.traffic_director as td_base +# Type aliases +_ComputeV1 = gcp.compute.ComputeV1 +GcpResource = _ComputeV1.GcpResource GrpcRoute = gcp.network_services.GrpcRoute Mesh = gcp.network_services.Mesh +logger = logging.getLogger(__name__) -class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): - MESH_NAME = "grpc-mesh" + +class TrafficDirectorCloudRunManager(td_base.TrafficDirectorAppNetManager): + MESH_NAME: Final[str] = "grpc-mesh" + NEG_NAME: Final[str] = "grpc-neg" def __init__( self, @@ -50,6 +54,7 @@ def __init__( # Managed resources self.grpc_route: Optional[GrpcRoute] = None self.mesh: Optional[Mesh] = None + self.neg: Optional[GcpResource] = None def backend_service_add_backends( self, @@ -85,3 +90,28 @@ def backend_service_add_backends( backends, is_cloud_run=True, ) + + def create_serverless_neg(self, region: str, service_name: str): + name = self.make_resource_name(self.NEG_NAME) + logger.info("Creating serverless NEG %s", name) + neg = self.compute.create_serverless_neg(name, region, service_name) + logger.info("Loading NEG %s", neg) + self.neg = self.compute.get_serverless_network_endpoint_group( + name, region + ) + return neg + + def delete_serverless_neg(self, region: str, force=False): + if force: + name = self.make_resource_name(self.NEG_NAME) + elif self.mesh: + name = self.neg.name + else: + return + logger.info("Deleting serverless NEG %s", name) + self.compute.delete_serverless_neg(name, region) + self.neg = None + + def cleanup(self, *, region, force=False): + self.delete_serverless_neg(region=region, force=force) + super().cleanup(force=force) diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 8f92f16c..4328ede0 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -53,7 +53,6 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): region: str = "us-central1" gcp_api_manager: gcp.api.GcpApiManager current_revision: Optional[str] = None - cloud_run_resource: Optional[cloud_run.GcpResource] = None run_history: collections.deque[RunHistory] @@ -93,8 +92,8 @@ def __init__( self._initalize_cloud_run_api_manager() def _initalize_cloud_run_api_manager(self): - """Initializes the CloudRunApiManager.""" - self.cloud_run_api_manager = cloud_run.CloudRunApiManager( + """Initializes the CloudRunV2.""" + self.cloud_run_api_manager = cloud_run.CloudRunV2( project=self.project, region=self.region, api_manager=self.gcp_api_manager, @@ -114,11 +113,10 @@ def run(self, **kwargs): self._reset_state() self.time_start_requested = _datetime.now() - self.cloud_run_resource = self.cloud_run_api_manager.deploy_service( + self.current_revision = self.cloud_run_api_manager.deploy_service( self.service_name, self.image_name, - ) - self.current_revision = self.cloud_run_resource.url + ).url def _start_completed(self): self.time_start_completed = _datetime.now() diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py new file mode 100644 index 00000000..e7b06a2c --- /dev/null +++ b/framework/test_cases/cloud_run_testcase.py @@ -0,0 +1,167 @@ +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime as dt +import logging +from typing import Any, List, Optional + +from framework import xds_flags +from framework import xds_k8s_testcase +from framework.helpers import retryers +from framework.infrastructure import k8s +from framework.infrastructure import traffic_director +from framework.infrastructure.gcp.compute import ComputeV1 +import framework.infrastructure.traffic_director_cloud_run as td_cloud_run +from framework.test_app import server_app +from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner +from framework.test_app.runners.k8s import k8s_xds_client_runner + +logger = logging.getLogger(__name__) + +# Type aliases +TrafficDirectorManager = traffic_director.TrafficDirectorManager +CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner +TrafficDirectorCloudRunManager = td_cloud_run.TrafficDirectorCloudRunManager +KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner +XdsTestServer = server_app.XdsTestServer + + +class CloudRunXdsKubernetesTestCase( + xds_k8s_testcase.SecurityXdsKubernetesTestCase +): + server_runner: CloudRunServerRunner + td: TrafficDirectorCloudRunManager + neg: Any + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) + cls.compute_v1 = ComputeV1( + cls.gcp_api_manager, cls.project, version=cls.compute_api_version + ) + cls.region = xds_flags.CLOUD_RUN_REGION.value + + def initTrafficDirectorManager(self) -> TrafficDirectorManager: + return TrafficDirectorCloudRunManager( + self.gcp_api_manager, + project=self.project, + resource_prefix=self.resource_prefix, + resource_suffix=self.resource_suffix, + network=self.network, + compute_api_version=self.compute_api_version, + enable_dualstack=self.enable_dualstack, + ) + + def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: + return KubernetesClientRunner( + k8s.KubernetesNamespace( + self.k8s_api_manager, self.client_namespace + ), + deployment_name=self.client_name, + image_name=self.client_image, + td_bootstrap_image=self.td_bootstrap_image, + gcp_project=self.project, + gcp_api_manager=self.gcp_api_manager, + gcp_service_account=self.gcp_service_account, + xds_server_uri=self.xds_server_uri, + network=self.network, + debug_use_port_forwarding=self.debug_use_port_forwarding, + enable_workload_identity=self.enable_workload_identity, + deployment_template="client.deployment.yaml", + stats_port=self.client_port, + reuse_namespace=self.server_namespace == self.client_namespace, + **kwargs, + ) + + def setupServerBackends( + self, + *, + wait_for_healthy_status=True, + server_runner=None, + max_rate_per_endpoint: Optional[int] = None, + ): + if server_runner is None: + server_runner = self.server_runner + service_url = server_runner.get_service_url() + self.td.backend_service_add_backends( + [service_url], max_rate_per_endpoint=max_rate_per_endpoint + ) + if wait_for_healthy_status: + self.td.wait_for_backends_healthy_status( + replica_count=server_runner.replica_count + ) + + def startTestServers(self, server_runner=None) -> List[XdsTestServer]: + if server_runner is None: + self.server_runner = CloudRunServerRunner( + project=self.project, + service_name=self.server_namespace, + image_name=self.server_image, + network=self.network, + region=self.region, + gcp_api_manager=self.gcp_api_manager, + ) + test_servers = self.server_runner.run() + for test_server in test_servers: + test_server.set_xds_address( + self.server_xds_host, self.server_xds_port + ) + return test_servers + + def assertEDSConfigExists(self, config): + """No-op for Cloud Run as EDS is not required.""" + _ = config + + def cleanup(self): + self.server_runner.cleanup(force=self.force_cleanup) + self.td.cleanup(region=self.region, force=self.force_cleanup) + self.client_runner.cleanup( + force=self.force_cleanup, force_namespace=self.force_cleanup + ) + + def tearDown(self): + logger.info("----- TestMethod %s teardown -----", self.test_name) + logger.debug("Getting pods restart times") + client_restarts: int = 0 + try: + client_restarts = self.client_runner.get_pod_restarts( + self.client_runner.deployment + ) + except (retryers.RetryError, k8s.NotFound) as e: + logger.exception(e) + + retryer = retryers.constant_retryer( + wait_fixed=dt.timedelta(seconds=10), + attempts=3, + log_level=logging.INFO, + ) + try: + retryer(self.cleanup) + except retryers.RetryError: + logger.exception("Got error during teardown") + finally: + logger.info("----- Test client/server logs -----") + self.client_runner.logs_explorer_run_history_links() + self.server_runner.logs_explorer_run_history_links() + + # Fail if any of the pods restarted. + self.assertEqual( + client_restarts, + 0, + msg=( + "Client container unexpectedly restarted" + f" {client_restarts} times during test. In most cases, this" + " is caused by the test client app crash." + ), + ) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index d26fdf86..2cc39cb2 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -40,14 +40,11 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director -from framework.infrastructure.gcp.compute import ComputeV1 -import framework.infrastructure.traffic_director_cloud_run as td_cloud_run from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing from framework.test_app import client_app from framework.test_app import server_app -from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner from framework.test_cases import base_testcase @@ -67,8 +64,6 @@ TrafficDirectorManager = traffic_director.TrafficDirectorManager TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager -CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner -TrafficDirectorCloudRunManager = td_cloud_run.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -567,14 +562,19 @@ def _assertRpcsEventuallyGoToGivenServers( f"Unexpected server {server_hostname} received RPCs", ) - @staticmethod - def assertEDSConfigExists(config: ClientConfig, seen: set, want: frozenset): - want = want.union(["endpoint_config"]) + def assertEDSConfigExists(self, config: ClientConfig): + seen = set() + want = frozenset(["endpoint_config"]) for generic_xds_config in config.generic_xds_configs: if re.search( r"\.ClusterLoadAssignment$", generic_xds_config.type_url ): seen.add("endpoint_config") + logger.debug( + "Received xDS config dump: %s", + json_format.MessageToJson(config, indent=2), + ) + self.assertSameElements(want, seen) def assertXdsConfigExists(self, test_client: XdsTestClient): config = test_client.csds.fetch_client_status(log_level=logging.INFO) @@ -599,8 +599,7 @@ def assertXdsConfigExists(self, test_client: XdsTestClient): elif re.search(r"\.Cluster$", generic_xds_config.type_url): seen.add("cluster_config") - self.assertEDSConfigExists(config, seen, want) - + self.assertEDSConfigExists(config) logger.debug( "Received xDS config dump: %s", json_format.MessageToJson(config, indent=2), @@ -1416,144 +1415,3 @@ def debug_cert(cert): return "missing" sha1 = hashlib.sha1(cert) return f"sha1={sha1.hexdigest()}, len={len(cert)}" - - -class CloudRunXdsKubernetesTestCase(SecurityXdsKubernetesTestCase): - server_runner: CloudRunServerRunner - td: TrafficDirectorCloudRunManager - neg: Any - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) - cls.compute_v1 = ComputeV1( - cls.gcp_api_manager, cls.project, version=cls.compute_api_version - ) - cls.region = xds_flags.CLOUD_RUN_REGION.value - - def initTrafficDirectorManager(self) -> TrafficDirectorManager: - return TrafficDirectorCloudRunManager( - self.gcp_api_manager, - project=self.project, - resource_prefix=self.resource_prefix, - resource_suffix=self.resource_suffix, - network=self.network, - compute_api_version=self.compute_api_version, - enable_dualstack=self.enable_dualstack, - ) - - def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: - return KubernetesClientRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.client_namespace - ), - deployment_name=self.client_name, - image_name=self.client_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - deployment_template="client.deployment.yaml", - stats_port=self.client_port, - reuse_namespace=self.server_namespace == self.client_namespace, - **kwargs, - ) - - def setupServerBackends( - self, - *, - wait_for_healthy_status=True, - server_runner=None, - max_rate_per_endpoint: Optional[int] = None, - ): - if server_runner is None: - server_runner = self.server_runner - service_url = server_runner.get_service_url() - self.td.backend_service_add_backends( - [service_url], max_rate_per_endpoint=max_rate_per_endpoint - ) - if wait_for_healthy_status: - self.td.wait_for_backends_healthy_status( - replica_count=server_runner.replica_count - ) - - def startTestServers(self, server_runner=None) -> List[XdsTestServer]: - if server_runner is None: - self.server_runner = CloudRunServerRunner( - project=self.project, - service_name=self.server_namespace, - image_name=self.server_image, - network=self.network, - region=self.region, - gcp_api_manager=self.gcp_api_manager, - ) - test_servers = self.server_runner.run() - for test_server in test_servers: - test_server.set_xds_address( - self.server_xds_host, self.server_xds_port - ) - return test_servers - - def backendServiceAddServerlessNegBackends(self): - name = self.td.make_resource_name("neg") - logger.info("Creating serverless NEG") - neg = self.compute_v1.create_serverless_neg( - name, - self.region, - self.server_namespace, - ) - self.neg = neg - return neg - - def assertEDSConfigExists(self, config, seen, want): - """No-op for Cloud Run as EDS is not required.""" - _ = (seen, config, want) - - def cleanup(self): - self.server_runner.cleanup(force=self.force_cleanup) - self.td.cleanup(force=self.force_cleanup) - self.compute_v1.delete_serverless_neg(self.neg["name"], self.region) - self.client_runner.cleanup( - force=self.force_cleanup, force_namespace=self.force_cleanup - ) - - def tearDown(self): - logger.info("----- TestMethod %s teardown -----", self.test_name) - logger.debug("Getting pods restart times") - client_restarts: int = 0 - try: - client_restarts = self.client_runner.get_pod_restarts( - self.client_runner.deployment - ) - except (retryers.RetryError, k8s.NotFound) as e: - logger.exception(e) - - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=10), - attempts=3, - log_level=logging.INFO, - ) - try: - retryer(self.cleanup) - except retryers.RetryError: - logger.exception("Got error during teardown") - finally: - logger.info("----- Test client/server logs -----") - self.client_runner.logs_explorer_run_history_links() - self.server_runner.logs_explorer_run_history_links() - - # Fail if any of the pods restarted. - self.assertEqual( - client_restarts, - 0, - msg=( - "Client container unexpectedly restarted" - f" {client_restarts} times during test. In most cases, this" - " is caused by the test client app crash." - ), - ) diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 231400ff..0ebfeaac 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -16,6 +16,7 @@ from absl.testing import absltest from framework import xds_k8s_testcase +from framework.test_cases import cloud_run_testcase logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ _XdsTestClient = xds_k8s_testcase.XdsTestClient -class CloudRunCsmInboundTest(xds_k8s_testcase.CloudRunXdsKubernetesTestCase): +class CloudRunCsmInboundTest(cloud_run_testcase.CloudRunXdsKubernetesTestCase): def test_gke_to_cloud_run(self): with self.subTest("0_create_mesh"): self.td.create_mesh() @@ -32,7 +33,7 @@ def test_gke_to_cloud_run(self): test_server: _XdsTestServer = self.startTestServers()[0] with self.subTest("2_create_serverless_neg"): - neg = self.backendServiceAddServerlessNegBackends() + self.td.create_serverless_neg(self.region, self.server_namespace) with self.subTest("3_create_backend_service"): self.td.create_backend_service( @@ -40,7 +41,7 @@ def test_gke_to_cloud_run(self): ) with self.subTest("4_add_server_backends_to_backend_service"): - self.td.backend_service_add_backends([neg], self.region) + self.td.backend_service_add_backends([self.td.neg], self.region) with self.subTest("5_create_grpc_route"): self.td.create_grpc_route( From 8bc4940ff1b9703359d4d4917c7948c64a7aeb98 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 11 Apr 2025 12:45:12 +0530 Subject: [PATCH 38/49] comments --- framework/infrastructure/traffic_director_cloud_run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/framework/infrastructure/traffic_director_cloud_run.py b/framework/infrastructure/traffic_director_cloud_run.py index b3cdfe0e..5c7b8a8c 100644 --- a/framework/infrastructure/traffic_director_cloud_run.py +++ b/framework/infrastructure/traffic_director_cloud_run.py @@ -112,6 +112,8 @@ def delete_serverless_neg(self, region: str, force=False): self.compute.delete_serverless_neg(name, region) self.neg = None - def cleanup(self, *, region, force=False): + def cleanup( + self, *, region, force=False + ): # pylint: disable=arguments-differ self.delete_serverless_neg(region=region, force=force) super().cleanup(force=force) From 224e2d8c65dda022a9a712b521bcc288be770878 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 15 Apr 2025 17:20:24 +0530 Subject: [PATCH 39/49] address comments --- framework/infrastructure/gcp/__init__.py | 1 + framework/infrastructure/gcp/compute.py | 8 ++--- .../cloud_run_mesh_manager.py} | 33 ++++++++---------- .../cloud_run/cloud_run_base_runner.py | 34 ++++++++----------- .../cloud_run/cloud_run_xds_server_runner.py | 28 ++++++++++----- .../runners/k8s/k8s_xds_client_runner.py | 3 +- framework/test_cases/cloud_run_testcase.py | 25 +++++++------- framework/xds_k8s_testcase.py | 4 --- kubernetes-manifests/client.deployment.yaml | 4 +-- tests/cloud_run_csm_inbound_test.py | 6 ++-- 10 files changed, 71 insertions(+), 75 deletions(-) rename framework/infrastructure/{traffic_director_cloud_run.py => mesh_resource_manager/cloud_run_mesh_manager.py} (76%) diff --git a/framework/infrastructure/gcp/__init__.py b/framework/infrastructure/gcp/__init__.py index 127ce04e..6d51da3f 100644 --- a/framework/infrastructure/gcp/__init__.py +++ b/framework/infrastructure/gcp/__init__.py @@ -16,3 +16,4 @@ from framework.infrastructure.gcp import iam from framework.infrastructure.gcp import network_security from framework.infrastructure.gcp import network_services +from framework.infrastructure.gcp import cloud_run diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 24219df4..aa14c7dd 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -566,7 +566,7 @@ def get_backend_service_backend_health(self, backend_service, backend): .execute() ) - def create_serverless_neg(self, name: str, region: str, service_name: str): + def create_neg_serverless(self, name: str, region: str, service_name: str): """Creates a serverless NEG. Args: @@ -593,7 +593,7 @@ def create_serverless_neg(self, name: str, region: str, service_name: str): region=region, ) - neg = self.get_serverless_network_endpoint_group(name, region) + neg = self.get_neg_serverless(name, region) logger.info("Created serverless NEG %s ", neg) return neg @@ -602,7 +602,7 @@ def delete_serverless_neg(self, name: str, region: str): Args: name: The name of the NEG to delete. - zone: The zone of the NEG. + region: The region of the NEG. """ logger.info("Deleting serverless NEG %s in %s", name, region) self._delete_resource( @@ -612,7 +612,7 @@ def delete_serverless_neg(self, name: str, region: str): region=region, ) - def get_serverless_network_endpoint_group(self, name, region): + def get_neg_serverless(self, name, region): neg = self._get_resource( self.api.regionNetworkEndpointGroups(), networkEndpointGroup=name, diff --git a/framework/infrastructure/traffic_director_cloud_run.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py similarity index 76% rename from framework/infrastructure/traffic_director_cloud_run.py rename to framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index 5c7b8a8c..532b1e26 100644 --- a/framework/infrastructure/traffic_director_cloud_run.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -class TrafficDirectorCloudRunManager(td_base.TrafficDirectorAppNetManager): +class CloudRunMeshManager(td_base.TrafficDirectorAppNetManager): MESH_NAME: Final[str] = "grpc-mesh" NEG_NAME: Final[str] = "grpc-neg" @@ -34,6 +34,7 @@ def __init__( self, gcp_api_manager: gcp.api.GcpApiManager, project: str, + region: str, *, resource_prefix: str, resource_suffix: Optional[str] = None, @@ -51,32 +52,28 @@ def __init__( enable_dualstack=enable_dualstack, ) + # Settings + self.region = region + # Managed resources self.grpc_route: Optional[GrpcRoute] = None self.mesh: Optional[Mesh] = None self.neg: Optional[GcpResource] = None - def backend_service_add_backends( + def backend_service_add_cloudrun_backends( self, backends: Sequence[str], balancing_mode: str = "CONNECTION", - max_rate_per_endpoint: Optional[int] = None, capacity_scaler: float = 1.0, - *, - circuit_breakers: Optional[dict[str, int]] = None, ): new_backends = [] for backend in backends: new_backend = { "group": backend, "balancingMode": balancing_mode, - "maxRatePerEndpoint": max_rate_per_endpoint, "capacityScaler": capacity_scaler, } - if circuit_breakers is not None: - new_backend["circuitBreakers"] = circuit_breakers - new_backends.append(new_backend) logging.info( @@ -91,17 +88,17 @@ def backend_service_add_backends( is_cloud_run=True, ) - def create_serverless_neg(self, region: str, service_name: str): + def create_neg_serverless(self, service_name: str): name = self.make_resource_name(self.NEG_NAME) logger.info("Creating serverless NEG %s", name) - neg = self.compute.create_serverless_neg(name, region, service_name) + neg = self.compute.create_neg_serverless(name, self.region, service_name) logger.info("Loading NEG %s", neg) - self.neg = self.compute.get_serverless_network_endpoint_group( - name, region + self.neg = self.compute.get_neg_serverless( + name, self.region ) return neg - def delete_serverless_neg(self, region: str, force=False): + def delete_serverless_neg(self, force=False): if force: name = self.make_resource_name(self.NEG_NAME) elif self.mesh: @@ -109,11 +106,9 @@ def delete_serverless_neg(self, region: str, force=False): else: return logger.info("Deleting serverless NEG %s", name) - self.compute.delete_serverless_neg(name, region) + self.compute.delete_serverless_neg(name, self.region) self.neg = None - def cleanup( - self, *, region, force=False - ): # pylint: disable=arguments-differ - self.delete_serverless_neg(region=region, force=force) + def cleanup(self, *, force=False): super().cleanup(force=force) + self.delete_serverless_neg(force=force) diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 4328ede0..548cc0dc 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -23,23 +23,23 @@ import framework from framework.infrastructure import gcp -from framework.infrastructure.gcp import cloud_run from framework.test_app.runners import base_runner +import framework.helpers.datetime +import framework.helpers.highlighter logger = logging.getLogger(__name__) _HighlighterYaml = framework.helpers.highlighter.HighlighterYaml _helper_datetime = framework.helpers.datetime -_datetime = dt.datetime _timedelta = dt.timedelta @dataclasses.dataclass(frozen=True) class RunHistory: revision_id: str - time_start_requested: _datetime - time_start_completed: Optional[_datetime] - time_stopped: _datetime + time_start_requested: dt.datetime + time_start_completed: Optional[dt.datetime] + time_stopped: dt.datetime class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): @@ -50,7 +50,7 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): image_name: str network: Optional[str] = None tag: str = "latest" - region: str = "us-central1" + region: str gcp_api_manager: gcp.api.GcpApiManager current_revision: Optional[str] = None @@ -93,7 +93,7 @@ def __init__( def _initalize_cloud_run_api_manager(self): """Initializes the CloudRunV2.""" - self.cloud_run_api_manager = cloud_run.CloudRunV2( + self.cloud_run_api_manager = gcp.cloud_run.CloudRunV2( project=self.project, region=self.region, api_manager=self.gcp_api_manager, @@ -112,17 +112,13 @@ def run(self, **kwargs): ) self._reset_state() - self.time_start_requested = _datetime.now() - self.current_revision = self.cloud_run_api_manager.deploy_service( - self.service_name, - self.image_name, - ).url + self.time_start_requested = dt.datetime.now() def _start_completed(self): - self.time_start_completed = _datetime.now() + self.time_start_completed = dt.datetime.now() def _stop(self): - self.time_stopped = _datetime.now() + self.time_stopped = dt.datetime.now() if self.time_start_requested: run_history = RunHistory( revision_id=self.current_revision, @@ -140,13 +136,13 @@ def _logs_explorer_link( gcp_project: str, gcp_ui_url: str, location: str, - start_time: Optional[_datetime] = None, - end_time: Optional[_datetime] = None, - cursor_time: Optional[_datetime] = None, + start_time: Optional[dt.datetime] = None, + end_time: Optional[dt.datetime] = None, + cursor_time: Optional[dt.datetime] = None, ): """Output the link to test server/client logs in GCP Logs Explorer.""" if not start_time: - start_time = _datetime.now() + start_time = dt.datetime.now() if not end_time: end_time = start_time + _timedelta(minutes=30) @@ -194,7 +190,7 @@ def logs_explorer_run_history_links(self): end_time=run.time_stopped, ) - def stop(self): + def cleanup(self,*,force: bool = False): """Deletes Cloud Run Service""" logger.info("Deleting Cloud Run service: %s", self.service_name) self.cloud_run_api_manager.delete_service(self.service_name) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index ae5501c0..ea70eaff 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -15,13 +15,12 @@ Run xDS Test Server on Cloud Run. """ import logging -from typing import List from typing_extensions import override from framework.infrastructure import gcp from framework.test_app.runners.cloud_run import cloud_run_base_runner -from framework.test_app.server_app import XdsTestServer +from framework.test_app import server_app logger = logging.getLogger(__name__) @@ -46,9 +45,14 @@ def __init__( region=region, gcp_api_manager=gcp_api_manager, ) + + self._initalize_cloud_run_api_manager() + # Mutable state associated with each run. self._reset_state() - + self.service : gcp.cloud_run.CloudRunV2 + self.current_revision: str + @override def _reset_state(self): super()._reset_state() @@ -57,7 +61,7 @@ def _reset_state(self): self.replica_count = 0 @override - def run(self, **kwargs) -> List[XdsTestServer]: + def run(self, **kwargs) -> list[server_app.XdsTestServer]: """Deploys and manages the xDS Test Server on Cloud Run.""" logger.info( "Starting cloud run server with service %s and image %s", @@ -66,21 +70,27 @@ def run(self, **kwargs) -> List[XdsTestServer]: ) super().run(**kwargs) + self.service = self.cloud_run_api_manager.deploy_service( + self.service_name, + self.image_name, + ) + self.current_revision = self.service.url servers = [ - XdsTestServer( + server_app.XdsTestServer( ip="0.0.0.0", rpc_port=0, hostname=self.current_revision ) ] - self.servers = servers # Add servers to the list self._start_completed() return servers - def get_service_url(self): - return self.cloud_run_api_manager.get_service(self.service_name).url + def get_service(self): + return self.cloud_run_api_manager.get_service(self.service_name) @override def cleanup(self, *, force=False): + # TODO(emchandwani) : Collect service logs in a file. try: - self.stop() + super().cleanup(force=force) + self.service=None finally: self._stop() diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index 535c7b7b..851ada85 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -137,7 +137,7 @@ def run( # pylint: disable=arguments-differ ( 'Deploying xDS test client "%s" to k8s namespace %s: ' "server_target=%s rpc=%s qps=%s metadata=%r secure_mode=%s " - "print_response=%s is_trusted_xds_server_experimental=%s" + "print_response=%s" ), self.deployment_name, self.k8s_namespace.name, @@ -147,7 +147,6 @@ def run( # pylint: disable=arguments-differ metadata, secure_mode, print_response, - is_trusted_xds_server_experimental, ) super().run() diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index e7b06a2c..c1825592 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -14,14 +14,15 @@ import datetime as dt import logging from typing import Any, List, Optional +from typing_extensions import override from framework import xds_flags from framework import xds_k8s_testcase from framework.helpers import retryers from framework.infrastructure import k8s from framework.infrastructure import traffic_director -from framework.infrastructure.gcp.compute import ComputeV1 -import framework.infrastructure.traffic_director_cloud_run as td_cloud_run +from framework.infrastructure.gcp import compute +import framework.infrastructure.mesh_resource_manager.cloud_run_mesh_manager as td_cloud_run from framework.test_app import server_app from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner @@ -31,7 +32,7 @@ # Type aliases TrafficDirectorManager = traffic_director.TrafficDirectorManager CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner -TrafficDirectorCloudRunManager = td_cloud_run.TrafficDirectorCloudRunManager +CloudRunMeshManager = td_cloud_run.CloudRunMeshManager KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner XdsTestServer = server_app.XdsTestServer @@ -40,22 +41,23 @@ class CloudRunXdsKubernetesTestCase( xds_k8s_testcase.SecurityXdsKubernetesTestCase ): server_runner: CloudRunServerRunner - td: TrafficDirectorCloudRunManager + td: CloudRunMeshManager neg: Any @classmethod def setUpClass(cls): super().setUpClass() cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) - cls.compute_v1 = ComputeV1( + cls.compute_v1 = compute.ComputeV1( cls.gcp_api_manager, cls.project, version=cls.compute_api_version ) cls.region = xds_flags.CLOUD_RUN_REGION.value def initTrafficDirectorManager(self) -> TrafficDirectorManager: - return TrafficDirectorCloudRunManager( + return CloudRunMeshManager( self.gcp_api_manager, project=self.project, + region=self.region, resource_prefix=self.resource_prefix, resource_suffix=self.resource_suffix, network=self.network, @@ -93,10 +95,8 @@ def setupServerBackends( ): if server_runner is None: server_runner = self.server_runner - service_url = server_runner.get_service_url() - self.td.backend_service_add_backends( - [service_url], max_rate_per_endpoint=max_rate_per_endpoint - ) + cloud_run_service = server_runner.get_service() + self.td.backend_service_add_cloudrun_backends([cloud_run_service.url]) if wait_for_healthy_status: self.td.wait_for_backends_healthy_status( replica_count=server_runner.replica_count @@ -118,14 +118,15 @@ def startTestServers(self, server_runner=None) -> List[XdsTestServer]: self.server_xds_host, self.server_xds_port ) return test_servers - + + @override def assertEDSConfigExists(self, config): """No-op for Cloud Run as EDS is not required.""" _ = config def cleanup(self): self.server_runner.cleanup(force=self.force_cleanup) - self.td.cleanup(region=self.region, force=self.force_cleanup) + self.td.cleanup(force=self.force_cleanup) self.client_runner.cleanup( force=self.force_cleanup, force_namespace=self.force_cleanup ) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index 2cc39cb2..206df19c 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -570,10 +570,6 @@ def assertEDSConfigExists(self, config: ClientConfig): r"\.ClusterLoadAssignment$", generic_xds_config.type_url ): seen.add("endpoint_config") - logger.debug( - "Received xDS config dump: %s", - json_format.MessageToJson(config, indent=2), - ) self.assertSameElements(want, seen) def assertXdsConfigExists(self, test_client: XdsTestClient): diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index b5cfa2b9..ec52c75d 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -57,8 +57,6 @@ spec: ports: - containerPort: ${stats_port} env: - - name: GRPC_TRACE - value: "xds_client" - name: GRPC_XDS_BOOTSTRAP value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE @@ -125,7 +123,7 @@ spec: - "--xds-server-uri=${xds_server_uri}" % endif % if config_mesh: - - "--config-mesh=${config_mesh}" + - "--config-mesh-experimental=${config_mesh}" % endif % if generate_mesh_id: - "--generate-mesh-id-experimental" diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 0ebfeaac..6159d7d8 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -32,8 +32,8 @@ def test_gke_to_cloud_run(self): with self.subTest("1_start_cloud_run_test_server"): test_server: _XdsTestServer = self.startTestServers()[0] - with self.subTest("2_create_serverless_neg"): - self.td.create_serverless_neg(self.region, self.server_namespace) + with self.subTest("2_create_neg_serverless"): + self.td.create_neg_serverless(self.server_namespace) with self.subTest("3_create_backend_service"): self.td.create_backend_service( @@ -41,7 +41,7 @@ def test_gke_to_cloud_run(self): ) with self.subTest("4_add_server_backends_to_backend_service"): - self.td.backend_service_add_backends([self.td.neg], self.region) + self.td.backend_service_add_cloudrun_backends([self.td.neg], self.region) with self.subTest("5_create_grpc_route"): self.td.create_grpc_route( From aec4ad8adc25c8c617a3bb622b74c5ffdff878d9 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 15 Apr 2025 21:07:35 +0530 Subject: [PATCH 40/49] pulint --- .github/psm-interop.code-workspace | 25 +++++++++++++++++++ framework/infrastructure/gcp/__init__.py | 2 +- .../cloud_run_mesh_manager.py | 8 +++--- .../cloud_run/cloud_run_base_runner.py | 6 ++--- .../cloud_run/cloud_run_xds_server_runner.py | 8 +++--- framework/test_cases/cloud_run_testcase.py | 6 ++--- tests/cloud_run_csm_inbound_test.py | 4 ++- 7 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 .github/psm-interop.code-workspace diff --git a/.github/psm-interop.code-workspace b/.github/psm-interop.code-workspace new file mode 100644 index 00000000..48208802 --- /dev/null +++ b/.github/psm-interop.code-workspace @@ -0,0 +1,25 @@ +{ + "folders": [ + { + "path": ".", + }, + ], + "settings": { + "files.exclude": { + "out/": true, + "**/__pycache__": true, + }, + // "explorer.excludeGitIgnore": true, + "python.analysis.diagnosticSeverityOverrides": { + // reports on protobuf + "reportInvalidTypeForm": "information", + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + }, + "black-formatter.importStrategy": "fromEnvironment", + }, + + } + \ No newline at end of file diff --git a/framework/infrastructure/gcp/__init__.py b/framework/infrastructure/gcp/__init__.py index 6d51da3f..24604026 100644 --- a/framework/infrastructure/gcp/__init__.py +++ b/framework/infrastructure/gcp/__init__.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from framework.infrastructure.gcp import api +from framework.infrastructure.gcp import cloud_run from framework.infrastructure.gcp import compute from framework.infrastructure.gcp import iam from framework.infrastructure.gcp import network_security from framework.infrastructure.gcp import network_services -from framework.infrastructure.gcp import cloud_run diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index 532b1e26..55ab4838 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -91,11 +91,11 @@ def backend_service_add_cloudrun_backends( def create_neg_serverless(self, service_name: str): name = self.make_resource_name(self.NEG_NAME) logger.info("Creating serverless NEG %s", name) - neg = self.compute.create_neg_serverless(name, self.region, service_name) - logger.info("Loading NEG %s", neg) - self.neg = self.compute.get_neg_serverless( - name, self.region + neg = self.compute.create_neg_serverless( + name, self.region, service_name ) + logger.info("Loading NEG %s", neg) + self.neg = self.compute.get_neg_serverless(name, self.region) return neg def delete_serverless_neg(self, force=False): diff --git a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py index 548cc0dc..eca7f9f0 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -22,10 +22,10 @@ from typing import Optional import framework -from framework.infrastructure import gcp -from framework.test_app.runners import base_runner import framework.helpers.datetime import framework.helpers.highlighter +from framework.infrastructure import gcp +from framework.test_app.runners import base_runner logger = logging.getLogger(__name__) @@ -190,7 +190,7 @@ def logs_explorer_run_history_links(self): end_time=run.time_stopped, ) - def cleanup(self,*,force: bool = False): + def cleanup(self, *, force: bool = False): """Deletes Cloud Run Service""" logger.info("Deleting Cloud Run service: %s", self.service_name) self.cloud_run_api_manager.delete_service(self.service_name) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index ea70eaff..078dd743 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -19,8 +19,8 @@ from typing_extensions import override from framework.infrastructure import gcp -from framework.test_app.runners.cloud_run import cloud_run_base_runner from framework.test_app import server_app +from framework.test_app.runners.cloud_run import cloud_run_base_runner logger = logging.getLogger(__name__) @@ -50,9 +50,9 @@ def __init__( # Mutable state associated with each run. self._reset_state() - self.service : gcp.cloud_run.CloudRunV2 + self.service: gcp.cloud_run.CloudRunV2 self.current_revision: str - + @override def _reset_state(self): super()._reset_state() @@ -91,6 +91,6 @@ def cleanup(self, *, force=False): # TODO(emchandwani) : Collect service logs in a file. try: super().cleanup(force=force) - self.service=None + self.service = None finally: self._stop() diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index c1825592..8c9e3f46 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -14,6 +14,7 @@ import datetime as dt import logging from typing import Any, List, Optional + from typing_extensions import override from framework import xds_flags @@ -91,8 +92,7 @@ def setupServerBackends( *, wait_for_healthy_status=True, server_runner=None, - max_rate_per_endpoint: Optional[int] = None, - ): + ): # pylint: disable=arguments-differ if server_runner is None: server_runner = self.server_runner cloud_run_service = server_runner.get_service() @@ -118,7 +118,7 @@ def startTestServers(self, server_runner=None) -> List[XdsTestServer]: self.server_xds_host, self.server_xds_port ) return test_servers - + @override def assertEDSConfigExists(self, config): """No-op for Cloud Run as EDS is not required.""" diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 6159d7d8..95fe5ed0 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -41,7 +41,9 @@ def test_gke_to_cloud_run(self): ) with self.subTest("4_add_server_backends_to_backend_service"): - self.td.backend_service_add_cloudrun_backends([self.td.neg], self.region) + self.td.backend_service_add_cloudrun_backends( + [self.td.neg], self.region + ) with self.subTest("5_create_grpc_route"): self.td.create_grpc_route( From 74676f884e16f63575838b1f4a9d817cdc1bb675 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 15 Apr 2025 21:32:01 +0530 Subject: [PATCH 41/49] pulint --- framework/test_cases/cloud_run_testcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index 8c9e3f46..44c14657 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -13,7 +13,7 @@ # limitations under the License. import datetime as dt import logging -from typing import Any, List, Optional +from typing import Any from typing_extensions import override @@ -102,7 +102,7 @@ def setupServerBackends( replica_count=server_runner.replica_count ) - def startTestServers(self, server_runner=None) -> List[XdsTestServer]: + def startTestServers(self, server_runner=None) -> list[XdsTestServer]: if server_runner is None: self.server_runner = CloudRunServerRunner( project=self.project, From 96609837068b2943a671b45d99eab982a0aa2d88 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 17 Apr 2025 16:16:54 +0530 Subject: [PATCH 42/49] comments --- .github/psm-interop.code-workspace | 25 ------------------- framework/infrastructure/gcp/cloud_run.py | 2 +- framework/infrastructure/gcp/compute.py | 2 +- .../cloud_run_mesh_manager.py | 17 ++++++------- .../cloud_run/cloud_run_xds_server_runner.py | 9 ++++--- framework/test_cases/cloud_run_testcase.py | 12 ++------- kubernetes-manifests/client.deployment.yaml | 2 +- tests/cloud_run_csm_inbound_test.py | 7 +++--- 8 files changed, 22 insertions(+), 54 deletions(-) delete mode 100644 .github/psm-interop.code-workspace diff --git a/.github/psm-interop.code-workspace b/.github/psm-interop.code-workspace deleted file mode 100644 index 48208802..00000000 --- a/.github/psm-interop.code-workspace +++ /dev/null @@ -1,25 +0,0 @@ -{ - "folders": [ - { - "path": ".", - }, - ], - "settings": { - "files.exclude": { - "out/": true, - "**/__pycache__": true, - }, - // "explorer.excludeGitIgnore": true, - "python.analysis.diagnosticSeverityOverrides": { - // reports on protobuf - "reportInvalidTypeForm": "information", - }, - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter", - "editor.formatOnSave": true, - }, - "black-formatter.importStrategy": "fromEnvironment", - }, - - } - \ No newline at end of file diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index c9aab6a5..69da6d98 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -35,7 +35,7 @@ def from_response( ) -> "CloudRunService": return cls( service_name=name, - url=response["urls"], + url=response["uri"], ) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index aa14c7dd..29d78eac 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -597,7 +597,7 @@ def create_neg_serverless(self, name: str, region: str, service_name: str): logger.info("Created serverless NEG %s ", neg) return neg - def delete_serverless_neg(self, name: str, region: str): + def delete_neg_serverless(self, name: str, region: str): """Deletes a serverless NEG. Args: diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index 55ab4838..2fd3003c 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -18,8 +18,7 @@ import framework.infrastructure.traffic_director as td_base # Type aliases -_ComputeV1 = gcp.compute.ComputeV1 -GcpResource = _ComputeV1.GcpResource +GcpResource = gcp.compute.ComputeV1.GcpResource GrpcRoute = gcp.network_services.GrpcRoute Mesh = gcp.network_services.Mesh @@ -62,12 +61,12 @@ def __init__( def backend_service_add_cloudrun_backends( self, - backends: Sequence[str], + *, balancing_mode: str = "CONNECTION", capacity_scaler: float = 1.0, ): new_backends = [] - for backend in backends: + for backend in self.backends: new_backend = { "group": backend, "balancingMode": balancing_mode, @@ -77,14 +76,14 @@ def backend_service_add_cloudrun_backends( new_backends.append(new_backend) logging.info( - "Adding backends to Backend Service %s: %r", + "Adding Cloud Run backends to Backend Service %s: %r", self.backend_service.name, new_backends, ) self.compute.backend_service_patch_backends( self.backend_service, - backends, + self.backends, is_cloud_run=True, ) @@ -98,7 +97,7 @@ def create_neg_serverless(self, service_name: str): self.neg = self.compute.get_neg_serverless(name, self.region) return neg - def delete_serverless_neg(self, force=False): + def delete_neg_serverless(self, force=False): if force: name = self.make_resource_name(self.NEG_NAME) elif self.mesh: @@ -106,9 +105,9 @@ def delete_serverless_neg(self, force=False): else: return logger.info("Deleting serverless NEG %s", name) - self.compute.delete_serverless_neg(name, self.region) + self.compute.delete_neg_serverless(name, self.region) self.neg = None def cleanup(self, *, force=False): super().cleanup(force=force) - self.delete_serverless_neg(force=force) + self.delete_neg_serverless(force=force) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 078dd743..2da29f27 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -15,7 +15,7 @@ Run xDS Test Server on Cloud Run. """ import logging - +from typing import Optional from typing_extensions import override from framework.infrastructure import gcp @@ -28,6 +28,9 @@ class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" + service: Optional[gcp.cloud_run.CloudRunV2] = None + current_revision: Optional[str] = None + def __init__( self, project: str, @@ -50,13 +53,12 @@ def __init__( # Mutable state associated with each run. self._reset_state() - self.service: gcp.cloud_run.CloudRunV2 - self.current_revision: str @override def _reset_state(self): super()._reset_state() self.service = None + self.current_revision = None self.pods_to_servers = {} self.replica_count = 0 @@ -92,5 +94,6 @@ def cleanup(self, *, force=False): try: super().cleanup(force=force) self.service = None + self.current_revision = None finally: self._stop() diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index 44c14657..8e4418b4 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -13,7 +13,7 @@ # limitations under the License. import datetime as dt import logging -from typing import Any +from typing import Optional from typing_extensions import override @@ -22,7 +22,6 @@ from framework.helpers import retryers from framework.infrastructure import k8s from framework.infrastructure import traffic_director -from framework.infrastructure.gcp import compute import framework.infrastructure.mesh_resource_manager.cloud_run_mesh_manager as td_cloud_run from framework.test_app import server_app from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner @@ -37,21 +36,15 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner XdsTestServer = server_app.XdsTestServer - class CloudRunXdsKubernetesTestCase( xds_k8s_testcase.SecurityXdsKubernetesTestCase ): server_runner: CloudRunServerRunner td: CloudRunMeshManager - neg: Any @classmethod def setUpClass(cls): super().setUpClass() - cls.compute = cls.gcp_api_manager.compute(cls.compute_api_version) - cls.compute_v1 = compute.ComputeV1( - cls.gcp_api_manager, cls.project, version=cls.compute_api_version - ) cls.region = xds_flags.CLOUD_RUN_REGION.value def initTrafficDirectorManager(self) -> TrafficDirectorManager: @@ -95,8 +88,7 @@ def setupServerBackends( ): # pylint: disable=arguments-differ if server_runner is None: server_runner = self.server_runner - cloud_run_service = server_runner.get_service() - self.td.backend_service_add_cloudrun_backends([cloud_run_service.url]) + self.td.backend_service_add_cloudrun_backends() if wait_for_healthy_status: self.td.wait_for_backends_healthy_status( replica_count=server_runner.replica_count diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index ec52c75d..a15f8005 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -129,7 +129,7 @@ spec: - "--generate-mesh-id-experimental" % endif % if is_trusted_xds_server_experimental: - - "--is-trusted-xds-server-experimental=true" + - "--is-trusted-xds-server-experimental" % endif resources: limits: diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 95fe5ed0..d23f0a87 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -16,6 +16,7 @@ from absl.testing import absltest from framework import xds_k8s_testcase +from framework.infrastructure import gcp from framework.test_cases import cloud_run_testcase logger = logging.getLogger(__name__) @@ -37,13 +38,11 @@ def test_gke_to_cloud_run(self): with self.subTest("3_create_backend_service"): self.td.create_backend_service( - protocol=self.compute_v1.BackendServiceProtocol.HTTP2, + protocol=gcp.compute.ComputeV1.BackendServiceProtocol.HTTP2, ) with self.subTest("4_add_server_backends_to_backend_service"): - self.td.backend_service_add_cloudrun_backends( - [self.td.neg], self.region - ) + self.td.backend_service_add_cloudrun_backends() with self.subTest("5_create_grpc_route"): self.td.create_grpc_route( From 1e7c4a564847cf3496cce2241a665a7a0a2b49b5 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 17 Apr 2025 09:23:03 -0700 Subject: [PATCH 43/49] Moving is_trusted_xds_server_experimental to client deployment args pair programming --- .../runners/k8s/k8s_xds_client_runner.py | 3 +-- tests/cloud_run_csm_inbound_test.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index 851ada85..2cf2b53f 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -32,6 +32,7 @@ class ClientDeploymentArgs: csm_workload_name: str = "" csm_canonical_service_name: str = "" enable_dualstack: bool = False + is_trusted_xds_server_experimental: bool = False def as_dict(self): return dataclasses.asdict(self) @@ -131,7 +132,6 @@ def run( # pylint: disable=arguments-differ log_to_stdout: bool = False, request_payload_size: int = 0, response_payload_size: int = 0, - is_trusted_xds_server_experimental: bool = False, ) -> client_app.XdsTestClient: logger.info( ( @@ -192,7 +192,6 @@ def run( # pylint: disable=arguments-differ config_mesh=config_mesh, generate_mesh_id=generate_mesh_id, print_response=print_response, - is_trusted_xds_server_experimental=is_trusted_xds_server_experimental, **self.deployment_args.as_dict(), ) diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 95fe5ed0..e7a17d8c 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -12,19 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +from typing import TypeAlias from absl.testing import absltest +from typing_extensions import override from framework import xds_k8s_testcase +from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_cases import cloud_run_testcase logger = logging.getLogger(__name__) -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient +_XdsTestServer: TypeAlias = xds_k8s_testcase.XdsTestServer +_XdsTestClient: TypeAlias = xds_k8s_testcase.XdsTestClient +KubernetesClientRunner: TypeAlias = k8s_xds_client_runner.KubernetesClientRunner +ClientDeploymentArgs: TypeAlias = k8s_xds_client_runner.ClientDeploymentArgs class CloudRunCsmInboundTest(cloud_run_testcase.CloudRunXdsKubernetesTestCase): + @override + def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: + return super().initKubernetesClientRunner( + deployment_args=ClientDeploymentArgs( + is_trusted_xds_server_experimental=True, + ) + ) + def test_gke_to_cloud_run(self): with self.subTest("0_create_mesh"): self.td.create_mesh() @@ -54,7 +67,6 @@ def test_gke_to_cloud_run(self): test_client: _XdsTestClient = self.startSecureTestClient( test_server, config_mesh=self.td.mesh.name, - is_trusted_xds_server_experimental=True, ) with self.subTest("7_test_client_xds_config_exists"): From 5128b988b3292a1c2ce8f6cbbf4bf032f5370ced Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 17 Apr 2025 09:27:04 -0700 Subject: [PATCH 44/49] Specify supported cloud run langs/versions in is_supported --- tests/cloud_run_csm_inbound_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index e7a17d8c..57d83b9f 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -18,11 +18,14 @@ from typing_extensions import override from framework import xds_k8s_testcase +from framework.helpers import skips from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_cases import cloud_run_testcase logger = logging.getLogger(__name__) +# Type aliases. +_Lang: TypeAlias = skips.Lang _XdsTestServer: TypeAlias = xds_k8s_testcase.XdsTestServer _XdsTestClient: TypeAlias = xds_k8s_testcase.XdsTestClient KubernetesClientRunner: TypeAlias = k8s_xds_client_runner.KubernetesClientRunner @@ -30,6 +33,13 @@ class CloudRunCsmInboundTest(cloud_run_testcase.CloudRunXdsKubernetesTestCase): + @staticmethod + @override + def is_supported(config: skips.TestConfig) -> bool: + if config.client_lang is _Lang.CPP: + return config.version_gte("v1.71.x") + return False + @override def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: return super().initKubernetesClientRunner( From 984dd501caef7291904af49a0af57384c82f5e93 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 18 Apr 2025 00:52:10 +0530 Subject: [PATCH 45/49] refactor deploy_service --- framework/infrastructure/gcp/cloud_run.py | 33 +-------------- .../cloud_run_mesh_manager.py | 40 ++++++++++++++++++- .../cloud_run/cloud_run_xds_server_runner.py | 8 +--- .../runners/k8s/k8s_xds_client_runner.py | 2 - framework/test_cases/cloud_run_testcase.py | 4 ++ 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 69da6d98..344d97a3 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -21,9 +21,6 @@ logger = logging.getLogger(__name__) -DEFAULT_TEST_PORT: Final[int] = 8080 - - @dataclasses.dataclass(frozen=True) class CloudRunService: service_name: str @@ -47,7 +44,7 @@ class CloudRunV2(gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta): region: str def __init__( - self, project: str, region: str, api_manager: gcp.api.GcpApiManager + self,api_manager: gcp.api.GcpApiManager, project: str, region: str ): if not project: raise ValueError("Project ID cannot be empty or None.") @@ -94,31 +91,3 @@ def delete_service(self, service_name: str): service_name, self.SERVICES, self.region ), ) - - def deploy_service( - self, - service_name: str, - image_name: str, - *, - test_port: int = DEFAULT_TEST_PORT, - ): - if not service_name: - raise ValueError("service_name cannot be empty or None") - if not image_name: - raise ValueError("image_name cannot be empty or None") - - service_body = { - "launch_stage": "alpha", - "template": { - "containers": [ - { - "image": image_name, - "ports": [{"containerPort": test_port, "name": "h2c"}], - } - ], - }, - } - - logger.info("Deploying Cloud Run service '%s'", service_name) - self.create_service(service_name, service_body) - return self.get_service(service_name) diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index 2fd3003c..b04cd5b3 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Final, Optional, Sequence +from typing import Final, Optional from framework.infrastructure import gcp import framework.infrastructure.traffic_director as td_base @@ -24,11 +24,17 @@ logger = logging.getLogger(__name__) +_CloudRunV2 = gcp.cloud_run.CloudRunV2 +CloudRunService = gcp.cloud_run.CloudRunService + +DEFAULT_TEST_PORT: Final[int] = 8080 class CloudRunMeshManager(td_base.TrafficDirectorAppNetManager): MESH_NAME: Final[str] = "grpc-mesh" NEG_NAME: Final[str] = "grpc-neg" + cloudrun: _CloudRunV2 + def __init__( self, gcp_api_manager: gcp.api.GcpApiManager, @@ -59,6 +65,9 @@ def __init__( self.mesh: Optional[Mesh] = None self.neg: Optional[GcpResource] = None + # API + self.cloudrun = _CloudRunV2(gcp_api_manager, project, region) + def backend_service_add_cloudrun_backends( self, *, @@ -108,6 +117,35 @@ def delete_neg_serverless(self, force=False): self.compute.delete_neg_serverless(name, self.region) self.neg = None + def deploy_service( + self, + service_name: str, + image_name: str, + *, + test_port: int = DEFAULT_TEST_PORT, + ): + if not service_name: + raise ValueError("service_name cannot be empty or None") + if not image_name: + raise ValueError("image_name cannot be empty or None") + + service_body = { + "launch_stage": "alpha", + "template": { + "containers": [ + { + "image": image_name, + "ports": [{"containerPort": test_port, "name": "h2c"}], + } + ], + }, + } + + logger.info("Deploying Cloud Run service '%s'", service_name) + self.cloudrun.create_service(service_name, service_body) + return self.cloudrun.get_service(service_name) + + def cleanup(self, *, force=False): super().cleanup(force=force) self.delete_neg_serverless(force=force) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 2da29f27..4573e318 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -51,9 +51,6 @@ def __init__( self._initalize_cloud_run_api_manager() - # Mutable state associated with each run. - self._reset_state() - @override def _reset_state(self): super()._reset_state() @@ -72,10 +69,7 @@ def run(self, **kwargs) -> list[server_app.XdsTestServer]: ) super().run(**kwargs) - self.service = self.cloud_run_api_manager.deploy_service( - self.service_name, - self.image_name, - ) + self.current_revision = self.service.url servers = [ server_app.XdsTestServer( diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index 851ada85..b1f4e142 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -131,7 +131,6 @@ def run( # pylint: disable=arguments-differ log_to_stdout: bool = False, request_payload_size: int = 0, response_payload_size: int = 0, - is_trusted_xds_server_experimental: bool = False, ) -> client_app.XdsTestClient: logger.info( ( @@ -192,7 +191,6 @@ def run( # pylint: disable=arguments-differ config_mesh=config_mesh, generate_mesh_id=generate_mesh_id, print_response=print_response, - is_trusted_xds_server_experimental=is_trusted_xds_server_experimental, **self.deployment_args.as_dict(), ) diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index 8e4418b4..b1dceb43 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -104,6 +104,10 @@ def startTestServers(self, server_runner=None) -> list[XdsTestServer]: region=self.region, gcp_api_manager=self.gcp_api_manager, ) + self.server_runner.service=self.td.deploy_service( + self.server_namespace, + self.server_image, + ) test_servers = self.server_runner.run() for test_server in test_servers: test_server.set_xds_address( From f45b38330da2f5ecd9c65150357ed024df3f5b56 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Fri, 18 Apr 2025 01:54:34 +0530 Subject: [PATCH 46/49] deploy_services --- framework/infrastructure/gcp/cloud_run.py | 2 +- .../cloud_run_mesh_manager.py | 38 ------------------ .../cloud_run/cloud_run_xds_server_runner.py | 40 ++++++++++++++++++- framework/test_cases/cloud_run_testcase.py | 6 +-- 4 files changed, 41 insertions(+), 45 deletions(-) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index 344d97a3..bf11a7e5 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -44,7 +44,7 @@ class CloudRunV2(gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta): region: str def __init__( - self,api_manager: gcp.api.GcpApiManager, project: str, region: str + self, api_manager: gcp.api.GcpApiManager, project: str, region: str ): if not project: raise ValueError("Project ID cannot be empty or None.") diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index b04cd5b3..deeda8e8 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -24,17 +24,11 @@ logger = logging.getLogger(__name__) -_CloudRunV2 = gcp.cloud_run.CloudRunV2 -CloudRunService = gcp.cloud_run.CloudRunService - -DEFAULT_TEST_PORT: Final[int] = 8080 class CloudRunMeshManager(td_base.TrafficDirectorAppNetManager): MESH_NAME: Final[str] = "grpc-mesh" NEG_NAME: Final[str] = "grpc-neg" - cloudrun: _CloudRunV2 - def __init__( self, gcp_api_manager: gcp.api.GcpApiManager, @@ -65,9 +59,6 @@ def __init__( self.mesh: Optional[Mesh] = None self.neg: Optional[GcpResource] = None - # API - self.cloudrun = _CloudRunV2(gcp_api_manager, project, region) - def backend_service_add_cloudrun_backends( self, *, @@ -117,35 +108,6 @@ def delete_neg_serverless(self, force=False): self.compute.delete_neg_serverless(name, self.region) self.neg = None - def deploy_service( - self, - service_name: str, - image_name: str, - *, - test_port: int = DEFAULT_TEST_PORT, - ): - if not service_name: - raise ValueError("service_name cannot be empty or None") - if not image_name: - raise ValueError("image_name cannot be empty or None") - - service_body = { - "launch_stage": "alpha", - "template": { - "containers": [ - { - "image": image_name, - "ports": [{"containerPort": test_port, "name": "h2c"}], - } - ], - }, - } - - logger.info("Deploying Cloud Run service '%s'", service_name) - self.cloudrun.create_service(service_name, service_body) - return self.cloudrun.get_service(service_name) - - def cleanup(self, *, force=False): super().cleanup(force=force) self.delete_neg_serverless(force=force) diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 4573e318..05c60888 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -15,7 +15,8 @@ Run xDS Test Server on Cloud Run. """ import logging -from typing import Optional +from typing import Final, Optional + from typing_extensions import override from framework.infrastructure import gcp @@ -24,6 +25,8 @@ logger = logging.getLogger(__name__) +DEFAULT_TEST_PORT: Final[int] = 8080 + class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" @@ -51,6 +54,8 @@ def __init__( self._initalize_cloud_run_api_manager() + self._reset_state() + @override def _reset_state(self): super()._reset_state() @@ -69,7 +74,10 @@ def run(self, **kwargs) -> list[server_app.XdsTestServer]: ) super().run(**kwargs) - + self.service = self.deploy_service( + service_name=self.service_name, + image_name=self.image_name, + ) self.current_revision = self.service.url servers = [ server_app.XdsTestServer( @@ -79,6 +87,34 @@ def run(self, **kwargs) -> list[server_app.XdsTestServer]: self._start_completed() return servers + def deploy_service( + self, + service_name: str, + image_name: str, + *, + test_port: int = DEFAULT_TEST_PORT, + ): + if not service_name: + raise ValueError("service_name cannot be empty or None") + if not image_name: + raise ValueError("image_name cannot be empty or None") + + service_body = { + "launch_stage": "alpha", + "template": { + "containers": [ + { + "image": image_name, + "ports": [{"containerPort": test_port, "name": "h2c"}], + } + ], + }, + } + + logger.info("Deploying Cloud Run service '%s'", service_name) + self.cloud_run_api_manager.create_service(service_name, service_body) + return self.cloud_run_api_manager.get_service(service_name) + def get_service(self): return self.cloud_run_api_manager.get_service(self.service_name) diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index b1dceb43..490bdb18 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -36,6 +36,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner XdsTestServer = server_app.XdsTestServer + class CloudRunXdsKubernetesTestCase( xds_k8s_testcase.SecurityXdsKubernetesTestCase ): @@ -104,10 +105,7 @@ def startTestServers(self, server_runner=None) -> list[XdsTestServer]: region=self.region, gcp_api_manager=self.gcp_api_manager, ) - self.server_runner.service=self.td.deploy_service( - self.server_namespace, - self.server_image, - ) + test_servers = self.server_runner.run() for test_server in test_servers: test_server.set_xds_address( From f9a9c209820e8dfca3760c62c4e438acee319a77 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 21 Apr 2025 16:04:48 +0530 Subject: [PATCH 47/49] fix test --- framework/infrastructure/gcp/cloud_run.py | 7 ++--- framework/infrastructure/gcp/compute.py | 2 +- .../cloud_run_mesh_manager.py | 29 +++++-------------- .../cloud_run/cloud_run_xds_server_runner.py | 9 ++---- tests/cloud_run_csm_inbound_test.py | 27 +++++++++++++++-- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/framework/infrastructure/gcp/cloud_run.py b/framework/infrastructure/gcp/cloud_run.py index bf11a7e5..380713fe 100644 --- a/framework/infrastructure/gcp/cloud_run.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -13,18 +13,15 @@ # limitations under the License. import abc import dataclasses -import logging from typing import Any, Final from framework.infrastructure import gcp -logger = logging.getLogger(__name__) - @dataclasses.dataclass(frozen=True) class CloudRunService: service_name: str - url: str + uri: str @classmethod def from_response( @@ -32,7 +29,7 @@ def from_response( ) -> "CloudRunService": return cls( service_name=name, - url=response["uri"], + uri=response["uri"], ) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 29d78eac..caffbe37 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -643,7 +643,7 @@ def _exists_resource( def _insert_resource( self, collection: discovery.Resource, - body: Dict[str, Any], + body: dict[str, Any], region: str = None, ) -> "GcpResource": logger.info( diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index deeda8e8..d71f5d31 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -19,8 +19,6 @@ # Type aliases GcpResource = gcp.compute.ComputeV1.GcpResource -GrpcRoute = gcp.network_services.GrpcRoute -Mesh = gcp.network_services.Mesh logger = logging.getLogger(__name__) @@ -55,30 +53,19 @@ def __init__( self.region = region # Managed resources - self.grpc_route: Optional[GrpcRoute] = None - self.mesh: Optional[Mesh] = None self.neg: Optional[GcpResource] = None - def backend_service_add_cloudrun_backends( - self, - *, - balancing_mode: str = "CONNECTION", - capacity_scaler: float = 1.0, - ): - new_backends = [] - for backend in self.backends: - new_backend = { - "group": backend, - "balancingMode": balancing_mode, - "capacityScaler": capacity_scaler, - } - - new_backends.append(new_backend) + def backend_service_add_cloudrun_backends(self): + if not self.neg: + raise ValueError( + f"Needed to create serverless NEG {self.NEG_NAME} first" + ) + self.backends = [self.neg] logging.info( "Adding Cloud Run backends to Backend Service %s: %r", self.backend_service.name, - new_backends, + self.backends, ) self.compute.backend_service_patch_backends( @@ -100,7 +87,7 @@ def create_neg_serverless(self, service_name: str): def delete_neg_serverless(self, force=False): if force: name = self.make_resource_name(self.NEG_NAME) - elif self.mesh: + elif self.neg: name = self.neg.name else: return diff --git a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index 05c60888..4c61f968 100644 --- a/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -31,7 +31,7 @@ class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" - service: Optional[gcp.cloud_run.CloudRunV2] = None + service: Optional[gcp.cloud_run.CloudRunService] = None current_revision: Optional[str] = None def __init__( @@ -78,7 +78,7 @@ def run(self, **kwargs) -> list[server_app.XdsTestServer]: service_name=self.service_name, image_name=self.image_name, ) - self.current_revision = self.service.url + self.current_revision = self.service.uri servers = [ server_app.XdsTestServer( ip="0.0.0.0", rpc_port=0, hostname=self.current_revision @@ -93,7 +93,7 @@ def deploy_service( image_name: str, *, test_port: int = DEFAULT_TEST_PORT, - ): + ) -> gcp.cloud_run.CloudRunService: if not service_name: raise ValueError("service_name cannot be empty or None") if not image_name: @@ -115,9 +115,6 @@ def deploy_service( self.cloud_run_api_manager.create_service(service_name, service_body) return self.cloud_run_api_manager.get_service(service_name) - def get_service(self): - return self.cloud_run_api_manager.get_service(self.service_name) - @override def cleanup(self, *, force=False): # TODO(emchandwani) : Collect service logs in a file. diff --git a/tests/cloud_run_csm_inbound_test.py b/tests/cloud_run_csm_inbound_test.py index 875cd2e0..29229481 100644 --- a/tests/cloud_run_csm_inbound_test.py +++ b/tests/cloud_run_csm_inbound_test.py @@ -12,20 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +from typing import TypeAlias from absl.testing import absltest +from typing_extensions import override from framework import xds_k8s_testcase +from framework.helpers import skips from framework.infrastructure import gcp +from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_cases import cloud_run_testcase logger = logging.getLogger(__name__) -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient +# Type aliases. +_Lang: TypeAlias = skips.Lang +_XdsTestServer: TypeAlias = xds_k8s_testcase.XdsTestServer +_XdsTestClient: TypeAlias = xds_k8s_testcase.XdsTestClient +KubernetesClientRunner: TypeAlias = k8s_xds_client_runner.KubernetesClientRunner +ClientDeploymentArgs: TypeAlias = k8s_xds_client_runner.ClientDeploymentArgs class CloudRunCsmInboundTest(cloud_run_testcase.CloudRunXdsKubernetesTestCase): + @staticmethod + @override + def is_supported(config: skips.TestConfig) -> bool: + if config.client_lang is _Lang.CPP: + return config.version_gte("v1.71.x") + return False + + @override + def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: + return super().initKubernetesClientRunner( + deployment_args=ClientDeploymentArgs( + is_trusted_xds_server_experimental=True, + ) + ) + def test_gke_to_cloud_run(self): with self.subTest("0_create_mesh"): self.td.create_mesh() From 544c3619c2a92dbf9d191ae29a0c1f43d58cc53b Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Mon, 21 Apr 2025 16:09:18 +0530 Subject: [PATCH 48/49] test --- framework/infrastructure/gcp/compute.py | 2 +- framework/test_cases/cloud_run_testcase.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index caffbe37..e97f905a 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -15,7 +15,7 @@ import datetime import enum import logging -from typing import Any, Dict, List, Optional, Set +from typing import Any, List, Optional, Set from googleapiclient import discovery import googleapiclient.errors diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index 490bdb18..7d0383ae 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -13,7 +13,6 @@ # limitations under the License. import datetime as dt import logging -from typing import Optional from typing_extensions import override From 97cbf2f404e9c18e5403a080c84c75a3b4bc9ef4 Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Tue, 22 Apr 2025 11:17:19 +0530 Subject: [PATCH 49/49] change test --- framework/infrastructure/gcp/compute.py | 47 +++++++++++-------- .../cloud_run_mesh_manager.py | 22 +++++++-- framework/test_cases/cloud_run_testcase.py | 14 ------ 3 files changed, 44 insertions(+), 39 deletions(-) diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index e97f905a..6f1945b0 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -200,30 +200,13 @@ def patch_backend_service(self, backend_service, body, **kwargs): **kwargs, ) - def backend_service_patch_backends( + def backend_service_patch_backends_with_body( self, - backend_service, - backends, - max_rate_per_endpoint: Optional[int] = None, + backend_service: GcpResource, + backend_list: list[dict[str, Any]], *, circuit_breakers: Optional[dict[str, int]] = None, - is_cloud_run: bool = False, ): - if is_cloud_run: - backend_list = [{"group": backend.url} for backend in backends] - else: - if max_rate_per_endpoint is None: - max_rate_per_endpoint = 5 - - backend_list = [ - { - "group": backend.url, - "balancingMode": "RATE", - "maxRatePerEndpoint": max_rate_per_endpoint, - } - for backend in backends - ] - request = {"backends": backend_list} if circuit_breakers: request["circuitBreakers"] = circuit_breakers @@ -234,6 +217,30 @@ def backend_service_patch_backends( backendService=backend_service.name, ) + def backend_service_patch_backends( + self, + backend_service: GcpResource, + backends: set[NegGcpResource], + max_rate_per_endpoint: Optional[int] = None, + *, + circuit_breakers: Optional[dict[str, int]] = None, + ): + if max_rate_per_endpoint is None: + max_rate_per_endpoint = 5 + + backend_list = [ + { + "group": backend.url, + "balancingMode": "RATE", + "maxRatePerEndpoint": max_rate_per_endpoint, + } + for backend in backends + ] + + self.backend_service_patch_backends_with_body( + backend_service, backend_list, circuit_breakers=circuit_breakers + ) + def backend_service_remove_all_backends(self, backend_service): self._patch_resource( collection=self.api.backendServices(), diff --git a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py index d71f5d31..41906820 100644 --- a/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py +++ b/framework/infrastructure/mesh_resource_manager/cloud_run_mesh_manager.py @@ -55,26 +55,38 @@ def __init__( # Managed resources self.neg: Optional[GcpResource] = None - def backend_service_add_cloudrun_backends(self): + def backend_service_add_cloudrun_backends( + self, + *, + capacity_scaler: float = 1.0, + ): if not self.neg: raise ValueError( f"Needed to create serverless NEG {self.NEG_NAME} first" ) self.backends = [self.neg] + new_backends = [] + for backend in self.backends: + new_backend = { + "group": backend.url, + "capacityScaler": capacity_scaler, + } + + new_backends.append(new_backend) + logging.info( "Adding Cloud Run backends to Backend Service %s: %r", self.backend_service.name, self.backends, ) - self.compute.backend_service_patch_backends( + self.compute.backend_service_patch_backends_with_body( self.backend_service, - self.backends, - is_cloud_run=True, + new_backends, ) - def create_neg_serverless(self, service_name: str): + def create_neg_serverless(self, service_name: str) -> GcpResource: name = self.make_resource_name(self.NEG_NAME) logger.info("Creating serverless NEG %s", name) neg = self.compute.create_neg_serverless( diff --git a/framework/test_cases/cloud_run_testcase.py b/framework/test_cases/cloud_run_testcase.py index 7d0383ae..ca7589e3 100644 --- a/framework/test_cases/cloud_run_testcase.py +++ b/framework/test_cases/cloud_run_testcase.py @@ -80,20 +80,6 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner: **kwargs, ) - def setupServerBackends( - self, - *, - wait_for_healthy_status=True, - server_runner=None, - ): # pylint: disable=arguments-differ - if server_runner is None: - server_runner = self.server_runner - self.td.backend_service_add_cloudrun_backends() - if wait_for_healthy_status: - self.td.wait_for_backends_healthy_status( - replica_count=server_runner.replica_count - ) - def startTestServers(self, server_runner=None) -> list[XdsTestServer]: if server_runner is None: self.server_runner = CloudRunServerRunner(