feat: enable cert rotation for allocation API service (fixes #422)

docs/quickstart/installing-thundernetes.md

@@ -80,7 +80,7 @@ There are two ways to generate a certificate.
 
 #### Using cert-manager to generate certificates
 
-Since cert-manager is already installed in the cluster, it can be used to generate a certificate for mTLS authentication. This is the recommended approach.
+Since cert-manager is already installed in the cluster, it can be used to generate a certificate for mTLS authentication. This is the recommended approach. When using cert-manager, certificate rotation is handled automatically — Thundernetes watches the mounted certificate files and reloads them when they change, so there is no need to restart the controller pod when certificates are renewed.
 
 First of all, you need to create the namespace `thundernetes-system`:
 
@@ -155,6 +155,8 @@ kubectl apply --server-side -f https://raw.githubusercontent.com/PlayFab/thunder
 
 **Note:** The two installation files (operator.yaml and operator_with_security.yaml) are identical except for the API_SERVICE_SECURITY environment variable that is passed into the controller container.
 
+> **Certificate rotation:** When TLS is enabled, Thundernetes automatically monitors the mounted TLS secret for changes and reloads the certificate without requiring a pod restart. This works with cert-manager automatic renewal or manual secret updates. The controller polls for certificate file changes every 30 seconds, so renewed certificates are picked up shortly after kubelet syncs the updated secret to the pod (typically within ~60 seconds total).
+
 ### Next steps
 
 Check the [.NET sample](sample-dotnet.md) document to learn how to test your installation by using our fake .NET game server sample.

installfiles/operator.yaml

@@ -17658,6 +17658,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: TLS_CERT_DIR
+          value: /tmp/alloc-api-serving-certs
         image: ghcr.io/playfab/thundernetes-operator:0.6.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -17690,6 +17692,9 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
         volumeMounts:
+        - mountPath: /tmp/alloc-api-serving-certs
+          name: alloc-api-cert
+          readOnly: true
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
@@ -17700,6 +17705,11 @@ spec:
       serviceAccountName: thundernetes-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: alloc-api-cert
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: tls-secret
       - name: cert
         secret:
           defaultMode: 420

installfiles/operator_with_monitoring.yaml

@@ -17658,6 +17658,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: TLS_CERT_DIR
+          value: /tmp/alloc-api-serving-certs
         image: ghcr.io/playfab/thundernetes-operator:0.6.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -17690,6 +17692,9 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
         volumeMounts:
+        - mountPath: /tmp/alloc-api-serving-certs
+          name: alloc-api-cert
+          readOnly: true
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
@@ -17700,6 +17705,11 @@ spec:
       serviceAccountName: thundernetes-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: alloc-api-cert
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: tls-secret
       - name: cert
         secret:
           defaultMode: 420

installfiles/operator_with_security.yaml

@@ -17658,6 +17658,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: TLS_CERT_DIR
+          value: /tmp/alloc-api-serving-certs
         image: ghcr.io/playfab/thundernetes-operator:0.6.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -17690,6 +17692,9 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
         volumeMounts:
+        - mountPath: /tmp/alloc-api-serving-certs
+          name: alloc-api-cert
+          readOnly: true
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
@@ -17700,6 +17705,11 @@ spec:
       serviceAccountName: thundernetes-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: alloc-api-cert
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: tls-secret
       - name: cert
         secret:
           defaultMode: 420

installfiles/operator_with_security_and_monitoring.yaml

@@ -17658,6 +17658,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: TLS_CERT_DIR
+          value: /tmp/alloc-api-serving-certs
         image: ghcr.io/playfab/thundernetes-operator:0.6.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -17690,6 +17692,9 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
         volumeMounts:
+        - mountPath: /tmp/alloc-api-serving-certs
+          name: alloc-api-cert
+          readOnly: true
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
@@ -17700,6 +17705,11 @@ spec:
       serviceAccountName: thundernetes-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: alloc-api-cert
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: tls-secret
       - name: cert
         secret:
           defaultMode: 420

pkg/operator/config/default/kustomization.yaml

@@ -38,6 +38,10 @@ patchesStrategicMerge:
 # crd/kustomization.yaml
 - manager_webhook_patch.yaml
 
+# [TLS] Mount the tls-secret for the allocation API service cert rotation
+# The secret is optional so the pod starts even without TLS configured
+- manager_alloc_tls_patch.yaml
+
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection

pkg/operator/config/default/manager_alloc_tls_patch.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        volumeMounts:
+        - mountPath: /tmp/alloc-api-serving-certs
+          name: alloc-api-cert
+          readOnly: true
+      volumes:
+      - name: alloc-api-cert
+        secret:
+          defaultMode: 420
+          secretName: tls-secret
+          optional: true

pkg/operator/config/manager/manager.yaml

@@ -54,6 +54,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: TLS_CERT_DIR
+              value: "/tmp/alloc-api-serving-certs"
           name: manager
           securityContext:
             allowPrivilegeEscalation: false

pkg/operator/controllers/allocation_api_server.go

@@ -3,7 +3,6 @@ package controllers
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -38,10 +37,9 @@ const (
 // so it can be added to our Manager
 type AllocationApiServer struct {
 	Client client.Client
-	// CrtBytes is the PEM-encoded certificate
-	CrtBytes []byte
-	// KeyBytes is the PEM-encoded key
-	KeyBytes []byte
+	// certWatcher watches and reloads TLS certificates from disk for dynamic cert rotation.
+	// If nil, the server runs without TLS.
+	certWatcher *CertificateWatcher
 	// gameServerQueue is a map of priority queues for game servers
 	gameServerQueue *GameServersQueue
 	// events is a buffered channel of GenericEvent
@@ -51,10 +49,9 @@ type AllocationApiServer struct {
 	listeningPort int32
 }
 
-func NewAllocationApiServer(crt, key []byte, cl client.Client, port int32) *AllocationApiServer {
+func NewAllocationApiServer(certWatcher *CertificateWatcher, cl client.Client, port int32) *AllocationApiServer {
 	return &AllocationApiServer{
-		CrtBytes:      crt,
-		KeyBytes:      key,
+		certWatcher:   certWatcher,
 		Client:        cl,
 		events:        make(chan event.GenericEvent, 100),
 		logger:        log.Log.WithName("allocation-api"),
@@ -103,25 +100,19 @@ func (s *AllocationApiServer) Start(ctx context.Context) error {
 		close(done)
 	}()
 
-	if s.CrtBytes != nil && s.KeyBytes != nil {
-		s.logger.Info("starting TLS enabled allocation API service")
-		// Generate a key pair from your pem-encoded cert and key ([]byte).
-		cert, err := tls.X509KeyPair(s.CrtBytes, s.KeyBytes)
-		if err != nil {
-			return nil
-		}
-		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM(s.CrtBytes)
-		// Construct a tls.config
+	if s.certWatcher != nil {
+		s.logger.Info("starting TLS enabled allocation API service with dynamic certificate rotation")
+		// Use dynamic TLS configuration via CertificateWatcher callbacks
+		// This enables automatic certificate rotation without server restart
 		tlsConfig := &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			ClientCAs:    caCertPool,
-			ClientAuth:   tls.RequireAndVerifyClientCert,
+			GetCertificate:     s.certWatcher.GetCertificate,
+			GetConfigForClient: s.certWatcher.GetConfigForClient,
+			ClientAuth:         tls.RequireAndVerifyClientCert,
 		}
 
 		// Build a server:
 		srv.TLSConfig = tlsConfig
-		// Finally: serve.
+		// Finally: serve. Empty strings because certs are provided via GetCertificate callback.
 		if err := srv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
 			return err
 		}

pkg/operator/controllers/allocation_api_server_test.go

@@ -27,7 +27,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 	It("empty body should return error", func() {
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", nil)
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, nil, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, nil, allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -38,7 +38,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 	It("GET method should return error", func() {
 		req := httptest.NewRequest(http.MethodGet, "/api/v1/allocate", nil)
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, nil, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, nil, allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -49,7 +49,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 	It("bad body should return error", func() {
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", bytes.NewBufferString("{\"foo\":\"bar\"}"))
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, nil, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, nil, allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -60,7 +60,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 	It("buildID should be a GUID", func() {
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", bytes.NewBufferString("{\"buildID\":\"NOT_A_GUID\",\"sessionID\":\"9bb3bbb2-5031-42fd-8982-5a3f76ef2c8a\"}"))
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, nil, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, nil, allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -71,7 +71,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 	It("should return NotFound on an empty list", func() {
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", bytes.NewBufferString("{\"sessionID\":\"9bb3bbb2-5031-42fd-8982-5a3f76ef2c8a\",\"buildID\":\"9bb3bbb2-5031-42fd-8982-5a3f76ef2c8a\"}"))
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, testNewSimpleK8sClient(), allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, testNewSimpleK8sClient(), allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -85,7 +85,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 		Expect(err).ToNot(HaveOccurred())
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", bytes.NewBufferString(fmt.Sprintf("{\"sessionID\":\"%s\",\"buildID\":\"%s\"}", sessionID1, buildID1)))
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, client, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, client, allocationApiSvcPort)
 		h.handleAllocationRequest(w, req)
 		res := w.Result()
 		defer res.Body.Close()
@@ -105,7 +105,7 @@ var _ = Describe("allocation API service input validation tests", func() {
 		Expect(err).ToNot(HaveOccurred())
 		req := httptest.NewRequest(http.MethodPost, "/api/v1/allocate", bytes.NewBufferString(fmt.Sprintf("{\"sessionID\":\"%s\",\"buildID\":\"%s\"}", sessionID1, buildID1)))
 		w := httptest.NewRecorder()
-		h := NewAllocationApiServer(nil, nil, client, allocationApiSvcPort)
+		h := NewAllocationApiServer(nil, client, allocationApiSvcPort)
 		h.gameServerQueue = NewGameServersQueue()
 		h.gameServerQueue.PushToQueue(&GameServerForQueue{
 			Name:            gsName,