feat: add Kubernetes deployment support with a Helm chart

backend/cmd/api/main.go

@@ -1,11 +1,24 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
+	"github.com/chaitin/panda-wiki/health"
+	"github.com/chaitin/panda-wiki/log"
 	"github.com/chaitin/panda-wiki/setup"
 )
 
+// shutdownTimeout bounds how long in-flight requests, including streaming
+// chat responses, are allowed to finish after a termination signal.
+const shutdownTimeout = 30 * time.Second
+
 // @title panda-wiki API
 // @version 1.0
 // @description panda-wiki API documentation
@@ -19,10 +32,60 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if err := setup.CheckInitCert(); err != nil {
-		panic(err)
+	// The self-signed certificate only exists for the bundled admin nginx, which
+	// reads it from a shared volume. Deployment targets where the certificate
+	// comes from elsewhere, or where the filesystem is read-only, turn it off.
+	if app.Config.Setup.InitCert {
+		if err := setup.CheckInitCert(); err != nil {
+			panic(err)
+		}
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if conn, ok := app.MQProducer.(health.Connection); ok {
+		app.Health.RegisterMQProbe("nats", conn)
+	}
+
+	if app.Health.Enabled() {
+		go func() {
+			if err := app.Health.Start(ctx); err != nil {
+				app.Logger.Error("health server stopped", log.Error(err))
+			}
+		}()
 	}
+
 	port := app.Config.HTTP.Port
 	app.Logger.Info(fmt.Sprintf("Starting server on port %d", port))
-	app.HTTPServer.Echo.Logger.Fatal(app.HTTPServer.Echo.Start(fmt.Sprintf(":%d", port)))
+
+	serveErr := make(chan error, 1)
+	go func() {
+		if err := app.HTTPServer.Echo.Start(fmt.Sprintf(":%d", port)); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			serveErr <- err
+		}
+	}()
+
+	var exitCode int
+	select {
+	case err := <-serveErr:
+		app.Logger.Error("http server stopped unexpectedly", log.Error(err))
+		// Exit non-zero so Kubernetes treats this as a crash and restarts the
+		// pod, rather than as a clean shutdown.
+		exitCode = 1
+	case <-ctx.Done():
+		app.Logger.Info("termination signal received, draining in-flight requests")
+	}
+
+	// Stop the periodic reporter first so it cannot delay the drain.
+	app.Telemetry.Stop()
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+	if err := app.HTTPServer.Echo.Shutdown(shutdownCtx); err != nil {
+		app.Logger.Error("graceful shutdown did not finish", log.Error(err))
+	}
+	if exitCode != 0 {
+		os.Exit(exitCode)
+	}
 }

backend/cmd/api/wire.go

@@ -8,7 +8,9 @@ import (
 	"github.com/chaitin/panda-wiki/config"
 	share "github.com/chaitin/panda-wiki/handler/share"
 	v1 "github.com/chaitin/panda-wiki/handler/v1"
+	"github.com/chaitin/panda-wiki/health"
 	"github.com/chaitin/panda-wiki/log"
+	"github.com/chaitin/panda-wiki/mq"
 	"github.com/chaitin/panda-wiki/server/http"
 	"github.com/chaitin/panda-wiki/telemetry"
 )
@@ -22,6 +24,7 @@ func createApp() (*App, error) {
 			telemetry.ProviderSet,
 
 			http.ProviderSet,
+			health.ProviderSet,
 			v1.ProviderSet,
 			share.ProviderSet,
 		),
@@ -36,4 +39,6 @@ type App struct {
 	Config        *config.Config
 	Logger        *log.Logger
 	Telemetry     *telemetry.Client
+	Health        *health.Server
+	MQProducer    mq.MQProducer
 }

backend/cmd/consumer/main.go

@@ -2,17 +2,49 @@ package main
 
 import (
 	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/chaitin/panda-wiki/health"
+	"github.com/chaitin/panda-wiki/log"
 )
 
 func main() {
 	app, err := createApp()
 	if err != nil {
 		panic(err)
 	}
-	if err := app.MQConsumer.StartConsumerHandlers(context.Background()); err != nil {
-		panic(err)
+
+	// Cancelling this context is what stops the consumer handlers, so it has to
+	// be wired to the termination signal rather than left as context.Background().
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if conn, ok := app.MQConsumer.(health.Connection); ok {
+		app.Health.RegisterMQProbe("nats", conn)
+	}
+
+	if app.Health.Enabled() {
+		go func() {
+			if err := app.Health.Start(ctx); err != nil {
+				app.Logger.Error("health server stopped", log.Error(err))
+			}
+		}()
+	}
+
+	if err := app.MQConsumer.StartConsumerHandlers(ctx); err != nil {
+		// This only returns on cancellation, so a non-nil error is unexpected.
+		app.Logger.Error("consumer handlers stopped unexpectedly", log.Error(err))
+		os.Exit(1)
 	}
+
+	// Order matters: stop scheduling new cron jobs before tearing the
+	// subscriptions down, so no job starts against a closed connection.
+	app.StatCronHandler.Stop()
 	if err := app.MQConsumer.Close(); err != nil {
-		panic(err)
+		// Reported, but not as a crash: this runs because a signal asked the
+		// process to stop, and JetStream redelivers anything left unacked.
+		app.Logger.Error("failed to close the message queue consumer", log.Error(err))
 	}
 }

backend/cmd/consumer/wire.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/chaitin/panda-wiki/config"
 	handler "github.com/chaitin/panda-wiki/handler/mq"
+	"github.com/chaitin/panda-wiki/health"
 	"github.com/chaitin/panda-wiki/log"
 	"github.com/chaitin/panda-wiki/mq"
 )
@@ -18,6 +19,7 @@ func createApp() (*App, error) {
 			config.ProviderSet,
 			log.ProviderSet,
 			handler.ProviderSet,
+			health.ProviderSet,
 		),
 	)
 	return &App{}, nil
@@ -28,4 +30,6 @@ type App struct {
 	Config          *config.Config
 	MQHandlers      *handler.MQHandlers
 	StatCronHandler *handler.CronHandler
+	Health          *health.Server
+	Logger          *log.Logger
 }