diff --git a/cmd/root.go b/cmd/root.go index 9d9b0b9..769bc65 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,6 +22,8 @@ import ( "mittens/internal/pkg/safe" "mittens/internal/pkg/warmup" "os" + "os/signal" + "syscall" "time" ) @@ -135,11 +137,18 @@ func Min(x, y int) int { return x } -// block blocks forever unless `-exit-after-warmup` is set to true +// block keeps the process alive until SIGINT/SIGTERM so the container stays up +// after warm-up and shuts down gracefully. Waiting on a signal (rather than a +// bare `select {}`) avoids tripping Go's deadlock detector once the process is +// idle (see #366). Returns immediately when `-exit-after-warmup` is set. func block() { - if !opts.ExitAfterWarmup { - select {} + if opts.ExitAfterWarmup { + return } + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) + <-sig } // postProcess includes steps that run once the warmup finishes. diff --git a/test/root_test.go b/test/root_test.go index 98ce26e..6c6a68a 100644 --- a/test/root_test.go +++ b/test/root_test.go @@ -15,6 +15,7 @@ package test import ( + "bytes" "compress/gzip" "context" "fmt" @@ -24,6 +25,9 @@ import ( "mittens/internal/pkg/probe" "net/http" "os" + "os/exec" + "path/filepath" + "syscall" "testing" "time" @@ -273,6 +277,65 @@ func TestCompressWithGZip(t *testing.T) { assert.Equal(t, requestBody, decompressedBody, "Assert that server-side decompressed body is equal to client request body") } +// TestKeepAliveExitsGracefullyOnSigterm runs the built binary the way a +// Kubernetes sidecar does: warm-up finishes (here the target never becomes +// ready, so no requests are sent) and the process is left idle with +// -exit-after-warmup unset. A bare `select {}` keep-alive would abort with +// "all goroutines are asleep - deadlock!" once idle; the signal-based wait must +// instead stay up and then exit cleanly on SIGTERM — the signal kubelet +// delivers when it terminates a pod. +func TestKeepAliveExitsGracefullyOnSigterm(t *testing.T) { + bin := filepath.Join(t.TempDir(), "mittens") + build := exec.Command("go", "build", "-o", bin, ".") + build.Dir = ".." + build.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("failed to build binary: %v\n%s", err, out) + } + + // Point readiness at a port nothing listens on so the target never becomes + // ready: warm-up gives up quickly and the process goes idle. + proc := exec.Command(bin, + "-http-requests=get:/hello-world", + "-target-readiness-port=9999", + "-target-readiness-http-path=/non-existent", + "-max-readiness-wait-seconds=1", + "-max-duration-seconds=2", + ) + var out bytes.Buffer + proc.Stdout = &out + proc.Stderr = &out + if err := proc.Start(); err != nil { + t.Fatalf("failed to start binary: %v", err) + } + + waitErr := make(chan error, 1) + go func() { waitErr <- proc.Wait() }() + + // Once idle the process must stay alive — a bare select{} would already have + // deadlocked and exited by now. + select { + case err := <-waitErr: + t.Fatalf("process exited before receiving a signal (deadlock regression?): %v\n%s", err, out.String()) + case <-time.After(3 * time.Second): + } + + // Kubernetes terminates a pod with SIGTERM; the container must exit promptly + // and cleanly (status 0). + if err := proc.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatalf("failed to send SIGTERM: %v", err) + } + select { + case err := <-waitErr: + if err != nil { + t.Fatalf("process did not exit cleanly on SIGTERM: %v\n%s", err, out.String()) + } + case <-time.After(5 * time.Second): + _ = proc.Process.Kill() + t.Fatalf("process did not exit within 5s of SIGTERM\n%s", out.String()) + } +} + func setup() { fmt.Println("Starting up http server") mockHttpServer, mockHttpServerPort = fixture.StartHttpTargetTestServer([]fixture.PathResponseHandler{