Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"mittens/internal/pkg/safe"
"mittens/internal/pkg/warmup"
"os"
"os/signal"
"syscall"
"time"
)

Expand Down Expand Up @@ -135,11 +137,18 @@ func Min(x, y int) int {
return x
}

// block blocks forever unless `-exit-after-warmup` is set to true
// block keeps the process alive until SIGINT/SIGTERM so the container stays up
// after warm-up and shuts down gracefully. Waiting on a signal (rather than a
// bare `select {}`) avoids tripping Go's deadlock detector once the process is
// idle (see #366). Returns immediately when `-exit-after-warmup` is set.
func block() {
if !opts.ExitAfterWarmup {
select {}
if opts.ExitAfterWarmup {
return
}
sig := make(chan os.Signal, 1)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you test this in a kubernetes environment and confirm the behaviour is as expected?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added TestKeepAliveExitsGracefullyOnSigterm (1ebb539) that reproduces the pod scenario at process level: it runs the built binary with the target never becoming ready (so warm-up sends nothing and the process goes idle β€” the exact condition that used to deadlock), asserts the process stays up, then sends SIGTERM (what kubelet delivers on pod termination) and asserts a clean exit (status 0). Before the fix that idle state aborts with all goroutines are asleep - deadlock!; after it, the container stays up and exits 0 on SIGTERM. Runs in ~3.5s locally and in CI.

We've also already validated the behaviour on live pods in our own Kubernetes environment. Happy to go into more detail over internal channels if that would help.

signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(sig)
<-sig
}

// postProcess includes steps that run once the warmup finishes.
Expand Down
63 changes: 63 additions & 0 deletions test/root_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package test

import (
"bytes"
"compress/gzip"
"context"
"fmt"
Expand All @@ -24,6 +25,9 @@ import (
"mittens/internal/pkg/probe"
"net/http"
"os"
"os/exec"
"path/filepath"
"syscall"
"testing"
"time"

Expand Down Expand Up @@ -273,6 +277,65 @@ func TestCompressWithGZip(t *testing.T) {
assert.Equal(t, requestBody, decompressedBody, "Assert that server-side decompressed body is equal to client request body")
}

// TestKeepAliveExitsGracefullyOnSigterm runs the built binary the way a
// Kubernetes sidecar does: warm-up finishes (here the target never becomes
// ready, so no requests are sent) and the process is left idle with
// -exit-after-warmup unset. A bare `select {}` keep-alive would abort with
// "all goroutines are asleep - deadlock!" once idle; the signal-based wait must
// instead stay up and then exit cleanly on SIGTERM β€” the signal kubelet
// delivers when it terminates a pod.
func TestKeepAliveExitsGracefullyOnSigterm(t *testing.T) {
bin := filepath.Join(t.TempDir(), "mittens")
build := exec.Command("go", "build", "-o", bin, ".")
build.Dir = ".."
build.Env = append(os.Environ(), "CGO_ENABLED=0")
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("failed to build binary: %v\n%s", err, out)
}

// Point readiness at a port nothing listens on so the target never becomes
// ready: warm-up gives up quickly and the process goes idle.
proc := exec.Command(bin,
"-http-requests=get:/hello-world",
"-target-readiness-port=9999",
"-target-readiness-http-path=/non-existent",
"-max-readiness-wait-seconds=1",
"-max-duration-seconds=2",
)
var out bytes.Buffer
proc.Stdout = &out
proc.Stderr = &out
if err := proc.Start(); err != nil {
t.Fatalf("failed to start binary: %v", err)
}

waitErr := make(chan error, 1)
go func() { waitErr <- proc.Wait() }()

// Once idle the process must stay alive β€” a bare select{} would already have
// deadlocked and exited by now.
select {
case err := <-waitErr:
t.Fatalf("process exited before receiving a signal (deadlock regression?): %v\n%s", err, out.String())
case <-time.After(3 * time.Second):
}

// Kubernetes terminates a pod with SIGTERM; the container must exit promptly
// and cleanly (status 0).
if err := proc.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatalf("failed to send SIGTERM: %v", err)
}
select {
case err := <-waitErr:
if err != nil {
t.Fatalf("process did not exit cleanly on SIGTERM: %v\n%s", err, out.String())
}
case <-time.After(5 * time.Second):
_ = proc.Process.Kill()
t.Fatalf("process did not exit within 5s of SIGTERM\n%s", out.String())
}
}

func setup() {
fmt.Println("Starting up http server")
mockHttpServer, mockHttpServerPort = fixture.StartHttpTargetTestServer([]fixture.PathResponseHandler{
Expand Down
Loading