From f704ffdb86713b980c9f7c9893ad7eb73989ab21 Mon Sep 17 00:00:00 2001 From: trent Date: Wed, 15 Jul 2026 20:59:55 +0000 Subject: [PATCH 1/5] session: make healthcheck tolerance configurable via env vars Under severe memory pressure (e.g. a build maxing out RAM and driving iowait >90%), the buildx client can stall long enough to miss two consecutive healthchecks, causing buildkitd to drop the session that carries registry credentials. The subsequent push then fails with 'no active session ... context deadline exceeded'. Allow tuning the session healthcheck interval, timeout, and consecutive failure threshold via BUILDKIT_SESSION_HEALTHCHECK_INTERVAL, BUILDKIT_SESSION_HEALTHCHECK_TIMEOUT, and BUILDKIT_SESSION_HEALTHCHECK_MAX_FAILURES. Defaults are unchanged. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- session/grpc.go | 69 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/session/grpc.go b/session/grpc.go index d177fd54e79b..fbc5449b09ac 100644 --- a/session/grpc.go +++ b/session/grpc.go @@ -4,6 +4,9 @@ import ( "context" "math" "net" + "os" + "strconv" + "sync" "sync/atomic" "time" @@ -68,17 +71,63 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. return ctx, cc, nil } +// Session healthcheck tunables, overridable via environment variables so a +// daemon serving builds on memory-constrained machines can tolerate longer +// client stalls (e.g. page-cache thrashing) without dropping the session that +// carries registry credentials for the final push. +const ( + envHealthcheckInterval = "BUILDKIT_SESSION_HEALTHCHECK_INTERVAL" + envHealthcheckTimeout = "BUILDKIT_SESSION_HEALTHCHECK_TIMEOUT" + envHealthcheckMaxFailures = "BUILDKIT_SESSION_HEALTHCHECK_MAX_FAILURES" +) + +type healthcheckConfig struct { + interval time.Duration + timeout time.Duration + maxFailures int +} + +var getHealthcheckConfig = sync.OnceValue(func() healthcheckConfig { + cfg := healthcheckConfig{ + interval: 5 * time.Second, + timeout: 30 * time.Second, + maxFailures: 2, + } + if v := os.Getenv(envHealthcheckInterval); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + cfg.interval = d + } else { + bklog.L.Warnf("invalid %s value %q, using default %s", envHealthcheckInterval, v, cfg.interval) + } + } + if v := os.Getenv(envHealthcheckTimeout); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + cfg.timeout = d + } else { + bklog.L.Warnf("invalid %s value %q, using default %s", envHealthcheckTimeout, v, cfg.timeout) + } + } + if v := os.Getenv(envHealthcheckMaxFailures); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.maxFailures = n + } else { + bklog.L.Warnf("invalid %s value %q, using default %d", envHealthcheckMaxFailures, v, cfg.maxFailures) + } + } + return cfg +}) + func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(error)) { defer cancelConn(errors.WithStack(context.Canceled)) defer cc.Close() - ticker := time.NewTicker(5 * time.Second) + cfg := getHealthcheckConfig() + ticker := time.NewTicker(cfg.interval) defer ticker.Stop() healthClient := grpc_health_v1.NewHealthClient(cc) - failedBefore := false + consecutiveFailures := 0 consecutiveSuccessful := 0 - defaultHealthcheckDuration := 30 * time.Second lastHealthcheckDuration := time.Duration(0) for { @@ -91,7 +140,7 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err healthcheckStart := time.Now() - timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5)) + timeout := time.Duration(math.Max(float64(cfg.timeout), float64(lastHealthcheckDuration)*1.5)) ctx, cancel := context.WithCancelCause(ctx) ctx, _ = context.WithTimeoutCause(ctx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet @@ -110,19 +159,19 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err return default: } - if failedBefore { - bklog.G(ctx).Error("healthcheck failed fatally") + consecutiveFailures++ + consecutiveSuccessful = 0 + if consecutiveFailures >= cfg.maxFailures { + bklog.G(ctx).WithFields(logFields).Errorf("healthcheck failed fatally after %d consecutive failures", consecutiveFailures) return } - failedBefore = true - consecutiveSuccessful = 0 bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed") } else { consecutiveSuccessful++ - if consecutiveSuccessful >= 5 && failedBefore { - failedBefore = false + if consecutiveSuccessful >= 5 && consecutiveFailures > 0 { + consecutiveFailures = 0 bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure") } } From e97680697cc6d3039386d772d2c9a69ad7aa8b7a Mon Sep 17 00:00:00 2001 From: trent Date: Wed, 15 Jul 2026 21:04:07 +0000 Subject: [PATCH 2/5] session: use separate context for healthcheck rpc The per-check cancel left the shadowed loop context done, so the error branch returned on the first failed healthcheck and the failure tolerance never applied. Use a dedicated checkCtx for the rpc so only monitor shutdown short-circuits the loop. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- session/grpc.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/session/grpc.go b/session/grpc.go index fbc5449b09ac..b858ae7c7084 100644 --- a/session/grpc.go +++ b/session/grpc.go @@ -142,9 +142,9 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err timeout := time.Duration(math.Max(float64(cfg.timeout), float64(lastHealthcheckDuration)*1.5)) - ctx, cancel := context.WithCancelCause(ctx) - ctx, _ = context.WithTimeoutCause(ctx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet - _, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + checkCtx, cancel := context.WithCancelCause(ctx) + checkCtx, _ = context.WithTimeoutCause(checkCtx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet + _, err := healthClient.Check(checkCtx, &grpc_health_v1.HealthCheckRequest{}) cancel(errors.WithStack(context.Canceled)) lastHealthcheckDuration = time.Since(healthcheckStart) From d0b61a353323df1346125e8b214864f804075d8a Mon Sep 17 00:00:00 2001 From: trent Date: Wed, 15 Jul 2026 21:04:53 +0000 Subject: [PATCH 3/5] workflows: tolerate empty includes input in test prepare js-yaml 5.x throws on empty input; the unpinned npm install now pulls 5.x, breaking test/prepare for callers that pass no includes. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 65f5b6d757b5..7185e22635e4 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -88,7 +88,7 @@ jobs: core.setOutput('tags', JSON.stringify(matrix)); }); await core.group(`Set includes`, async () => { - const includes = yaml.load(core.getInput('includes')); + const includes = yaml.load(core.getInput('includes') || '[]'); core.info(JSON.stringify(includes, null, 2)); core.setOutput('includes', JSON.stringify(includes ?? [])); }); From 235fdbbed57ae24d58d4927ddf1044623ec374a0 Mon Sep 17 00:00:00 2001 From: trent Date: Wed, 15 Jul 2026 21:14:08 +0000 Subject: [PATCH 4/5] workflows: remove vagrant dependency workaround Port of moby/buildkit#6925; the stale VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT now causes a gem conflict when installing vagrant-libvirt on the freebsd test job. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-os.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-os.yml b/.github/workflows/test-os.yml index 86773cd56c0c..f6d4e37bb941 100644 --- a/.github/workflows/test-os.yml +++ b/.github/workflows/test-os.yml @@ -198,8 +198,6 @@ jobs: - build env: GOOS: freebsd - # https://github.com/hashicorp/vagrant/issues/13652 - VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT: 1 steps: - name: Checkout From c38c77ea0edbf4f4a9b00291e71e1636a08b28cd Mon Sep 17 00:00:00 2001 From: trent Date: Wed, 15 Jul 2026 21:20:58 +0000 Subject: [PATCH 5/5] hack: sync freebsd vagrantfile with upstream The generic/freebsd14 box tracks the EOL 14.0 release whose release_2 pkg repo has been removed, breaking the freebsd test job. Upstream switched to a pinned bento/freebsd-14 box. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- hack/Vagrantfile.freebsd | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/hack/Vagrantfile.freebsd b/hack/Vagrantfile.freebsd index a37f3d41285c..52760b32ba5c 100644 --- a/hack/Vagrantfile.freebsd +++ b/hack/Vagrantfile.freebsd @@ -2,19 +2,19 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "generic/freebsd14" + config.vm.box = "bento/freebsd-14" + config.vm.box_version = "202508.03.0" config.vm.boot_timeout = 900 config.vm.synced_folder ".", "/vagrant", type: "rsync" config.ssh.keep_alive = true config.vm.provision "init", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux freebsd-version -kru - kldload nullfs - # switching to "release_2" ensures compatibility with the current Vagrant box - sed -i '' 's/latest/release_2/' /usr/local/etc/pkg/repos/FreeBSD.conf + kldstat -m nullfs >/dev/null 2>&1 || kldload nullfs + pkg bootstrap pkg install -y git runj mkdir -p /vagrant/coverage /vagrant/.tmp/logs SHELL @@ -22,29 +22,29 @@ Vagrant.configure("2") do |config| config.vm.provision "install-buildkitd", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux cd /vagrant install -m 755 bin/buildkitd /usr/local/bin/buildkitd - type /usr/local/bin/buildkitd + test -x /usr/local/bin/buildkitd buildkitd --version SHELL end config.vm.provision "install-buildctl", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux cd /vagrant install -m 755 bin/buildctl /usr/local/bin/buildctl - type /usr/local/bin/buildctl + test -x /usr/local/bin/buildctl SHELL end config.vm.provision "run-containerd", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux cd /vagrant install -m 755 bin/containerd /bin/containerd containerd --version @@ -54,8 +54,8 @@ Vagrant.configure("2") do |config| config.vm.provision "run-buildkitd", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux mkdir -p /run/buildkit daemon -o /vagrant/.tmp/logs/buildkitd /usr/local/bin/buildkitd sleep 3 @@ -64,8 +64,8 @@ Vagrant.configure("2") do |config| config.vm.provision "test-smoke", type: "shell", run: "never" do |sh| sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail + #!/bin/sh + set -eux mkdir -p /vagrant/.tmp/freebsd-smoke cd /vagrant/.tmp/freebsd-smoke cat > Dockerfile <