From ecf1fc623c181abbbd1c6177bd97c18db49d3d75 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 13 Aug 2026 10:43:50 -0700 Subject: [PATCH 1/2] hack: fix DNS on IPv6-only kind clusters CoreDNS runs dnsPolicy: Default and inherits the node's Docker-generated /etc/resolv.conf, which always names an IPv4 resolver -- unreachable from a v6-only pod, so every external lookup SERVFAILs and anything that fetches at runtime never starts. Behind that sits a second failure: the registry is wired into containerd on the node, but atelet pulls actor images from its own pod netns, where kind-registry does not resolve at all. On IPv6-only clusters CoreDNS now answers for kind-registry and forwards everything else to an IPv6 upstream, overridable with IPV6_DNS_UPSTREAM where the default is unreachable. A pod resolves external names and reaches the registry, so the control plane installs and actors boot. IPv4 and dual-stack clusters are unchanged. --- hack/create-kind-cluster.sh | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c95..e943b9c64 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,68 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry (ipv6 only) +# +# CoreDNS runs dnsPolicy: Default, so it inherits the node's Docker-generated +# /etc/resolv.conf, which always names an IPv4 resolver. Pods here have no IPv4 +# address, so without this every external lookup SERVFAILs -- "connect: network +# is unreachable" in the CoreDNS log -- and anything that fetches at runtime, +# atelet pulling the gVisor tarball for one, never starts. Step 3 wired the +# registry into containerd on the *node*, which does not help a pod: atelet +# pulls actor images from its own netns, where "kind-registry" NXDOMAINs. Two +# Corefile clauses fix both. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name that is not the registry + # NXDOMAINs, trading one outage for a worse one. Both sides are left unquoted + # -- bash 3.2 would splice the quotes in literally. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " a silent no-op here is the whole failure mode; inspect it by hand" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Probe from a pod, never from the node: the node is dual-stack and resolves + # both names either way, so a node-side check proves nothing. The registry leg + # fetches rather than resolves, because the hosts entry above is AAAA-only and + # `nslookup kind-registry` fails on its A query even though every real client + # (getaddrinfo, and so containerd and atelet) is satisfied by the AAAA. + echo "Verifying DNS from a pod..." + if ! kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "nslookup storage.googleapis.com >/dev/null && + wget -q -T10 -O/dev/null http://${reg_name}:5000/v2/"; then + echo "error: a pod cannot resolve an external name and reach '${reg_name}'" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + exit 1 + fi +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat < Date: Tue, 18 Aug 2026 12:13:00 -0700 Subject: [PATCH 2/2] hack: add a local verification script for IPv6-only DNS Nothing exercised the IPv6-only CoreDNS rewrite except building a cluster and watching what happened, so a transform that quietly stopped matching kind's default Corefile would still have looked like a pass. The rewrite now lives in a sourceable function, and hack/verify-ipv6-dns.sh runs it against a pinned copy of that Corefile with neither Docker nor a cluster, then -- given a cluster -- asserts the shape that landed and resolves from a pod rather than the node, which is dual-stack and answers either way. Two of the offline checks are controls that must fail: an unrecognised Corefile, and re-patching an already-patched one. Reference branch stacked on the DNS fix, not a merge candidate. --- hack/create-kind-cluster.sh | 19 +--- hack/util/coredns-ipv6.sh | 46 ++++++++ hack/verify-ipv6-dns.sh | 208 ++++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 15 deletions(-) create mode 100644 hack/util/coredns-ipv6.sh create mode 100755 hack/verify-ipv6-dns.sh diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index e943b9c64..e12c278b5 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -17,6 +17,8 @@ set -o errexit -o nounset -o pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" +source "${ROOT}/hack/util/coredns-ipv6.sh" + KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" @@ -220,21 +222,8 @@ if [[ "${IP_FAMILY}" == "ipv6" ]]; then corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ -o jsonpath='{.data.Corefile}')" - # fallthrough is load-bearing: without it every name that is not the registry - # NXDOMAINs, trading one outage for a worse one. Both sides are left unquoted - # -- bash 3.2 would splice the quotes in literally. - search="forward . /etc/resolv.conf" - replace="hosts { - ${reg_v6} ${reg_name} - fallthrough - } - forward . ${IPV6_DNS_UPSTREAM}" - patched="${corefile/$search/$replace}" - if [[ "${patched}" == "${corefile}" ]]; then - echo "error: '${search}' not found in the CoreDNS Corefile" >&2 - echo " a silent no-op here is the whole failure mode; inspect it by hand" >&2 - exit 1 - fi + patched="$(coredns_ipv6_corefile \ + "${corefile}" "${reg_v6}" "${reg_name}" "${IPV6_DNS_UPSTREAM}")" # A YAML patch file avoids escaping the Corefile's newlines into JSON. { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ diff --git a/hack/util/coredns-ipv6.sh b/hack/util/coredns-ipv6.sh new file mode 100644 index 000000000..12efd59eb --- /dev/null +++ b/hack/util/coredns-ipv6.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The CoreDNS Corefile rewrite that IPv6-only kind clusters need, kept in a pure +# function so hack/verify-ipv6-dns.sh can exercise it without building a cluster. + +# coredns_ipv6_corefile +# +# Echoes with the resolv.conf forwarder replaced by a hosts block for +# the registry plus a forwarder to ; returns 1 if the search string is +# absent. The search is a prefix of kind's block form, so options the block +# carries (max_concurrent) re-attach to the new forwarder. +coredns_ipv6_corefile() { + local corefile="$1" reg_addr="$2" reg_name="$3" upstream="$4" + + local search="forward . /etc/resolv.conf" + # fallthrough is load-bearing: without it every name but the registry NXDOMAINs. + local replace="hosts { + ${reg_addr} ${reg_name} + fallthrough + } + forward . ${upstream}" + + # Both sides unquoted -- bash 3.2 would splice the quotes in literally. + local patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " a silent no-op here is the whole failure mode; inspect it by hand" >&2 + return 1 + fi + + printf '%s\n' "${patched}" +} diff --git a/hack/verify-ipv6-dns.sh b/hack/verify-ipv6-dns.sh new file mode 100755 index 000000000..c9e0edd3b --- /dev/null +++ b/hack/verify-ipv6-dns.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Checks the CoreDNS rewrite that create-kind-cluster.sh applies to IPv6-only +# clusters. The offline half needs neither Docker nor a cluster; the live half +# needs `IP_FAMILY=ipv6 hack/create-kind-cluster.sh`. + +set -o errexit -o nounset -o pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)"; cd "${ROOT}" +source "${ROOT}/hack/util/coredns-ipv6.sh" + +CTX="${KUBECTL_CONTEXT:-kind-${KIND_CLUSTER_NAME:-kind}}" +REG_NAME="${REG_NAME:-kind-registry}" +UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" +OFFLINE_ONLY=false + +if [[ $# -gt 0 ]]; then + case "$1" in + --offline) OFFLINE_ONLY=true ;; + -h|--help) + echo "Usage: $0 [--offline]" + echo "Verifies the IPv6-only CoreDNS rewrite. Offline checks always run;" + echo "live checks need a reachable cluster and are skipped without one." + echo + echo " --offline Skip the live checks entirely." + echo " KUBECTL_CONTEXT Context to check (default: kind-\${KIND_CLUSTER_NAME:-kind})." + exit 0 + ;; + *) echo "error: unknown argument '$1'; see --help" >&2; exit 1 ;; + esac +fi + +passed=0 +failed=0 +ok() { printf ' ok %s\n' "$1"; passed=$((passed + 1)); } +bad() { printf ' FAIL %s\n' "$1"; failed=$((failed + 1)); } + +# want +want() { + if [[ "$3" == *"$2"* ]]; then ok "$1"; else bad "$1 -- expected to find '$2'"; fi +} +# reject +reject() { + if [[ "$3" != *"$2"* ]]; then ok "$1"; else bad "$1 -- expected NOT to find '$2'"; fi +} + +# hosts_block -- the exact block the transform must produce +hosts_block() { + printf 'hosts {\n %s %s\n fallthrough\n }' "$1" "${REG_NAME}" +} + +# A snapshot of what kind installs, so the offline checks run with no cluster. +# The live checks below are what catch it drifting. +STOCK_COREFILE='.:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 { + disable success cluster.local + disable denial cluster.local + } + loop + reload + loadbalance +}' + +echo "== offline: the Corefile transform ==" +if patched="$(coredns_ipv6_corefile "${STOCK_COREFILE}" "fc00::3" "${REG_NAME}" "${UPSTREAM}")"; then + ok "transform applies to kind's default Corefile" + # Matched whole: "fallthrough" alone also appears in the kubernetes plugin. + want "hosts block is complete" "$(hosts_block fc00::3)" "${patched}" + want "forwarder points at the upstream" "forward . ${UPSTREAM}" "${patched}" + reject "resolv.conf forwarder is gone" "/etc/resolv.conf" "${patched}" + want "max_concurrent survives" "max_concurrent 1000" "${patched}" + want "the kubernetes plugin is untouched" "kubernetes cluster.local in-addr.arpa ip6.arpa" "${patched}" +else + bad "transform applies to kind's default Corefile" + patched="" +fi + +custom="$(coredns_ipv6_corefile "${STOCK_COREFILE}" "fc00::3" "${REG_NAME}" "2001:db8::1")" +want "IPV6_DNS_UPSTREAM is honoured" "forward . 2001:db8::1" "${custom}" + +echo "== offline: controls that must fail ==" +no_search="${STOCK_COREFILE/forward . \/etc\/resolv.conf/forward . 8.8.8.8}" +if coredns_ipv6_corefile "${no_search}" "fc00::3" "${REG_NAME}" "${UPSTREAM}" >/dev/null 2>&1; then + bad "control: an unrecognised Corefile is rejected" +else + ok "control: an unrecognised Corefile is rejected" +fi + +if [[ -n "${patched}" ]] && + coredns_ipv6_corefile "${patched}" "fc00::3" "${REG_NAME}" "${UPSTREAM}" >/dev/null 2>&1; then + bad "control: re-patching an already-patched Corefile is rejected" +else + ok "control: re-patching an already-patched Corefile is rejected" +fi + +if [[ "${OFFLINE_ONLY}" == "true" ]]; then + echo + echo "${passed} passed, ${failed} failed (live checks skipped)" + [[ "${failed}" -eq 0 ]] + exit +fi + +echo "== live: cluster '${CTX}' ==" +K=(kubectl --context "${CTX}") +if ! "${K[@]}" version --request-timeout=10s >/dev/null 2>&1; then + echo " skip no reachable cluster at '${CTX}'; run with --offline to silence this" + echo + echo "${passed} passed, ${failed} failed" + [[ "${failed}" -eq 0 ]] + exit +fi + +cidrs="$("${K[@]}" get nodes -o jsonpath='{.items[0].spec.podCIDRs}')" +live_corefile="$("${K[@]}" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}')" + +if [[ "${cidrs}" != *":"* ]]; then + echo " (IPv4 cluster -- asserting the rewrite did NOT run)" + want "resolv.conf forwarder is intact" "forward . /etc/resolv.conf" "${live_corefile}" + reject "no hosts block was added" "hosts {" "${live_corefile}" +elif [[ "${cidrs}" == *"."* ]]; then + echo " (dual-stack cluster -- asserting the rewrite did NOT run)" + want "resolv.conf forwarder is intact" "forward . /etc/resolv.conf" "${live_corefile}" + reject "no hosts block was added" "hosts {" "${live_corefile}" +else + echo " (IPv6-only cluster -- asserting the rewrite ran and works)" + recorded="$(printf '%s\n' "${live_corefile}" | + awk '/hosts \{/{h=1} h && /'"${REG_NAME}"'/{print $1; exit}')" + if [[ -z "${recorded}" ]]; then + bad "hosts block names ${REG_NAME}" + recorded="" + else + want "hosts block is complete" "$(hosts_block "${recorded}")" "${live_corefile}" + fi + + # The hosts entry is a point-in-time snapshot. Recreating the registry moves + # its address and every older cluster then resolves the name to a dead one -- + # connection refused rather than NXDOMAIN, which reads like a registry outage. + if ! command -v docker >/dev/null 2>&1; then + echo " skip docker is not on PATH; cannot check the recorded registry address" + elif ! actual="$(docker inspect "${REG_NAME}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null)" || + [[ -z "${actual}" ]]; then + bad "registry '${REG_NAME}' is gone or off the 'kind' network; Corefile still points at ${recorded}" + elif [[ "${recorded}" == "${actual}" ]]; then + ok "recorded registry address is current (${actual})" + else + bad "recorded registry address is stale: Corefile says ${recorded}, registry is at ${actual}" + fi + + echo " (probing from a pod, not the node -- the node is dual-stack)" + if "${K[@]}" run "verify-ipv6-dns-$$" --rm --attach --quiet --restart=Never \ + --image=busybox:1.36 --command -- \ + sh -c "nslookup storage.googleapis.com >/dev/null" >/dev/null 2>&1; then + ok "a pod resolves an external name" + else + bad "a pod resolves an external name -- IPV6_DNS_UPSTREAM is '${UPSTREAM}'" + fi + + # Fetched, not resolved: the hosts entry is AAAA-only, so nslookup fails on + # its A query even though getaddrinfo -- containerd, atelet -- is satisfied. + if "${K[@]}" run "verify-ipv6-reg-$$" --rm --attach --quiet --restart=Never \ + --image=busybox:1.36 --command -- \ + sh -c "wget -q -T10 -O/dev/null http://${REG_NAME}:5000/v2/" >/dev/null 2>&1; then + ok "a pod reaches the registry by name" + else + bad "a pod reaches the registry by name" + fi + + # The kubernetes plugin only sees this query if the hosts block declines it. + if "${K[@]}" run "verify-ipv6-svc-$$" --rm --attach --quiet --restart=Never \ + --image=busybox:1.36 --command -- \ + sh -c "nslookup kubernetes.default.svc.cluster.local >/dev/null" >/dev/null 2>&1; then + ok "a pod resolves an in-cluster name (fallthrough works)" + else + bad "a pod resolves an in-cluster name (fallthrough works)" + fi +fi + +echo +echo "${passed} passed, ${failed} failed" +[[ "${failed}" -eq 0 ]]