diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c95..b99376e7c 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,55 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry +# +# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which +# no pod here can reach, so external lookups SERVFAIL. Step 3's registry +# wiring is node-side, so it misses atelet too: that pull runs in atelet's own +# netns, where "kind-registry" NXDOMAINs. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name but the registry + # NXDOMAINs, cluster.local included -- one outage traded for a worse one. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + # $search unquoted: bash 3.2 splices the quotes in literally. The block's + # first line inherits the matched line's indentation. + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Its own script so it can be re-run against a live cluster: the hosts entry + # above is a snapshot of an address the registry can move off (#1049). + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" REG_NAME="${reg_name}" \ + IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM}" "${ROOT}"/hack/verify-ipv6-dns.sh +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat <&2 + exit 1 + ;; + esac +fi + +# Best-effort: only used to make the registry failure message actionable. +reg_v6="$(docker inspect "${REG_NAME}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" +reg_at="${reg_v6:+ at [${reg_v6}]:5000}" + +echo "Verifying DNS from a pod..." +# Probe from a pod, not the node: the node is dual-stack and passes either way. +# The registry leg fetches rather than resolves -- the hosts entry is AAAA-only, +# which fails nslookup's A query but satisfies getaddrinfo. +# +# --attach gives one stream and only the last leg's exit status, so each leg +# reports a marker on stdout and no failure message may contain one; PROBE_RAN +# separates a failed leg from a pod that never ran. Retry the pod, not the +# query: one that asks before CoreDNS settles stays broken for ~30s, while a +# fresh pod 10s later resolves first try. +probe="" +probe_max=4 +for ((probe_attempt = 1; probe_attempt <= probe_max; probe_attempt++)); do + attempt_out="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$-${probe_attempt}" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + if out=\$(nslookup storage.googleapis.com 2>&1); then + echo RESOLVE_OK + else + echo \"resolve failed: \$(echo \"\$out\" | tail -2 | tr '\n' ' ')\" + fi + if out=\$(wget -T10 -O/dev/null http://${REG_NAME}:5000/v2/ 2>&1); then + echo REGISTRY_OK + else + echo \"registry fetch failed: \$(echo \"\$out\" | tail -1)\" + fi")" || true + # A pod that never started must not bury an earlier one's real failure. + if [[ "${attempt_out}" == *PROBE_RAN* ]]; then probe="${attempt_out}"; fi + # Only the resolve leg is a settling race; a down registry will not fix itself. + [[ "${probe}" == *RESOLVE_OK* ]] && break + if ((probe_attempt < probe_max)); then + echo " the cluster is not resolving yet; re-probing (attempt $((probe_attempt + 1)) of ${probe_max})..." + sleep 10 + fi +done +if [[ "${probe}" != *RESOLVE_OK* || "${probe}" != *REGISTRY_OK* ]]; then + if [[ "${probe}" != *PROBE_RAN* ]]; then + echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 + echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 + elif [[ "${probe}" != *RESOLVE_OK* ]]; then + echo "error: a pod cannot resolve an external name" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + else + echo "error: DNS works but a pod cannot reach '${REG_NAME}'${reg_at}" >&2 + echo " check the registry container is up and on the 'kind' network" >&2 + fi + if [[ -n "${probe}" ]]; then + echo " probe output was:" >&2 + printf '%s\n' "${probe}" | sed 's/^/ /' >&2 + fi + exit 1 +fi