fix: fix node-local-dns on cilium 1.16.5+

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3635,6 +3635,12 @@ spec:
                           config for node local dns by the user - it will include
                           the original CoreFile made by kOps.
                         type: string
+                      ciliumBPFCompatibility:
+                        description: CiliumBPFCompatibility allows user to enable
+                          cilium bpf host routing compatibility mode, which is required
+                          for cilium 1.16.5+ and above, when the user is using cilium
+                          as an externally managed daemonset.
+                        type: boolean
                       cpuRequest:
                         anyOf:
                         - type: integer

pkg/apis/kops/cluster.go

@@ -625,6 +625,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/v1alpha2/cluster.go

@@ -604,6 +604,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/v1alpha3/cluster.go

@@ -590,6 +590,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/validation/validation.go

@@ -1839,14 +1839,21 @@ func validateNodeLocalDNS(spec *kops.ClusterSpec, fldpath *field.Path) field.Err
 		}
 	}
 
-	if (spec.KubeProxy != nil && spec.KubeProxy.ProxyMode == "ipvs") || spec.Networking.Cilium != nil {
-		if spec.Kubelet != nil && spec.Kubelet.ClusterDNS != "" && spec.Kubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP {
-			allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP"))
+	// Helper function to validate ClusterDNS settings
+	validateClusterDNS := func(kubelet *kops.KubeletConfigSpec, expectedIP string, fieldName string) {
+		if kubelet != nil && kubelet.ClusterDNS != "" && kubelet.ClusterDNS != expectedIP {
+			allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"),
+				fmt.Sprintf("%s ClusterDNS must be set to %s but it was set to %s", fieldName, expectedIP, kubelet.ClusterDNS)))
 		}
+	}
 
-		if spec.ControlPlaneKubelet != nil && spec.ControlPlaneKubelet.ClusterDNS != "" && spec.ControlPlaneKubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP {
-			allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "ControlPlaneKubelet ClusterDNS must be set to the default IP address for LocalIP"))
-		}
+	// When cilium is used, Node Local DNS pods use the service with a cilium LRP
+	if spec.Networking.Cilium != nil || fi.ValueOf(spec.KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) {
+		validateClusterDNS(spec.Kubelet, spec.KubeDNS.ServerIP, "Kubelet")
+		validateClusterDNS(spec.ControlPlaneKubelet, spec.KubeDNS.ServerIP, "ControlPlaneKubelet")
+	} else if spec.KubeProxy != nil && spec.KubeProxy.ProxyMode == "ipvs" {
+		validateClusterDNS(spec.Kubelet, spec.KubeDNS.NodeLocalDNS.LocalIP, "Kubelet")
+		validateClusterDNS(spec.ControlPlaneKubelet, spec.KubeDNS.NodeLocalDNS.LocalIP, "ControlPlaneKubelet")
 	}
 
 	return allErrs

pkg/apis/kops/validation/validation_test.go

@@ -1456,12 +1456,13 @@ func Test_Validate_NodeLocalDNS(t *testing.T) {
 		{
 			Input: kops.ClusterSpec{
 				Kubelet: &kops.KubeletConfigSpec{
-					ClusterDNS: "169.254.20.10",
+					ClusterDNS: "100.64.0.10",
 				},
 				KubeProxy: &kops.KubeProxyConfig{
 					ProxyMode: "iptables",
 				},
 				KubeDNS: &kops.KubeDNSConfig{
+					ServerIP: "100.64.0.10",
 					Provider: "CoreDNS",
 					NodeLocalDNS: &kops.NodeLocalDNSConfig{
 						Enabled: fi.PtrTo(true),
@@ -1474,6 +1475,26 @@ func Test_Validate_NodeLocalDNS(t *testing.T) {
 			},
 			ExpectedErrors: []string{},
 		},
+		{
+			Input: kops.ClusterSpec{
+				Kubelet: &kops.KubeletConfigSpec{
+					ClusterDNS: "100.64.0.10",
+				},
+				KubeProxy: &kops.KubeProxyConfig{
+					ProxyMode: "iptables",
+				},
+				KubeDNS: &kops.KubeDNSConfig{
+					ServerIP: "100.64.0.10",
+					Provider: "CoreDNS",
+					NodeLocalDNS: &kops.NodeLocalDNSConfig{
+						Enabled:                fi.PtrTo(true),
+						LocalIP:                "169.254.20.10",
+						CiliumBPFCompatibility: fi.PtrTo(true),
+					},
+				},
+			},
+			ExpectedErrors: []string{},
+		},
 	}
 
 	for _, g := range grid {

pkg/model/components/kubelet.go

@@ -87,7 +87,7 @@ func (b *KubeletOptionsBuilder) configureKubelet(cluster *kops.Cluster, kubelet
 	}
 
 	if kubelet.ClusterDNS == "" {
-		if cluster.Spec.KubeDNS != nil && cluster.Spec.KubeDNS.NodeLocalDNS != nil && fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.Enabled) {
+		if cluster.Spec.KubeDNS != nil && cluster.Spec.KubeDNS.NodeLocalDNS != nil && fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.Enabled) && !fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) && cluster.Spec.Networking.Cilium == nil {
 			kubelet.ClusterDNS = cluster.Spec.KubeDNS.NodeLocalDNS.LocalIP
 		} else {
 			ip, err := WellKnownServiceIP(&cluster.Spec.Networking, 10)

upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template

@@ -52,20 +52,20 @@ data:
         }
         reload
         loop
-        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}
+        bind 0.0.0.0
         forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
         prometheus :9253
-        health {{ joinHostPort KubeDNS.NodeLocalDNS.LocalIP NodeLocalDNSHealthCheck }}
+        health :{{ NodeLocalDNSHealthCheck }}
     }
     {{- if WithDefaultBool KubeDNS.NodeLocalDNS.ForwardToKubeDNS false }}
     .:53 {
         errors
         cache 30
         reload
         loop
-        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}
+        bind 0.0.0.0
         forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
@@ -77,7 +77,7 @@ data:
         cache 30
         reload
         loop
-        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}
+        bind 0.0.0.0
         forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
@@ -88,7 +88,7 @@ data:
         cache 30
         reload
         loop
-        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}
+        bind 0.0.0.0
         forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
@@ -99,7 +99,7 @@ data:
         cache 30
         reload
         loop
-        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}
+        bind 0.0.0.0
         forward . __PILLAR__UPSTREAM__SERVERS__
         prometheus :9253
         {{- if IsIPv6Only }}
@@ -130,6 +130,7 @@ spec:
       labels:
         k8s-app: node-local-dns
       annotations:
+        policy.cilium.io/no-track-port: "53"
         prometheus.io/port: "9253"
         prometheus.io/scrape: "true"
         {{- range $key, $value := KubeDNS.NodeLocalDNS.PodAnnotations }}
@@ -138,7 +139,11 @@ spec:
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: node-local-dns
+      {{- if or (.Networking.Cilium) ( .KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) }}
+      hostNetwork: false
+      {{- else }}
       hostNetwork: true
+      {{- end }}
       dnsPolicy: Default  # Don't use cluster DNS.
       tolerations:
       - key: "CriticalAddonsOnly"
@@ -155,7 +160,13 @@ spec:
             cpu: {{ KubeDNS.NodeLocalDNS.CPURequest }}
             memory: {{ KubeDNS.NodeLocalDNS.MemoryRequest }}
         args:
+          {{- if or (.Networking.Cilium) ( .KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) }}
+          - -localip={{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ .KubeDNS.ServerIP }}
+          - -skipteardown=true
+          - -setupinterface=false
+          {{- else }}
           - -localip={{ .KubeDNS.NodeLocalDNS.LocalIP }}
+          {{- end }}
           - -conf=/etc/Corefile
           - -upstreamsvc=kube-dns-upstream
           - -setupiptables=false
@@ -208,3 +219,27 @@ spec:
           items:
             - key: Corefile
               path: Corefile.base
+{{- if or (.Networking.Cilium) ( .KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) }}
+---
+apiVersion: "cilium.io/v2"
+kind: CiliumLocalRedirectPolicy
+metadata:
+  name: "nodelocaldns"
+  namespace: kube-system
+spec:
+  redirectFrontend:
+    serviceMatcher:
+      serviceName: kube-dns
+      namespace: kube-system
+  redirectBackend:
+    localEndpointSelector:
+      matchLabels:
+        k8s-app: node-local-dns
+    toPorts:
+      - port: "53"
+        name: dns
+        protocol: UDP
+      - port: "53"
+        name: dns-tcp
+        protocol: TCP
+{{- end }}