fix node-local-dns on cilium 1.16.5+ when BPF host networking is enabled

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

Author: rsafonseca

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3635,6 +3635,12 @@ spec:
                           config for node local dns by the user - it will include
                           the original CoreFile made by kOps.
                         type: string
+                      ciliumBPFCompatibility:
+                        description: CiliumBPFCompatibility allows user to enable
+                          cilium bpf host routing compatibility mode, which is required
+                          for cilium 1.16.5+ and above, when the user is using cilium
+                          as an externally managed daemonset.
+                        type: boolean
                       cpuRequest:
                         anyOf:
                         - type: integer

pkg/apis/kops/cluster.go

@@ -625,6 +625,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/v1alpha2/cluster.go

@@ -604,6 +604,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -590,6 +590,8 @@ type NodeLocalDNSConfig struct {
 	// PodAnnotations makes possible to add additional annotations to node-local-dns.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
+	// CiliumBPFCompatibility allows user to enable cilium bpf host routing compatibility mode, which is required for cilium 1.16.5+ and above, when the user is using cilium as an externally managed daemonset.
+	CiliumBPFCompatibility *bool `json:"ciliumBPFCompatibility,omitempty"`
 }
 
 type ExternalDNSProvider string

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -87,7 +87,7 @@ func (b *KubeletOptionsBuilder) configureKubelet(cluster *kops.Cluster, kubelet
 	}
 
 	if kubelet.ClusterDNS == "" {
-		if cluster.Spec.KubeDNS != nil && cluster.Spec.KubeDNS.NodeLocalDNS != nil && fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.Enabled) {
+		if cluster.Spec.KubeDNS != nil && cluster.Spec.KubeDNS.NodeLocalDNS != nil && fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.Enabled) && !fi.ValueOf(cluster.Spec.KubeDNS.NodeLocalDNS.CiliumBPFCompatibility) && cluster.Spec.Networking.Cilium == nil {
 			kubelet.ClusterDNS = cluster.Spec.KubeDNS.NodeLocalDNS.LocalIP
 		} else {
 			ip, err := WellKnownServiceIP(&cluster.Spec.Networking, 10)