nixos/kubernetes: refactor feature gates to attrsOf bool, making it possible to disable featureGates

This is a breaking change, requiring users of `featureGates` to change
from a `listOf str` to `attrsOf bool`.

Before:
```nix
featureGates = [ "EphemeralContainers" ];
extraOpts = pkgs.lib.concatStringsSep " " (
[
  "--container-runtime=remote"
  ''--feature-gates="CSIMigration=false"''
});
```

After:
```nix
featureGates = {EphemeralContainers = true; CSIMigration=false;};
```

This is much nicer, and sets us up for later work of migrating to
configuration files for other services, like e.g. has been happening
with kubelet (see: #290119).

Signed-off-by: Christina Sørensen <christina@cafkafk.com>

nixos/doc/manual/release-notes/rl-2411.section.md

@@ -132,6 +132,24 @@
   nvimpager settings: user commands in `-c` and `--cmd` now override the
   respective default settings because they are executed later.
 
+- Kubernetes `featureGates` have changed from a `listOf str` to `attrsOf bool`.
+  This refactor makes it possible to also disable feature gates, without having
+  to use `extraOpts` flags.
+
+  A previous configuration may have looked like this:
+  ```nix
+  featureGates = [ "EphemeralContainers" ];
+  extraOpts = pkgs.lib.concatStringsSep " " (
+  [
+    ''--feature-gates="CSIMigration=false"''
+  });
+  ```
+
+  Using an AttrSet instead, the new configuration would be:
+  ```nix
+  featureGates = {EphemeralContainers = true; CSIMigration=false;};
+  ```
+
 - `pkgs.nextcloud27` has been removed since it's EOL.
 
 - `services.forgejo.mailerPasswordFile` has been deprecated by the drop-in replacement `services.forgejo.secrets.mailer.PASSWD`,

nixos/modules/services/cluster/kubernetes/apiserver.nix

@@ -159,10 +159,10 @@ in
     };
 
     featureGates = mkOption {
-      description = "List set of feature gates";
+      description = "Attribute set of feature gates.";
       default = top.featureGates;
       defaultText = literalExpression "config.${otop.featureGates}";
-      type = listOf str;
+      type = attrsOf bool;
     };
 
     kubeletClientCaFile = mkOption {
@@ -349,8 +349,8 @@ in
                 "--etcd-certfile=${cfg.etcd.certFile}"} \
               ${optionalString (cfg.etcd.keyFile != null)
                 "--etcd-keyfile=${cfg.etcd.keyFile}"} \
-              ${optionalString (cfg.featureGates != [])
-                "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
+              ${optionalString (cfg.featureGates != {})
+                "--feature-gates=${(concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates)))}"} \
               ${optionalString (cfg.basicAuthFile != null)
                 "--basic-auth-file=${cfg.basicAuthFile}"} \
               ${optionalString (cfg.kubeletClientCaFile != null)

nixos/modules/services/cluster/kubernetes/controller-manager.nix

@@ -44,10 +44,10 @@ in
     };
 
     featureGates = mkOption {
-      description = "List set of feature gates";
+      description = "Attribute set of feature gates.";
       default = top.featureGates;
       defaultText = literalExpression "config.${otop.featureGates}";
-      type = listOf str;
+      type = attrsOf bool;
     };
 
     kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager";
@@ -121,8 +121,8 @@ in
           --bind-address=${cfg.bindAddress} \
           ${optionalString (cfg.clusterCidr!=null)
             "--cluster-cidr=${cfg.clusterCidr}"} \
-          ${optionalString (cfg.featureGates != [])
-            "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
+          ${optionalString (cfg.featureGates != {})
+            "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \
           --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
           --leader-elect=${boolToString cfg.leaderElect} \
           ${optionalString (cfg.rootCaFile!=null)

nixos/modules/services/cluster/kubernetes/default.nix

@@ -155,8 +155,8 @@ in {
 
     featureGates = mkOption {
       description = "List set of feature gates.";
-      default = [];
-      type = types.listOf types.str;
+      default = {};
+      type = types.attrsOf types.bool;
     };
 
     masterAddress = mkOption {

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -65,7 +65,7 @@ let
     // lib.optionalAttrs (cfg.tlsKeyFile != null)   { tlsPrivateKeyFile = cfg.tlsKeyFile; }
     // lib.optionalAttrs (cfg.clusterDomain != "")  { clusterDomain = cfg.clusterDomain; }
     // lib.optionalAttrs (cfg.clusterDns != "")     { clusterDNS = [ cfg.clusterDns ] ; }
-    // lib.optionalAttrs (cfg.featureGates != [])   { featureGates = cfg.featureGates; }
+    // lib.optionalAttrs (cfg.featureGates != {})   { featureGates = cfg.featureGates; }
   ));
 
   manifestPath = "kubernetes/manifests";
@@ -185,10 +185,10 @@ in
     };
 
     featureGates = mkOption {
-      description = "List set of feature gates";
+      description = "Attribute set of feature gate";
       default = top.featureGates;
       defaultText = literalExpression "config.${otop.featureGates}";
-      type = listOf str;
+      type = attrsOf bool;
     };
 
     healthz = {

nixos/modules/services/cluster/kubernetes/proxy.nix

@@ -30,10 +30,10 @@ in
     };
 
     featureGates = mkOption {
-      description = "List set of feature gates";
+      description = "Attribute set of feature gates.";
       default = top.featureGates;
       defaultText = literalExpression "config.${otop.featureGates}";
-      type = listOf str;
+      type = attrsOf bool;
     };
 
     hostname = mkOption {
@@ -69,8 +69,8 @@ in
           --bind-address=${cfg.bindAddress} \
           ${optionalString (top.clusterCidr!=null)
             "--cluster-cidr=${top.clusterCidr}"} \
-          ${optionalString (cfg.featureGates != [])
-            "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
+          ${optionalString (cfg.featureGates != {})
+            "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \
           --hostname-override=${cfg.hostname} \
           --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \

nixos/modules/services/cluster/kubernetes/scheduler.nix

@@ -26,10 +26,10 @@ in
     };
 
     featureGates = mkOption {
-      description = "List set of feature gates";
+      description = "Attribute set of feature gates.";
       default = top.featureGates;
       defaultText = literalExpression "config.${otop.featureGates}";
-      type = listOf str;
+      type = attrsOf bool;
     };
 
     kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler";
@@ -67,8 +67,8 @@ in
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-scheduler \
           --bind-address=${cfg.address} \
-          ${optionalString (cfg.featureGates != [])
-            "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
+          ${optionalString (cfg.featureGates != {})
+            "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \
           --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
           --leader-elect=${boolToString cfg.leaderElect} \
           --secure-port=${toString cfg.port} \

nixos/tests/kubernetes/base.nix

@@ -59,6 +59,10 @@ let
                   securePort = 443;
                   advertiseAddress = master.ip;
                 };
+                # NOTE: what featureGates are useful for testing might change in
+                # the future, see link below to find new ones
+                # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
+                featureGates = {CPUManager = true; AppArmor= false;};
                 masterAddress = "${masterName}.${config.networking.domain}";
               };
             }