nixos/etc-overlay: avoid rebuilding the initrd every time the etc contents change

Before this change, the hash of the etc metadata image was included in
the mount unit that's responsible for mounting this metadata image in the
initrd.
And because this metadata image changes with every change to the etc
contents, the initrd would be rebuild every time as well.
This can lead to a lot of rebuilds (especially when revision info is
included in /etc/os-release) and all these initrd archives use up a lot of
space on the ESP.

With this change, we instead include a symlink to the metadata image in the
top-level directory, in the same way as we already do for things like init and
prepare-root, and we deduce the store path from the init= kernel parameter,
in the same way as we already do to find the path to init and prepare-root.

Doing so avoids rebuilding the initrd all the time.

nixos/modules/system/activation/top-level.nix

@@ -20,6 +20,12 @@ let
       ''}
 
       ln -s ${config.system.build.etc}/etc $out/etc
+
+      ${lib.optionalString config.system.etc.overlay.enable ''
+        ln -s ${config.system.build.etcMetadataImage} $out/etc-metadata-image
+        ln -s ${config.system.build.etcBasedir} $out/etc-basedir
+      ''}
+
       ln -s ${config.system.path} $out/sw
       ln -s "$systemd" $out/systemd
 

nixos/modules/system/boot/systemd/initrd.nix

@@ -516,6 +516,7 @@ in {
         };
         before = [ "shutdown.target" ];
         conflicts = [ "shutdown.target" ];
+        requiredBy = [ "initrd.target" ];
         serviceConfig = {
           Type = "oneshot";
           RemainAfterExit = true;
@@ -579,7 +580,7 @@ in {
       ];
 
       services.initrd-nixos-activation = {
-        wants = [
+        requires = [
           config.boot.initrd.systemd.services.initrd-find-nixos-closure.name
         ];
         after = [
@@ -587,7 +588,12 @@ in {
           config.boot.initrd.systemd.services.initrd-find-nixos-closure.name
         ];
         requiredBy = [ "initrd.target" ];
-        unitConfig.AssertPathExists = "/etc/initrd-release";
+        unitConfig = {
+          AssertPathExists = "/etc/initrd-release";
+          RequiresMountsFor = [
+            "/sysroot/run"
+          ];
+        };
         serviceConfig.Type = "oneshot";
         description = "NixOS Activation";
 

nixos/modules/system/etc/etc-activation.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, lib, pkgs, ... }:
 
 {
 
@@ -34,12 +34,30 @@
         mounts = [
           {
             where = "/run/etc-metadata";
-            what = "/sysroot${config.system.build.etcMetadataImage}";
+            what = "/etc-metadata-image";
             type = "erofs";
             options = "loop";
-            unitConfig.RequiresMountsFor = [
-              "/sysroot/nix/store"
+            unitConfig = {
+              # Since this unit depends on the nix store being mounted, it cannot
+              # be a dependency of local-fs.target, because if it did, we'd have
+              # local-fs.target ordered after the nix store mount which would cause
+              # things like network.target to only become active after the nix store
+              # has been mounted.
+              # This breaks for instance setups where sshd needs to be up before
+              # any encrypted disks can be mounted.
+              DefaultDependencies = false;
+              RequiresMountsFor = [
+                "/sysroot/nix/store"
+              ];
+            };
+            requires = [
+              config.boot.initrd.systemd.services.initrd-find-etc.name
             ];
+            after = [
+              config.boot.initrd.systemd.services.initrd-find-etc.name
+            ];
+            requiredBy = [ "initrd-fs.target" ];
+            before = [ "initrd-fs.target" ];
           }
           {
             where = "/sysroot/etc";
@@ -49,7 +67,7 @@
               "relatime"
               "redirect_dir=on"
               "metacopy=on"
-              "lowerdir=/run/etc-metadata::/sysroot${config.system.build.etcBasedir}"
+              "lowerdir=/run/etc-metadata::/etc-basedir"
             ] ++ lib.optionals config.system.etc.overlay.mutable [
               "rw"
               "upperdir=/sysroot/.rw-etc/upper"
@@ -59,28 +77,77 @@
             ]);
             requiredBy = [ "initrd-fs.target" ];
             before = [ "initrd-fs.target" ];
-            requires = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ];
-            after = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ];
-            unitConfig.RequiresMountsFor = [
-              "/sysroot/nix/store"
-              "/run/etc-metadata"
+            requires = [
+              config.boot.initrd.systemd.services.initrd-find-etc.name
+            ] ++ lib.optionals config.system.etc.overlay.mutable [
+              config.boot.initrd.systemd.services."rw-etc".name
             ];
+            after = [
+              config.boot.initrd.systemd.services.initrd-find-etc.name
+            ] ++ lib.optionals config.system.etc.overlay.mutable [
+              config.boot.initrd.systemd.services."rw-etc".name
+            ];
+            unitConfig = {
+              RequiresMountsFor = [
+                "/sysroot/nix/store"
+                "/run/etc-metadata"
+              ];
+              DefaultDependencies = false;
+            };
           }
         ];
-        services = lib.mkIf config.system.etc.overlay.mutable {
-          rw-etc = {
-            unitConfig = {
-              DefaultDependencies = false;
-              RequiresMountsFor = "/sysroot";
+        services = lib.mkMerge [
+          (lib.mkIf config.system.etc.overlay.mutable {
+            rw-etc = {
+              requiredBy = [ "initrd-fs.target" ];
+              before = [ "initrd-fs.target" ];
+              unitConfig = {
+                DefaultDependencies = false;
+                RequiresMountsFor = "/sysroot";
+              };
+              serviceConfig = {
+                Type = "oneshot";
+                ExecStart = ''
+                  /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
+                '';
+              };
             };
-            serviceConfig = {
-              Type = "oneshot";
-              ExecStart = ''
-                /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
+          })
+          {
+            initrd-find-etc = {
+              description = "Find the path to the etc metadata image and based dir";
+              requires = [
+                config.boot.initrd.systemd.services.initrd-find-nixos-closure.name
+              ];
+              after = [
+                config.boot.initrd.systemd.services.initrd-find-nixos-closure.name
+              ];
+              before = [ "shutdown.target" ];
+              conflicts = [ "shutdown.target" ];
+              requiredBy = [ "initrd.target" ];
+              unitConfig = {
+                DefaultDependencies = false;
+                RequiresMountsFor = "/sysroot/nix/store";
+              };
+              serviceConfig = {
+                Type = "oneshot";
+                RemainAfterExit = true;
+              };
+
+              script = /* bash */ ''
+                set -uo pipefail
+
+                closure="$(realpath /nixos-closure)"
+
+                metadata_image="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure/etc-metadata-image")"
+                ln -s "/sysroot$metadata_image" /etc-metadata-image
+
+                basedir="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure/etc-basedir")"
+                ln -s "/sysroot$basedir" /etc-basedir
               '';
             };
-          };
-        };
+          }
+        ];
       };
 
     })

nixos/tests/activation/etc-overlay-immutable.nix

@@ -26,6 +26,13 @@
   };
 
   testScript = ''
+    with subtest("/run/etc-metadata/ is mounted"):
+      print(machine.succeed("mountpoint /run/etc-metadata"))
+
+    with subtest("No temporary files leaked into stage 2"):
+      machine.succeed("[ ! -e /etc-metadata-image ]")
+      machine.succeed("[ ! -e /etc-basedir ]")
+
     with subtest("/etc is mounted as an overlay"):
       machine.succeed("findmnt --kernel --type overlay /etc")
 
@@ -50,6 +57,9 @@
     with subtest("switching to the same generation"):
       machine.succeed("/run/current-system/bin/switch-to-configuration test")
 
+    with subtest("the initrd didn't get rebuilt"):
+      machine.succeed("test /run/current-system/initrd -ef /run/current-system/specialisation/new-generation/initrd")
+
     with subtest("switching to a new generation"):
       machine.fail("stat /etc/newgen")
 

nixos/tests/activation/etc-overlay-mutable.nix

@@ -18,12 +18,22 @@
   };
 
   testScript = ''
+    with subtest("/run/etc-metadata/ is mounted"):
+      print(machine.succeed("mountpoint /run/etc-metadata"))
+
+    with subtest("No temporary files leaked into stage 2"):
+      machine.succeed("[ ! -e /etc-metadata-image ]")
+      machine.succeed("[ ! -e /etc-basedir ]")
+
     with subtest("/etc is mounted as an overlay"):
       machine.succeed("findmnt --kernel --type overlay /etc")
 
     with subtest("switching to the same generation"):
       machine.succeed("/run/current-system/bin/switch-to-configuration test")
 
+    with subtest("the initrd didn't get rebuilt"):
+      machine.succeed("test /run/current-system/initrd -ef /run/current-system/specialisation/new-generation/initrd")
+
     with subtest("switching to a new generation"):
       machine.fail("stat /etc/newgen")
       machine.succeed("echo -n 'mutable' > /etc/mutable")

nixos/tests/systemd-initrd-simple.nix

@@ -29,6 +29,8 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
         machine.succeed("[ -e /dev/shm ]") # /dev/shm
         machine.succeed("[ -e /dev/pts/ptmx ]") # /dev/pts
         machine.succeed("[ -e /run/keys ]") # /run/keys
+        # /nixos-closure didn't leak into stage-2
+        machine.succeed("[ ! -e /nixos-closure ]")
 
     with subtest("groups work"):
         machine.fail("journalctl -b 0 | grep 'systemd-udevd.*Unknown group.*ignoring'")