From 24bf6e9cb897baa1050317da7559caaaa330fd0e Mon Sep 17 00:00:00 2001 From: r-vdp Date: Mon, 9 Sep 2024 11:16:49 +0200 Subject: [PATCH] nixos/etc-overlay: avoid rebuilding the initrd every time the etc contents change Before this change, the hash of the etc metadata image was included in the mount unit that's responsible for mounting this metadata image in the initrd. And because this metadata image changes with every change to the etc contents, the initrd would be rebuild every time as well. This can lead to a lot of rebuilds (especially when revision info is included in /etc/os-release) and all these initrd archives use up a lot of space on the ESP. With this change, we instead include a symlink to the metadata image in the top-level directory, in the same way as we already do for things like init and prepare-root, and we deduce the store path from the init= kernel parameter, in the same way as we already do to find the path to init and prepare-root. Doing so avoids rebuilding the initrd all the time. --- nixos/modules/system/activation/top-level.nix | 6 + nixos/modules/system/boot/systemd/initrd.nix | 10 +- nixos/modules/system/etc/etc-activation.nix | 109 ++++++++++++++---- .../activation/etc-overlay-immutable.nix | 10 ++ .../tests/activation/etc-overlay-mutable.nix | 10 ++ nixos/tests/systemd-initrd-simple.nix | 2 + 6 files changed, 124 insertions(+), 23 deletions(-) diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 1b0a62c2e8e7..6abbd4b673c0 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -20,6 +20,12 @@ let ''} ln -s ${config.system.build.etc}/etc $out/etc + + ${lib.optionalString config.system.etc.overlay.enable '' + ln -s ${config.system.build.etcMetadataImage} $out/etc-metadata-image + ln -s ${config.system.build.etcBasedir} $out/etc-basedir + ''} + ln -s ${config.system.path} $out/sw ln -s "$systemd" $out/systemd diff --git a/nixos/modules/system/boot/systemd/initrd.nix b/nixos/modules/system/boot/systemd/initrd.nix index 0e62ff2b480c..6f47e6491c88 100644 --- a/nixos/modules/system/boot/systemd/initrd.nix +++ b/nixos/modules/system/boot/systemd/initrd.nix @@ -516,6 +516,7 @@ in { }; before = [ "shutdown.target" ]; conflicts = [ "shutdown.target" ]; + requiredBy = [ "initrd.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -579,7 +580,7 @@ in { ]; services.initrd-nixos-activation = { - wants = [ + requires = [ config.boot.initrd.systemd.services.initrd-find-nixos-closure.name ]; after = [ @@ -587,7 +588,12 @@ in { config.boot.initrd.systemd.services.initrd-find-nixos-closure.name ]; requiredBy = [ "initrd.target" ]; - unitConfig.AssertPathExists = "/etc/initrd-release"; + unitConfig = { + AssertPathExists = "/etc/initrd-release"; + RequiresMountsFor = [ + "/sysroot/run" + ]; + }; serviceConfig.Type = "oneshot"; description = "NixOS Activation"; diff --git a/nixos/modules/system/etc/etc-activation.nix b/nixos/modules/system/etc/etc-activation.nix index 6c6352b0419d..944920e92335 100644 --- a/nixos/modules/system/etc/etc-activation.nix +++ b/nixos/modules/system/etc/etc-activation.nix @@ -1,4 +1,4 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: { @@ -34,12 +34,30 @@ mounts = [ { where = "/run/etc-metadata"; - what = "/sysroot${config.system.build.etcMetadataImage}"; + what = "/etc-metadata-image"; type = "erofs"; options = "loop"; - unitConfig.RequiresMountsFor = [ - "/sysroot/nix/store" + unitConfig = { + # Since this unit depends on the nix store being mounted, it cannot + # be a dependency of local-fs.target, because if it did, we'd have + # local-fs.target ordered after the nix store mount which would cause + # things like network.target to only become active after the nix store + # has been mounted. + # This breaks for instance setups where sshd needs to be up before + # any encrypted disks can be mounted. + DefaultDependencies = false; + RequiresMountsFor = [ + "/sysroot/nix/store" + ]; + }; + requires = [ + config.boot.initrd.systemd.services.initrd-find-etc.name ]; + after = [ + config.boot.initrd.systemd.services.initrd-find-etc.name + ]; + requiredBy = [ "initrd-fs.target" ]; + before = [ "initrd-fs.target" ]; } { where = "/sysroot/etc"; @@ -49,7 +67,7 @@ "relatime" "redirect_dir=on" "metacopy=on" - "lowerdir=/run/etc-metadata::/sysroot${config.system.build.etcBasedir}" + "lowerdir=/run/etc-metadata::/etc-basedir" ] ++ lib.optionals config.system.etc.overlay.mutable [ "rw" "upperdir=/sysroot/.rw-etc/upper" @@ -59,28 +77,77 @@ ]); requiredBy = [ "initrd-fs.target" ]; before = [ "initrd-fs.target" ]; - requires = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ]; - after = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ]; - unitConfig.RequiresMountsFor = [ - "/sysroot/nix/store" - "/run/etc-metadata" + requires = [ + config.boot.initrd.systemd.services.initrd-find-etc.name + ] ++ lib.optionals config.system.etc.overlay.mutable [ + config.boot.initrd.systemd.services."rw-etc".name ]; + after = [ + config.boot.initrd.systemd.services.initrd-find-etc.name + ] ++ lib.optionals config.system.etc.overlay.mutable [ + config.boot.initrd.systemd.services."rw-etc".name + ]; + unitConfig = { + RequiresMountsFor = [ + "/sysroot/nix/store" + "/run/etc-metadata" + ]; + DefaultDependencies = false; + }; } ]; - services = lib.mkIf config.system.etc.overlay.mutable { - rw-etc = { - unitConfig = { - DefaultDependencies = false; - RequiresMountsFor = "/sysroot"; + services = lib.mkMerge [ + (lib.mkIf config.system.etc.overlay.mutable { + rw-etc = { + requiredBy = [ "initrd-fs.target" ]; + before = [ "initrd-fs.target" ]; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = "/sysroot"; + }; + serviceConfig = { + Type = "oneshot"; + ExecStart = '' + /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work + ''; + }; }; - serviceConfig = { - Type = "oneshot"; - ExecStart = '' - /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work + }) + { + initrd-find-etc = { + description = "Find the path to the etc metadata image and based dir"; + requires = [ + config.boot.initrd.systemd.services.initrd-find-nixos-closure.name + ]; + after = [ + config.boot.initrd.systemd.services.initrd-find-nixos-closure.name + ]; + before = [ "shutdown.target" ]; + conflicts = [ "shutdown.target" ]; + requiredBy = [ "initrd.target" ]; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = "/sysroot/nix/store"; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + + script = /* bash */ '' + set -uo pipefail + + closure="$(realpath /nixos-closure)" + + metadata_image="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure/etc-metadata-image")" + ln -s "/sysroot$metadata_image" /etc-metadata-image + + basedir="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure/etc-basedir")" + ln -s "/sysroot$basedir" /etc-basedir ''; }; - }; - }; + } + ]; }; }) diff --git a/nixos/tests/activation/etc-overlay-immutable.nix b/nixos/tests/activation/etc-overlay-immutable.nix index 6d56db43f0b2..2e5389f20227 100644 --- a/nixos/tests/activation/etc-overlay-immutable.nix +++ b/nixos/tests/activation/etc-overlay-immutable.nix @@ -26,6 +26,13 @@ }; testScript = '' + with subtest("/run/etc-metadata/ is mounted"): + print(machine.succeed("mountpoint /run/etc-metadata")) + + with subtest("No temporary files leaked into stage 2"): + machine.succeed("[ ! -e /etc-metadata-image ]") + machine.succeed("[ ! -e /etc-basedir ]") + with subtest("/etc is mounted as an overlay"): machine.succeed("findmnt --kernel --type overlay /etc") @@ -50,6 +57,9 @@ with subtest("switching to the same generation"): machine.succeed("/run/current-system/bin/switch-to-configuration test") + with subtest("the initrd didn't get rebuilt"): + machine.succeed("test /run/current-system/initrd -ef /run/current-system/specialisation/new-generation/initrd") + with subtest("switching to a new generation"): machine.fail("stat /etc/newgen") diff --git a/nixos/tests/activation/etc-overlay-mutable.nix b/nixos/tests/activation/etc-overlay-mutable.nix index 8561ff7fd230..fe6165212470 100644 --- a/nixos/tests/activation/etc-overlay-mutable.nix +++ b/nixos/tests/activation/etc-overlay-mutable.nix @@ -18,12 +18,22 @@ }; testScript = '' + with subtest("/run/etc-metadata/ is mounted"): + print(machine.succeed("mountpoint /run/etc-metadata")) + + with subtest("No temporary files leaked into stage 2"): + machine.succeed("[ ! -e /etc-metadata-image ]") + machine.succeed("[ ! -e /etc-basedir ]") + with subtest("/etc is mounted as an overlay"): machine.succeed("findmnt --kernel --type overlay /etc") with subtest("switching to the same generation"): machine.succeed("/run/current-system/bin/switch-to-configuration test") + with subtest("the initrd didn't get rebuilt"): + machine.succeed("test /run/current-system/initrd -ef /run/current-system/specialisation/new-generation/initrd") + with subtest("switching to a new generation"): machine.fail("stat /etc/newgen") machine.succeed("echo -n 'mutable' > /etc/mutable") diff --git a/nixos/tests/systemd-initrd-simple.nix b/nixos/tests/systemd-initrd-simple.nix index 2b7283a82193..b61cb8ddae7b 100644 --- a/nixos/tests/systemd-initrd-simple.nix +++ b/nixos/tests/systemd-initrd-simple.nix @@ -29,6 +29,8 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: { machine.succeed("[ -e /dev/shm ]") # /dev/shm machine.succeed("[ -e /dev/pts/ptmx ]") # /dev/pts machine.succeed("[ -e /run/keys ]") # /run/keys + # /nixos-closure didn't leak into stage-2 + machine.succeed("[ ! -e /nixos-closure ]") with subtest("groups work"): machine.fail("journalctl -b 0 | grep 'systemd-udevd.*Unknown group.*ignoring'")