buildFHSEnv: fix nested fhsenvs with LD_PRELOAD

I hate this, but I also kinda love this. It's very cursed. Please help.

Co-authored-by: Alyssa Ross <hi@alyssa.is>

pkgs/build-support/build-fhsenv-bubblewrap/container-init.cc

@@ -0,0 +1,63 @@
+#include <fstream>
+
+#include <spawn.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <sys/wait.h>
+
+const char LD_SO_CONF[] = R"(/lib
+/lib/x86_64-linux-gnu
+/lib64
+/usr/lib
+/usr/lib/x86_64-linux-gnu
+/usr/lib64
+/lib/i386-linux-gnu
+/lib32
+/usr/lib/i386-linux-gnu
+/usr/lib32
+/run/opengl-driver/lib
+/run/opengl-driver-32/lib
+)";
+
+int main(int, const char *argv[]) {
+  std::ofstream ld_so_conf;
+  ld_so_conf.open("/etc/ld.so.conf");
+  ld_so_conf << LD_SO_CONF;
+  ld_so_conf.close();
+  if (!ld_so_conf) {
+    perror("Failed to generate ld.so.conf");
+    return 1;
+  }
+
+  int e;
+  pid_t pid;
+  const char *ldconfig_argv[] = {"/bin/ldconfig", NULL};
+  char *ldconfig_envp[] = {NULL};
+  if ((e = posix_spawn(&pid, ldconfig_argv[0], NULL, NULL,
+                       (char *const *)ldconfig_argv, ldconfig_envp))) {
+    fprintf(stderr, "Failed to run ldconfig: %s\n", strerror(e));
+    return 1;
+  }
+
+  int status;
+  if (waitpid(pid, &status, 0) == -1) {
+    perror("Failed to wait for ldconfig");
+    return 1;
+  }
+  if (WIFEXITED(status)) {
+    if (WEXITSTATUS(status)) {
+      fprintf(stderr, "ldconfig exited %d\n", WEXITSTATUS(status));
+      return 1;
+    }
+  } else {
+    fprintf(stderr, "ldconfig killed by signal %d\n", WTERMSIG(status));
+    return 1;
+  }
+
+  argv[0] = "/init";
+  execv(argv[0], (char *const *)argv);
+
+  perror("Failed to exec stage 2 init");
+  return 1;
+}

pkgs/build-support/build-fhsenv-bubblewrap/default.nix

@@ -5,6 +5,7 @@
 , writeShellScript
 , glibc
 , pkgsi686Linux
+, runCommandCC
 , coreutils
 , bubblewrap
 }:
@@ -98,29 +99,30 @@ let
     ];
   in map (path: "/etc/${path}") files;
 
-  # Create this on the fly instead of linking from /nix
+  # Here's the problem case:
-  # The container might have to modify it and re-run ldconfig if there are
+  # - we need to run bash to run the init script
-  # issues running some binary with LD_LIBRARY_PATH
+  # - LD_PRELOAD may be set to another dynamic library, requiring us to discover its dependencies
-  createLdConfCache = ''
+  # - oops! ldconfig is part of the init script, and it hasn't run yet
-    cat > /etc/ld.so.conf <<EOF
+  # - everything explodes
-    /lib
+  #
-    /lib/x86_64-linux-gnu
+  # In particular, this happens with fhsenvs in fhsenvs, e.g. when running
-    /lib64
+  # a wrapped game from Steam.
-    /usr/lib
+  #
-    /usr/lib/x86_64-linux-gnu
+  # So, instead of doing that, we build a tiny static (important!) shim
-    /usr/lib64
+  # that executes ldconfig in a completely clean environment to generate
-    /lib/i386-linux-gnu
+  # the initial cache, and then execs into the "real" init, which is the
-    /lib32
+  # first time we see anything dynamically linked at all.
-    /usr/lib/i386-linux-gnu
+  #
-    /usr/lib32
+  # Also, the real init is placed strategically at /init, so we don't
-    /run/opengl-driver/lib
+  # have to recompile this every time.
-    /run/opengl-driver-32/lib
+  containerInit = runCommandCC "container-init" {
-    EOF
+    buildInputs = [ stdenv.cc.libc.static or null ];
-    ldconfig &> /dev/null
+  } ''
+    $CXX -static -s -o $out ${./container-init.cc}
   '';
-  init = run: writeShellScript "${name}-init" ''
+
+  realInit = run: writeShellScript "${name}-init" ''
     source /etc/profile
-    ${createLdConfCache}
     exec ${run} "$@"
   '';
 
@@ -253,6 +255,7 @@ let
       --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \
       --ro-bind ${glibc}/etc/rpc ${glibc}/etc/rpc \
       --remount-ro ${glibc}/etc \
+      --symlink ${realInit runScript} /init \
   '' + optionalString fhsenv.isMultiBuild (indentLines ''
       --tmpfs ${pkgsi686Linux.glibc}/etc \
       --symlink /etc/ld.so.conf ${pkgsi686Linux.glibc}/etc/ld.so.conf \
@@ -265,7 +268,7 @@ let
       "''${auto_mounts[@]}"
       "''${x11_args[@]}"
       ${concatStringsSep "\n  " extraBwrapArgs}
-      ${init runScript} ${initArgs}
+      ${containerInit} ${initArgs}
     )
     exec "''${cmd[@]}"
   '';