Fixes #18124: atomically replace /var/setuid-wrappers/ (#18186)

Before this commit updating /var/setuid-wrappers/ folder introduced
a small window where NixOS activation scripts could be terminated
and resulted into empty /var/setuid-wrappers/ folder.

That's very unfortunate because one might lose sudo binary.

Instead we use two atomic operations mv and ln (as described in
https://axialcorps.com/2013/07/03/atomically-replacing-files-and-directories/)
to achieve atomicity.

Since /var/setuid-wrappers is not a directory anymore, tmpfs mountpoints
were removed in installation scripts and in boot process.

Tested:

- upgrade /var/setuid-wrappers/ from folder to a symlink
- make sure /run/setuid-wrappers-dirs/ legacy symlink is really deleted

4 files changed, 35 insertions, 16 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1609.xml b/nixos/doc/manual/release-notes/rl-1609.xml
index 78b57dddf0761..70759ee25f863 100644
--- a/nixos/doc/manual/release-notes/rl-1609.xml
+++ b/nixos/doc/manual/release-notes/rl-1609.xml
@@ -58,6 +58,14 @@ following incompatible changes:</para>
   </listitem>
 
   <listitem>
+    <para>/var/setuid-wrappers/
+      <link xlink:href="https://github.com/NixOS/nixpkgs/pull/18124">is now a symlink so
+      it can be atomically updated</link>
+      and it's not mounted as tmpfs anymore since setuid binaries are located on /run/ as tmpfs.
+    </para>
+  </listitem>
+
+  <listitem>
     <para>Gitlab's maintainence script gitlab-runner was removed and split up into the more clearer
       gitlab-run and gitlab-rake scripts because gitlab-runner is a component of Gitlab CI.</para>
   </listitem>
diff --git a/nixos/modules/installer/tools/nixos-install.sh b/nixos/modules/installer/tools/nixos-install.sh
index 62ea98c4676c0..589a51fa70941 100644
--- a/nixos/modules/installer/tools/nixos-install.sh
+++ b/nixos/modules/installer/tools/nixos-install.sh
@@ -92,14 +92,12 @@ fi
 mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
 mkdir -m 01777 -p $mountPoint/tmp
 mkdir -m 0755 -p $mountPoint/tmp/root
-mkdir -m 0755 -p $mountPoint/var/setuid-wrappers
 mkdir -m 0700 -p $mountPoint/root
 mount --rbind /dev $mountPoint/dev
 mount --rbind /proc $mountPoint/proc
 mount --rbind /sys $mountPoint/sys
 mount --rbind / $mountPoint/tmp/root
 mount -t tmpfs -o "mode=0755" none $mountPoint/run
-mount -t tmpfs -o "mode=0755" none $mountPoint/var/setuid-wrappers
 rm -rf $mountPoint/var/run
 ln -s /run $mountPoint/var/run
 for f in /etc/resolv.conf /etc/hosts; do rm -f $mountPoint/$f; [ -f "$f" ] && cp -Lf $f $mountPoint/etc/; done
diff --git a/nixos/modules/security/setuid-wrappers.nix b/nixos/modules/security/setuid-wrappers.nix
index 99dd514feea3f..e1dca477d70a0 100644
--- a/nixos/modules/security/setuid-wrappers.nix
+++ b/nixos/modules/security/setuid-wrappers.nix
@@ -12,7 +12,7 @@ let
     installPhase = ''
       mkdir -p $out/bin
       cp ${./setuid-wrapper.c} setuid-wrapper.c
-      gcc -Wall -O2 -DWRAPPER_DIR=\"${wrapperDir}\" \
+      gcc -Wall -O2 -DWRAPPER_DIR=\"/run/setuid-wrapper-dirs\" \
           setuid-wrapper.c -o $out/bin/setuid-wrapper
     '';
   };
@@ -102,11 +102,11 @@ in
                 source=/nix/var/nix/profiles/default/bin/${program}
             fi
 
-            cp ${setuidWrapper}/bin/setuid-wrapper ${wrapperDir}/${program}
-            echo -n "$source" > ${wrapperDir}/${program}.real
-            chmod 0000 ${wrapperDir}/${program} # to prevent races
-            chown ${owner}.${group} ${wrapperDir}/${program}
-            chmod "u${if setuid then "+" else "-"}s,g${if setgid then "+" else "-"}s,${permissions}" ${wrapperDir}/${program}
+            cp ${setuidWrapper}/bin/setuid-wrapper $wrapperDir/${program}
+            echo -n "$source" > $wrapperDir/${program}.real
+            chmod 0000 $wrapperDir/${program} # to prevent races
+            chown ${owner}.${group} $wrapperDir/${program}
+            chmod "u${if setuid then "+" else "-"}s,g${if setgid then "+" else "-"}s,${permissions}" $wrapperDir/${program}
           '';
 
       in stringAfter [ "users" ]
@@ -115,9 +115,29 @@ in
           # programs to be wrapped.
           SETUID_PATH=${config.system.path}/bin:${config.system.path}/sbin
 
-          rm -f ${wrapperDir}/* # */
+          mkdir -p /run/setuid-wrapper-dirs
+          wrapperDir=$(mktemp --directory --tmpdir=/run/setuid-wrapper-dirs setuid-wrappers.XXXXXXXXXX)
 
           ${concatMapStrings makeSetuidWrapper setuidPrograms}
+
+          if [ -L ${wrapperDir} ]; then
+            # Atomically replace the symlink
+            # See https://axialcorps.com/2013/07/03/atomically-replacing-files-and-directories/
+            old=$(readlink ${wrapperDir})
+            ln --symbolic --force --no-dereference $wrapperDir ${wrapperDir}-tmp
+            mv --no-target-directory ${wrapperDir}-tmp ${wrapperDir}
+            rm --force --recursive $old
+          elif [ -d ${wrapperDir} ]; then
+            # Compatibility with old state, just remove the folder and symlink
+            rm -f ${wrapperDir}/*
+            # if it happens to be a tmpfs
+            umount ${wrapperDir} || true
+            rm -d ${wrapperDir}
+            ln -d --symbolic $wrapperDir ${wrapperDir}
+          else
+            # For initial setup
+            ln --symbolic $wrapperDir ${wrapperDir}
+          fi
         '';
 
   };
diff --git a/nixos/modules/system/boot/stage-2-init.sh b/nixos/modules/system/boot/stage-2-init.sh
index 7de85209a159c..704150e77d727 100644
--- a/nixos/modules/system/boot/stage-2-init.sh
+++ b/nixos/modules/system/boot/stage-2-init.sh
@@ -141,13 +141,6 @@ if [ -n "@useHostResolvConf@" -a -e /etc/resolv.conf ]; then
     cat /etc/resolv.conf | resolvconf -m 1000 -a host
 fi
 
-
-# Create /var/setuid-wrappers as a tmpfs.
-rm -rf /var/setuid-wrappers
-mkdir -m 0755 -p /var/setuid-wrappers
-mount -t tmpfs -o "mode=0755" tmpfs /var/setuid-wrappers
-
-
 # Log the script output to /dev/kmsg or /run/log/stage-2-init.log.
 # Only at this point are all the necessary prerequisites ready for these commands.
 exec {logOutFd}>&1 {logErrFd}>&2