From 46b31880458f0cbea2a45a42734d1ac76c9ea88f Mon Sep 17 00:00:00 2001
From: Parker Hoyes <contact@parkerhoyes.com>
Date: Mon, 26 Aug 2024 13:50:22 +0000
Subject: [PATCH] Move daemon process into sub-cgroup

The daemon process is now moved into a new sub-cgroup called nix-daemon when the
daemon starts. This is necessary to abide by the no-processes-in-inner-nodes
rule, because the service cgroup becomes an inner node when the child cgroups
for the build are created (see LocalDerivationGoal::startBuilder()).

See #9675
---
 .../unix/build/local-derivation-goal.cc       | 21 ++++++------
 src/libutil/linux/cgroup.cc                   | 32 +++++++++++++++++++
 src/libutil/linux/cgroup.hh                   |  9 ++++++
 src/nix/unix/daemon.cc                        | 25 +++++++++++++++
 4 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc
index 01a133766..d55278a52 100644
--- a/src/libstore/unix/build/local-derivation-goal.cc
+++ b/src/libstore/unix/build/local-derivation-goal.cc
@@ -444,25 +444,22 @@ void LocalDerivationGoal::startBuilder()
         #if __linux__
         experimentalFeatureSettings.require(Xp::Cgroups);
 
+        /* If we're running from the daemon, then this will return the
+           root cgroup of the service. Otherwise, it will return the
+           current cgroup. */
+        auto rootCgroup = getRootCgroup();
         auto cgroupFS = getCgroupFS();
         if (!cgroupFS)
             throw Error("cannot determine the cgroups file system");
-
-        auto ourCgroups = getCgroups("/proc/self/cgroup");
-        auto ourCgroup = ourCgroups[""];
-        if (ourCgroup == "")
-            throw Error("cannot determine cgroup name from /proc/self/cgroup");
-
-        auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup);
-
-        if (!pathExists(ourCgroupPath))
-            throw Error("expected cgroup directory '%s'", ourCgroupPath);
+        auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup);
+        if (!pathExists(rootCgroupPath))
+            throw Error("expected cgroup directory '%s'", rootCgroupPath);
 
         static std::atomic<unsigned int> counter{0};
 
         cgroup = buildUser
-            ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID())
-            : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++);
+            ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID())
+            : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++);
 
         debug("using cgroup '%s'", *cgroup);
 
diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc
index 140ff4566..706f0f159 100644
--- a/src/libutil/linux/cgroup.cc
+++ b/src/libutil/linux/cgroup.cc
@@ -6,6 +6,7 @@
 
 #include <chrono>
 #include <cmath>
+#include <mutex>
 #include <regex>
 #include <unordered_set>
 #include <thread>
@@ -144,4 +145,35 @@ CgroupStats destroyCgroup(const Path & cgroup)
     return destroyCgroup(cgroup, true);
 }
 
+std::string getCurrentCgroup()
+{
+    auto cgroupFS = getCgroupFS();
+    if (!cgroupFS)
+        throw Error("cannot determine the cgroups file system");
+
+    auto ourCgroups = getCgroups("/proc/self/cgroup");
+    auto ourCgroup = ourCgroups[""];
+    if (ourCgroup == "")
+        throw Error("cannot determine cgroup name from /proc/self/cgroup");
+    return ourCgroup;
+}
+
+static std::optional<std::string> rootCgroup;
+static std::mutex rootCgroupMutex;
+
+std::string getRootCgroup()
+{
+    {
+        std::lock_guard<std::mutex> guard(rootCgroupMutex);
+        if (rootCgroup)
+            return *rootCgroup;
+    }
+    auto current = getCurrentCgroup();
+    std::lock_guard<std::mutex> guard(rootCgroupMutex);
+    if (rootCgroup)
+        return *rootCgroup;
+    rootCgroup = current;
+    return current;
+}
+
 }
diff --git a/src/libutil/linux/cgroup.hh b/src/libutil/linux/cgroup.hh
index 783a0ab87..87d135ba6 100644
--- a/src/libutil/linux/cgroup.hh
+++ b/src/libutil/linux/cgroup.hh
@@ -25,4 +25,13 @@ struct CgroupStats
  */
 CgroupStats destroyCgroup(const Path & cgroup);
 
+std::string getCurrentCgroup();
+
+/**
+ * Get the cgroup that should be used as the parent when creating new
+ * sub-cgroups. The first time this is called, the current cgroup will be
+ * returned, and then all subsequent calls will return the original cgroup.
+ */
+std::string getRootCgroup();
+
 }
diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc
index 66d8dbcf0..746963a01 100644
--- a/src/nix/unix/daemon.cc
+++ b/src/nix/unix/daemon.cc
@@ -33,6 +33,10 @@
 #include <grp.h>
 #include <fcntl.h>
 
+#if __linux__
+#include "cgroup.hh"
+#endif
+
 #if __APPLE__ || __FreeBSD__
 #include <sys/ucred.h>
 #endif
@@ -312,6 +316,27 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
     //  Get rid of children automatically; don't let them become zombies.
     setSigChldAction(true);
 
+    #if __linux__
+    if (settings.useCgroups) {
+        experimentalFeatureSettings.require(Xp::Cgroups);
+
+        //  This also sets the root cgroup to the current one.
+        auto rootCgroup = getRootCgroup();
+        auto cgroupFS = getCgroupFS();
+        if (!cgroupFS)
+            throw Error("cannot determine the cgroups file system");
+        auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup);
+        if (!pathExists(rootCgroupPath))
+            throw Error("expected cgroup directory '%s'", rootCgroupPath);
+        auto daemonCgroupPath = rootCgroupPath + "/nix-daemon";
+        //  Create new sub-cgroup for the daemon.
+        if (mkdir(daemonCgroupPath.c_str(), 0755) != 0 && errno != EEXIST)
+            throw SysError("creating cgroup '%s'", daemonCgroupPath);
+        //  Move daemon into the new cgroup.
+        writeFile(daemonCgroupPath + "/cgroup.procs", fmt("%d", getpid()));
+    }
+    #endif
+
     //  Loop accepting connections.
     while (1) {