Hook fold

src/libstore/build/derivation-building-goal.cc

@@ -18,6 +18,7 @@
 #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts
 #include "nix/store/outputs-query.hh"
 #include "nix/store/globals.hh"
+#include "nix/store/machines.hh"
 #include "nix/util/current-process.hh"
 
 #include <chrono>
@@ -1233,9 +1234,27 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
     if (worker.settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
         return rpDecline;
 
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>(
-            worker.settings.buildHook, std::chrono::milliseconds(worker.settings.buildHookKillTimeout));
+    if (!worker.hook) {
+        auto timeout = std::chrono::milliseconds(worker.settings.buildHookKillTimeout);
+        /* Only a configured hook runs as a program. Compare by value:
+           `loadConfFile` resets `isOverridden()`. */
+        if (!worker.settings.buildHook.isDefault())
+            worker.hook = HookInstance::external(worker.settings.buildHook, timeout);
+        else {
+            /* No builders: the hook would decline permanently. */
+            bool haveMachines = true;
+            try {
+                haveMachines = !Machine::parseConfig({settings.thisSystem}, worker.settings.builders).empty();
+            } catch (Error &) {
+                /* the hook reports it */
+            }
+            if (!haveMachines) {
+                worker.tryBuildHook = false;
+                return rpDecline;
+            }
+            worker.hook = HookInstance::builtin(worker.store.config, timeout);
+        }
+    }
 
     try {
 

src/libstore/include/nix/store/machine-selection.hh

@@ -0,0 +1,85 @@
+#pragma once
+///@file
+
+#include <filesystem>
+#include <variant>
+
+#include "nix/store/machines.hh"
+#include "nix/store/pathlocks.hh"
+
+namespace nix {
+
+class Store;
+
+/**
+ * The directory holding the slot locks that record how many builds are
+ * running on each machine: `<state-dir>/current-load`.
+ */
+std::filesystem::path getCurrentLoadDir(Store & store);
+
+/**
+ * Replace `/` with `_` so that a store URI can be used as a single path
+ * component of a lock file name.
+ */
+std::string escapeUri(std::string uri);
+
+/**
+ * Whether `machine` could run a build for `neededSystem` with
+ * `requiredFeatures` at all, ignoring how busy it currently is.
+ */
+bool machineIsCandidate(const Machine & machine, const std::string & neededSystem, const StringSet & requiredFeatures);
+
+/**
+ * Whether `cand`, running `candLoad` builds, beats `best`, running
+ * `bestLoad`. Load is divided by speed factor; ties go to the faster.
+ */
+bool machineIsBetter(const Machine & cand, uint64_t candLoad, const Machine & best, uint64_t bestLoad);
+
+/**
+ * A machine with one of its build slots reserved. The reservation lasts
+ * as long as `slotLock` is held.
+ */
+struct AcquiredMachine
+{
+    /**
+     * Borrowed from the `machines` argument of `acquireMachineSlot`.
+     */
+    Machine * machine;
+    AutoCloseFD slotLock;
+};
+
+/**
+ * Why `acquireMachineSlot` found no machine.
+ */
+enum class NoMachine {
+    /**
+     * No machine supports this system and feature set, so waiting would
+     * not help.
+     */
+    WrongType,
+    /**
+     * A machine of the right type exists, but every one of its slots is
+     * taken. Waiting may help.
+     */
+    AllBusy,
+};
+
+/**
+ * Reserve a build slot on the least loaded machine that can run this
+ * build, under a lock on `currentLoadDir/main-lock`. The result points
+ * into `machines`.
+ */
+std::variant<AcquiredMachine, NoMachine> acquireMachineSlot(
+    const std::filesystem::path & currentLoadDir,
+    Machines & machines,
+    const std::string & neededSystem,
+    const StringSet & requiredFeatures);
+
+/**
+ * Take the lock that serialises uploads to `storeUri`, so that
+ * concurrent builds do not each copy the same closure to the same
+ * machine.
+ */
+AutoCloseFD openUploadLock(const std::filesystem::path & currentLoadDir, std::string_view storeUri);
+
+} // namespace nix

src/libstore/include/nix/store/meson.build

@@ -63,6 +63,7 @@ headers = [ config_pub_h ] + files(
   'local-settings.hh',
   'local-store.hh',
   'log-store.hh',
+  'machine-selection.hh',
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',

src/libstore/include/nix/store/worker-settings.hh

@@ -122,10 +122,9 @@ public:
         {"nix", "__build-remote"},
         "build-hook",
         R"(
-          The path to the helper program that executes remote builds.
+          The program that performs remote builds, speaking the build-hook protocol over `stdio` with the Nix that requested them.
 
-          Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon.
-          The default value is the internal Nix binary that implements remote building.
+          With the default value Nix runs its own implementation in a forked child of the building process (on platforms other than Linux and macOS, as `nix __build-remote`); an explicit `build-hook = nix __build-remote` selects the same. Any other value is run as a separate program.
 
           > **Important**
           >

src/libstore/machine-selection.cc

@@ -0,0 +1,136 @@
+#include <algorithm>
+
+#ifndef _WIN32
+#  ifdef __APPLE__
+#    include <sys/time.h>
+#  else
+#    include <sys/stat.h>
+#  endif
+#endif
+
+#include "nix/store/machine-selection.hh"
+#include "nix/store/globals.hh"
+#include "nix/store/local-fs-store.hh"
+#include "nix/util/hash.hh"
+
+namespace nix {
+
+std::filesystem::path getCurrentLoadDir(Store & store)
+{
+    /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
+       that gets cleared on reboot, but it wouldn't work on macOS. */
+    if (auto * localStore = dynamic_cast<LocalFSStore *>(&store))
+        return localStore->config.stateDir.get() / "current-load";
+    else
+        return std::filesystem::path{settings.nixStateDir} / "current-load";
+}
+
+std::string escapeUri(std::string uri)
+{
+    std::replace(uri.begin(), uri.end(), '/', '_');
+    return uri;
+}
+
+static AutoCloseFD openSlotLock(const std::filesystem::path & currentLoadDir, const Machine & m, uint64_t slot)
+{
+    return openLockFile(currentLoadDir / fmt("%s-%d", escapeUri(m.storeUri.render()), slot), true);
+}
+
+bool machineIsCandidate(const Machine & machine, const std::string & neededSystem, const StringSet & requiredFeatures)
+{
+    return machine.enabled && machine.systemSupported(neededSystem) && machine.allSupported(requiredFeatures)
+           && machine.mandatoryMet(requiredFeatures);
+}
+
+bool machineIsBetter(const Machine & cand, uint64_t candLoad, const Machine & best, uint64_t bestLoad)
+{
+    if (candLoad / cand.speedFactor < bestLoad / best.speedFactor)
+        return true;
+    if (candLoad / cand.speedFactor == bestLoad / best.speedFactor) {
+        if (cand.speedFactor > best.speedFactor)
+            return true;
+        if (cand.speedFactor == best.speedFactor)
+            return candLoad < bestLoad;
+    }
+    return false;
+}
+
+std::variant<AcquiredMachine, NoMachine> acquireMachineSlot(
+    const std::filesystem::path & currentLoadDir,
+    Machines & machines,
+    const std::string & neededSystem,
+    const StringSet & requiredFeatures)
+{
+    AutoCloseFD lock = openLockFile(currentLoadDir / "main-lock", true);
+    lockFile(lock.get(), ltWrite, true);
+
+    bool rightType = false;
+
+    Machine * bestMachine = nullptr;
+    uint64_t bestLoad = 0;
+    AutoCloseFD bestSlotLock;
+
+    for (auto & m : machines) {
+        debug("considering building on remote machine '%s'", m.storeUri.render());
+
+        if (!machineIsCandidate(m, neededSystem, requiredFeatures))
+            continue;
+
+        rightType = true;
+
+        AutoCloseFD free;
+        uint64_t load = 0;
+        for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
+            auto slotLock = openSlotLock(currentLoadDir, m, slot);
+            if (lockFile(slotLock.get(), ltWrite, false)) {
+                if (!free) {
+                    free = std::move(slotLock);
+                }
+            } else {
+                ++load;
+            }
+        }
+        if (!free) {
+            continue;
+        }
+        if (!bestSlotLock || machineIsBetter(m, load, *bestMachine, bestLoad)) {
+            bestLoad = load;
+            bestSlotLock = std::move(free);
+            bestMachine = &m;
+        }
+    }
+
+    if (!bestSlotLock)
+        return rightType ? NoMachine::AllBusy : NoMachine::WrongType;
+
+#ifndef _WIN32
+#  ifdef __APPLE__
+    futimes(bestSlotLock.get(), NULL);
+#  else
+    futimens(bestSlotLock.get(), NULL);
+#  endif
+#endif
+
+    return AcquiredMachine{
+        .machine = bestMachine,
+        .slotLock = std::move(bestSlotLock),
+    };
+}
+
+AutoCloseFD openUploadLock(const std::filesystem::path & currentLoadDir, std::string_view storeUri)
+{
+    auto open = [&](std::string_view fileName) {
+        return openLockFile(currentLoadDir / (escapeUri(std::string{fileName}) + ".upload-lock"), true);
+    };
+    try {
+        return open(storeUri);
+    } catch (SystemError & e) {
+        if (!e.is(std::errc::filename_too_long))
+            throw;
+        /* Try again hashing the store URL so we have a shorter path. */
+        auto h = hashString(HashAlgorithm::MD5, storeUri);
+        return open(h.to_string(HashFormat::Base64, false));
+    }
+}
+
+} // namespace nix

src/libstore/meson.build

@@ -301,6 +301,7 @@ sources = files(
   'local-overlay-store.cc',
   'local-store.cc',
   'log-store.cc',
+  'machine-selection.cc',
   'machines.cc',
   'make-content-addressed.cc',
   'misc.cc',