Trim worker.hh

2024-11-29 17:16:15 +02:00 · 2020-10-12 17:16:48 +00:00 · 2020-10-12 17:16:48 +00:00 · 542972f029
commit 542972f029
parent 0e2306204a
1 changed files with 4 additions and 593 deletions
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@ -1,127 +1,14 @@
 #pragma once
-#include "machines.hh"
+#include "types.hh"
 #include "parsed-derivations.hh"
 #include "lock.hh"
 #include "local-store.hh"
-
+#include "goal.hh"
 #include <memory>
 #include <algorithm>
 #include <iostream>
 #include <map>
 #include <sstream>
 #include <thread>
 #include <future>
 #include <chrono>
 #include <regex>
 #include <queue>
 namespace nix {
 using std::map;
 /* Forward definition. */
 class Worker;
 struct HookInstance;
 /* A pointer to a goal. */
 struct Goal;
 class DerivationGoal;
 typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
 struct CompareGoalPtrs {
    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
 };
 /* Set of goals. */
 typedef set<GoalPtr, CompareGoalPtrs> Goals;
 typedef list<WeakGoalPtr> WeakGoals;
 /* A map of paths to goals (and the other way around). */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 struct Goal : public std::enable_shared_from_this<Goal>
 {
    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
    /* Backlink to the worker. */
    Worker & worker;
    /* Goals that this goal is waiting for. */
    Goals waitees;
    /* Goals waiting for this one to finish.  Must use weak pointers
       here to prevent cycles. */
    WeakGoals waiters;
    /* Number of goals we are/were waiting for that have failed. */
    unsigned int nrFailed;
    /* Number of substitution goals we are/were waiting for that
       failed because there are no substituters. */
    unsigned int nrNoSubstituters;
    /* Number of substitution goals we are/were waiting for that
       failed because othey had unsubstitutable references. */
    unsigned int nrIncompleteClosure;
    /* Name of this goal for debugging purposes. */
    string name;
    /* Whether the goal is finished. */
    ExitCode exitCode;
    /* Exception containing an error message, if any. */
    std::optional<Error> ex;
    Goal(Worker & worker) : worker(worker)
    {
        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
        exitCode = ecBusy;
    }
    virtual ~Goal()
    {
        trace("goal destroyed");
    }
    virtual void work() = 0;
    void addWaitee(GoalPtr waitee);
    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
    virtual void handleChildOutput(int fd, const string & data)
    {
        abort();
    }
    virtual void handleEOF(int fd)
    {
        abort();
    }
    void trace(const FormatOrString & fs);
    string getName()
    {
        return name;
    }
    /* Callback in case of a timeout.  It should wake up its waiters,
       get rid of any running child processes that are being monitored
       by the worker (important!), etc. */
    virtual void timedOut(Error && ex) = 0;
    virtual string key() = 0;
    void amDone(ExitCode result, std::optional<Error> ex = {});
 };
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
@ -140,6 +27,8 @@ struct Child
    steady_time_point timeStarted;
 };
 /* Forward definition. */
 struct HookInstance;
 /* The worker class. */
 class Worker
@ -303,482 +192,4 @@ public:
    }
 };
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 class SubstitutionGoal;
 /* Unless we are repairing, we don't both to test validity and just assume it,
   so the choices are `Absent` or `Valid`. */
 enum struct PathStatus {
    Corrupt,
    Absent,
    Valid,
 };
 struct InitialOutputStatus {
    StorePath path;
    PathStatus status;
    /* Valid in the store, and additionally non-corrupt if we are repairing */
    bool isValid() const {
        return status == PathStatus::Valid;
    }
    /* Merely present, allowed to be corrupt */
    bool isPresent() const {
        return status == PathStatus::Corrupt
            || status == PathStatus::Valid;
    }
 };
 struct InitialOutput {
    bool wanted;
    std::optional<InitialOutputStatus> known;
 };
 class DerivationGoal : public Goal
 {
 private:
    /* Whether to use an on-disk .drv file. */
    bool useDerivation;
    /* The path of the derivation. */
    StorePath drvPath;
    /* The specific outputs that we need to build.  Empty means all of
       them. */
    StringSet wantedOutputs;
    /* Whether additional wanted outputs have been added. */
    bool needRestart = false;
    /* Whether to retry substituting the outputs after building the
       inputs. */
    bool retrySubstitution;
    /* The derivation stored at drvPath. */
    std::unique_ptr<BasicDerivation> drv;
    std::unique_ptr<ParsedDerivation> parsedDrv;
    /* The remainder is state held during the build. */
    /* Locks on (fixed) output paths. */
    PathLocks outputLocks;
    /* All input paths (that is, the union of FS closures of the
       immediate input paths). */
    StorePathSet inputPaths;
    std::map<std::string, InitialOutput> initialOutputs;
    /* User selected for running the builder. */
    std::unique_ptr<UserLock> buildUser;
    /* The process ID of the builder. */
    Pid pid;
    /* The temporary directory. */
    Path tmpDir;
    /* The path of the temporary directory in the sandbox. */
    Path tmpDirInSandbox;
    /* File descriptor for the log file. */
    AutoCloseFD fdLogFile;
    std::shared_ptr<BufferedSink> logFileSink, logSink;
    /* Number of bytes received from the builder's stdout/stderr. */
    unsigned long logSize;
    /* The most recent log lines. */
    std::list<std::string> logTail;
    std::string currentLogLine;
    size_t currentLogLinePos = 0; // to handle carriage return
    std::string currentHookLine;
    /* Pipe for the builder's standard output/error. */
    Pipe builderOut;
    /* Pipe for synchronising updates to the builder namespaces. */
    Pipe userNamespaceSync;
    /* The mount namespace of the builder, used to add additional
       paths to the sandbox as a result of recursive Nix calls. */
    AutoCloseFD sandboxMountNamespace;
    /* On Linux, whether we're doing the build in its own user
       namespace. */
    bool usingUserNamespace = true;
    /* The build hook. */
    std::unique_ptr<HookInstance> hook;
    /* Whether we're currently doing a chroot build. */
    bool useChroot = false;
    Path chrootRootDir;
    /* RAII object to delete the chroot directory. */
    std::shared_ptr<AutoDelete> autoDelChroot;
    /* The sort of derivation we are building. */
    DerivationType derivationType;
    /* Whether to run the build in a private network namespace. */
    bool privateNetwork = false;
    typedef void (DerivationGoal::*GoalState)();
    GoalState state;
    /* Stuff we need to pass to initChild(). */
    struct ChrootPath {
        Path source;
        bool optional;
        ChrootPath(Path source = "", bool optional = false)
            : source(source), optional(optional)
        { }
    };
    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
    DirsInChroot dirsInChroot;
    typedef map<string, string> Environment;
    Environment env;
 #if __APPLE__
    typedef string SandboxProfile;
    SandboxProfile additionalSandboxProfile;
 #endif
    /* Hash rewriting. */
    StringMap inputRewrites, outputRewrites;
    typedef map<StorePath, StorePath> RedirectedOutputs;
    RedirectedOutputs redirectedOutputs;
    /* The outputs paths used during the build.
       - Input-addressed derivations or fixed content-addressed outputs are
         sometimes built when some of their outputs already exist, and can not
         be hidden via sandboxing. We use temporary locations instead and
         rewrite after the build. Otherwise the regular predetermined paths are
         put here.
       - Floating content-addressed derivations do not know their final build
         output paths until the outputs are hashed, so random locations are
         used, and then renamed. The randomness helps guard against hidden
         self-references.
     */
    OutputPathMap scratchOutputs;
    /* The final output paths of the build.
       - For input-addressed derivations, always the precomputed paths
       - For content-addressed derivations, calcuated from whatever the hash
         ends up being. (Note that fixed outputs derivations that produce the
         "wrong" output still install that data under its true content-address.)
     */
    OutputPathMap finalOutputs;
    BuildMode buildMode;
    /* If we're repairing without a chroot, there may be outputs that
       are valid but corrupt.  So we redirect these outputs to
       temporary paths. */
    StorePathSet redirectedBadOutputs;
    BuildResult result;
    /* The current round, if we're building multiple times. */
    size_t curRound = 1;
    size_t nrRounds;
    /* Path registration info from the previous round, if we're
       building multiple times. Since this contains the hash, it
       allows us to compare whether two rounds produced the same
       result. */
    std::map<Path, ValidPathInfo> prevInfos;
    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
    const static Path homeDir;
    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
    std::unique_ptr<Activity> act;
    /* Activity that denotes waiting for a lock. */
    std::unique_ptr<Activity> actLock;
    std::map<ActivityId, Activity> builderActivities;
    /* The remote machine on which we're building. */
    std::string machineName;
    /* The recursive Nix daemon socket. */
    AutoCloseFD daemonSocket;
    /* The daemon main thread. */
    std::thread daemonThread;
    /* The daemon worker threads. */
    std::vector<std::thread> daemonWorkerThreads;
    /* Paths that were added via recursive Nix calls. */
    StorePathSet addedPaths;
    /* Recursive Nix calls are only allowed to build or realize paths
       in the original input closure or added via a recursive Nix call
       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
       /nix/store/<bla> is some arbitrary path in a binary cache). */
    bool isAllowed(const StorePath & path)
    {
        return inputPaths.count(path) || addedPaths.count(path);
    }
    friend struct RestrictedStore;
 public:
    DerivationGoal(const StorePath & drvPath,
        const StringSet & wantedOutputs, Worker & worker,
        BuildMode buildMode = bmNormal);
    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
        const StringSet & wantedOutputs, Worker & worker,
        BuildMode buildMode = bmNormal);
    ~DerivationGoal();
    /* Whether we need to perform hash rewriting if there are valid output paths. */
    bool needsHashRewrite();
    void timedOut(Error && ex) override;
    string key() override
    {
        /* Ensure that derivations get built in order of their name,
           i.e. a derivation named "aardvark" always comes before
           "baboon". And substitution goals always happen before
           derivation goals (due to "b$"). */
        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
    }
    void work() override;
    StorePath getDrvPath()
    {
        return drvPath;
    }
    /* Add wanted outputs to an already existing derivation goal. */
    void addWantedOutputs(const StringSet & outputs);
    BuildResult getResult() { return result; }
 private:
    /* The states. */
    void getDerivation();
    void loadDerivation();
    void haveDerivation();
    void outputsSubstitutionTried();
    void gaveUpOnSubstitution();
    void closureRepaired();
    void inputsRealised();
    void tryToBuild();
    void tryLocalBuild();
    void buildDone();
    void resolvedFinished();
    /* Is the build hook willing to perform the build? */
    HookReply tryBuildHook();
    /* Start building a derivation. */
    void startBuilder();
    /* Fill in the environment for the builder. */
    void initEnv();
    /* Setup tmp dir location. */
    void initTmpDir();
    /* Write a JSON file containing the derivation attributes. */
    void writeStructuredAttrs();
    void startDaemon();
    void stopDaemon();
    /* Add 'path' to the set of paths that may be referenced by the
       outputs, and make it appear in the sandbox. */
    void addDependency(const StorePath & path);
    /* Make a file owned by the builder. */
    void chownToBuilder(const Path & path);
    /* Run the builder's process. */
    void runChild();
    friend int childEntry(void *);
    /* Check that the derivation outputs all exist and register them
       as valid. */
    void registerOutputs();
    /* Check that an output meets the requirements specified by the
       'outputChecks' attribute (or the legacy
       '{allowed,disallowed}{References,Requisites}' attributes). */
    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
    /* Open a log file and a pipe to it. */
    Path openLogFile();
    /* Close the log file. */
    void closeLogFile();
    /* Delete the temporary directory, if we have one. */
    void deleteTmpDir(bool force);
    /* Callback used by the worker to write to the log. */
    void handleChildOutput(int fd, const string & data) override;
    void handleEOF(int fd) override;
    void flushLine();
    /* Wrappers around the corresponding Store methods that first consult the
       derivation.  This is currently needed because when there is no drv file
       there also is no DB entry. */
    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
    OutputPathMap queryDerivationOutputMap();
    /* Return the set of (in)valid paths. */
    void checkPathValidity();
    /* Forcibly kill the child process, if any. */
    void killChild();
    /* Create alternative path calculated from but distinct from the
       input, so we can avoid overwriting outputs (or other store paths)
       that already exist. */
    StorePath makeFallbackPath(const StorePath & path);
    /* Make a path to another based on the output name along with the
       derivation hash. */
    /* FIXME add option to randomize, so we can audit whether our
       rewrites caught everything */
    StorePath makeFallbackPath(std::string_view outputName);
    void repairClosure();
    void started();
    void done(
        BuildResult::Status status,
        std::optional<Error> ex = {});
    StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 class SubstitutionGoal : public Goal
 {
    friend class Worker;
 private:
    /* The store path that should be realised through a substitute. */
    StorePath storePath;
    /* The path the substituter refers to the path as. This will be
     * different when the stores have different names. */
    std::optional<StorePath> subPath;
    /* The remaining substituters. */
    std::list<ref<Store>> subs;
    /* The current substituter. */
    std::shared_ptr<Store> sub;
    /* Whether a substituter failed. */
    bool substituterFailed = false;
    /* Path info returned by the substituter's query info operation. */
    std::shared_ptr<const ValidPathInfo> info;
    /* Pipe for the substituter's standard output. */
    Pipe outPipe;
    /* The substituter thread. */
    std::thread thr;
    std::promise<void> promise;
    /* Whether to try to repair a valid path. */
    RepairFlag repair;
    /* Location where we're downloading the substitute.  Differs from
       storePath when doing a repair. */
    Path destPath;
    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
    typedef void (SubstitutionGoal::*GoalState)();
    GoalState state;
    /* Content address for recomputing store path */
    std::optional<ContentAddress> ca;
 public:
    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
    ~SubstitutionGoal();
    void timedOut(Error && ex) override { abort(); };
    string key() override
    {
        /* "a$" ensures substitution goals happen before derivation
           goals. */
        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
    }
    void work() override;
    /* The states. */
    void init();
    void tryNext();
    void gotInfo();
    void referencesValid();
    void tryToRun();
    void finished();
    /* Callback used by the worker to write to the log. */
    void handleChildOutput(int fd, const string & data) override;
    void handleEOF(int fd) override;
    StorePath getStorePath() { return storePath; }
 };
 struct HookInstance
 {
    /* Pipes for talking to the build hook. */
    Pipe toHook;
    /* Pipe for the hook's standard output/error. */
    Pipe fromHook;
    /* Pipe for the builder's standard output/error. */
    Pipe builderOut;
    /* The process ID of the hook. */
    Pid pid;
    FdSink sink;
    std::map<ActivityId, Activity> activities;
    HookInstance();
    ~HookInstance();
 };
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);
 }