From 3ec83565b1e01aeb7c3b98a5e03b2e98dfe1353c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Feb 2022 16:38:22 +0100 Subject: [PATCH] Checkpoint --- src/libexpr/eval.cc | 6 ++-- src/libexpr/flake/flake.cc | 15 +++++++-- src/libexpr/flake/flake.hh | 3 +- src/libexpr/paths.cc | 5 ++- src/libexpr/primops/fetchTree.cc | 23 +++++++++++--- src/libfetchers/fetchers.cc | 20 ++++++++++++ src/libfetchers/fetchers.hh | 7 ++++ src/libfetchers/input-accessor.cc | 14 +++++--- src/libfetchers/input-accessor.hh | 2 ++ src/libfetchers/path.cc | 53 ++++++++++++++++++++----------- src/nix/flake.cc | 7 ++++ 11 files changed, 116 insertions(+), 39 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 80f8986a8..304cb95f1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2070,14 +2070,14 @@ std::string EvalState::copyPathToStore(PathSet & context, const Path & path) auto path2 = unpackPath(path); #if 0 auto p = settings.readOnlyMode - ? store->computeStorePathForPath(std::string(baseNameOf(path)), canonPath(path)).first - : store->addToStore(std::string(baseNameOf(path)), canonPath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); + ? store->computeStorePathForPath(path2.baseName(), canonPath(path)).first + : store->addToStore(path2.baseName(), canonPath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); #endif auto source = sinkToSource([&](Sink & sink) { path2.accessor->dumpPath(path2.path, sink); }); // FIXME: readOnlyMode - auto p = store->addToStoreFromDump(*source, std::string(baseNameOf(path)), FileIngestionMethod::Recursive, htSHA256, repair); + auto p = store->addToStoreFromDump(*source, path2.baseName(), FileIngestionMethod::Recursive, htSHA256, repair); dstPath = store->printStorePath(p); allowPath(p); srcToStore.insert_or_assign(path, std::move(p)); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 7f2304e7e..7dffbc632 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -209,7 +209,7 @@ static Flake getFlake( .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + //.sourceInfo = std::make_shared(std::move(sourceInfo)) }; if (!pathExists(flakeFile)) @@ -326,6 +326,7 @@ LockedFlake lockFlake( state.store->setOptions(); } + #if 0 try { // FIXME: symlink attack @@ -669,6 +670,9 @@ LockedFlake lockFlake( e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); throw; } + #endif + + throw UnimplementedError("lockFlake"); } void callFlake(EvalState & state, @@ -683,13 +687,17 @@ void callFlake(EvalState & state, vLocks->mkString(lockedFlake.lockFile.to_string()); + #if 0 emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + //*lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc, false, lockedFlake.flake.forceDirty); + #endif + + throw UnimplementedError("callFlake"); vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); @@ -756,7 +764,8 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + "FIXME", + //flake.sourceInfo->storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 524b18af1..068d5c6af 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -63,7 +63,6 @@ struct Flake FlakeRef lockedRef; // the specific local store result of invoking the fetcher bool forceDirty = false; // pretend that 'lockedRef' is dirty std::optional description; - std::shared_ptr sourceInfo; FlakeInputs inputs; ConfigFile config; // 'nixConfig' attribute ~Flake(); @@ -139,7 +138,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const SourcePath & path, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index baf5b24f4..d2dcfb877 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -16,15 +16,14 @@ Path EvalState::packPath(const SourcePath & path) SourcePath EvalState::unpackPath(const Path & path) { + printError("UNPACK %s", path); if (hasPrefix(path, marker)) { auto s = path.substr(marker.size()); auto slash = s.find('/'); - assert(slash != s.npos); auto n = std::stoi(s.substr(0, slash)); - printError("GOT %d", n); auto i = inputAccessors.find(n); assert(i != inputAccessors.end()); - return {i->second, s.substr(slash)}; + return {i->second, slash != std::string::npos ? s.substr(slash) : "/"}; } else { printError("FIXME: %s", path); return rootPath(path); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 42c98e312..01691ef6b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -13,25 +13,32 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const SourcePath & path, const fetchers::Input & input, Value & v, bool emptyRevFallback, bool forceDirty) { - assert(input.isLocked()); + // FIXME? + //assert(input.isLocked()); auto attrs = state.buildBindings(8); + #if 0 auto storePath = state.store->printStorePath(tree.storePath); attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + #endif + + attrs.alloc(state.sOutPath).mkPath(state.packPath(path)); // FIXME: support arbitrary input attributes. + #if 0 auto narHash = input.getNarHash(); assert(narHash); attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + #endif if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -169,11 +176,17 @@ static void fetchTree( if (evalSettings.pureEval && !input.isLocked()) throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos); - auto [tree, input2] = input.fetch(state.store); + auto [accessor, input2] = input.lazyFetch(state.store); - state.allowPath(tree.storePath); + //state.allowPath(tree.storePath); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + emitTreeAttrs( + state, + {accessor, "/"}, + input2, + v, + params.emptyRevFallback, + false); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 976f40d3b..136784358 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -172,6 +172,19 @@ std::pair Input::fetch(ref store) const return {std::move(tree), input}; } +std::pair, Input> Input::lazyFetch(ref store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + + try { + return scheme->lazyFetch(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } +} + Input Input::applyOverrides( std::optional ref, std::optional rev) const @@ -289,4 +302,11 @@ void InputScheme::clone(const Input & input, const Path & destDir) throw Error("do not know how to clone input '%s'", input.to_string()); } +std::pair, Input> InputScheme::lazyFetch(ref store, const Input & input) +{ + auto [storePath, input2] = fetch(store, input); + + return {makeFSInputAccessor(store->toRealPath(storePath)), input2}; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b6f9d0648..9c67217ee 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -73,6 +73,11 @@ public: the Nix store and the locked input. */ std::pair fetch(ref store) const; + /* Return an InputAccessor that allows access to files in the + input without copying it to the store. Also return a possibly + unlocked input. */ + std::pair, Input> lazyFetch(ref store) const; + Input applyOverrides( std::optional ref, std::optional rev) const; @@ -132,6 +137,8 @@ struct InputScheme virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair, Input> lazyFetch(ref store, const Input & input); + virtual ref getAccessor() { throw UnimplementedError("getAccessor"); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index c625007a5..df6d7c39b 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -14,7 +14,7 @@ InputAccessor::InputAccessor() // FIXME: merge with archive.cc. const std::string narVersionMagic1 = "nix-archive-1"; -static string caseHackSuffix = "~nix~case~hack~"; +static std::string caseHackSuffix = "~nix~case~hack~"; void InputAccessor::dumpPath( const Path & path, @@ -51,12 +51,12 @@ void InputAccessor::dumpPath( /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ - std::map unhacked; + std::map unhacked; for (auto & i : readDirectory(path)) if (/* archiveSettings.useCaseHack */ false) { // FIXME - string name(i.first); + std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); - if (pos != string::npos) { + if (pos != std::string::npos) { debug(format("removing case hack suffix from '%s'") % (path + "/" + i.first)); name.erase(pos); } @@ -227,4 +227,10 @@ ref makeMemoryInputAccessor() return make_ref(); } +std::string_view SourcePath::baseName() const +{ + // FIXME + return path == "" || path == "/" ? "source" : baseNameOf(path); +} + } diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index d4260518a..8e59181a3 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -57,6 +57,8 @@ struct SourcePath { ref accessor; Path path; + + std::string_view baseName() const; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f0ef97da5..cc51bf89a 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -81,29 +81,36 @@ struct PathInputScheme : InputScheme // nothing to do } + Path getAbsPath(ref store, const Input & input) + { + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return path; + + if (!input.parent) + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); + + auto parent = canonPath(*input.parent); + + // the path isn't relative, prefix it + auto absPath = nix::absPath(path, parent); + + // for security, ensure that if the parent is a store path, it's inside it + if (store->isInStore(parent)) { + auto storePath = store->printStorePath(store->toStorePath(parent).first); + if (!isDirOrInDir(absPath, storePath)) + throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); + } + + return absPath; + } + std::pair fetch(ref store, const Input & _input) override { Input input(_input); - std::string absPath; - auto path = getStrAttr(input.attrs, "path"); - if (path[0] != '/') { - if (!input.parent) - throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); - - auto parent = canonPath(*input.parent); - - // the path isn't relative, prefix it - absPath = nix::absPath(path, parent); - - // for security, ensure that if the parent is a store path, it's inside it - if (store->isInStore(parent)) { - auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isDirOrInDir(absPath, storePath)) - throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); - } - } else - absPath = path; + auto absPath = getAbsPath(store, input); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); @@ -125,6 +132,14 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::pair, Input> lazyFetch(ref store, const Input & input) override + { + auto absPath = getAbsPath(store, input); + auto input2(input); + input2.attrs.emplace("path", absPath); + return {makeFSInputAccessor(absPath), std::move(input2)}; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 47a380238..66e8295ad 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -182,7 +182,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; + #if 0 j["path"] = store->printStorePath(flake.sourceInfo->storePath); + #endif j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -196,9 +198,11 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); + #if 0 logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", store->printStorePath(flake.sourceInfo->storePath)); + #endif if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -881,6 +885,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; + throw UnimplementedError("flake archive"); + #if 0 sources.insert(flake.flake.sourceInfo->storePath); if (jsonRoot) jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); @@ -911,6 +917,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); copyPaths(*store, *dstStore, sources); } + #endif } };