From 950b46821f644eb3f92725460584a3102f356179 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 30 May 2020 00:44:11 +0200 Subject: [PATCH] Remove TreeInfo The attributes previously stored in TreeInfo (narHash, revCount, lastModified) are now stored in Input. This makes it less arbitrary what attributes are stored where. As a result, the lock file format has changed. An entry like "info": { "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github" }, is now stored as "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github", "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, The 'Input' class is now a dumb set of attributes. All the fetcher implementations subclass InputScheme, not Input. This simplifies the API. Also, fix substitution of flake inputs. This was broken since lazy flake fetching started using fetchTree internally. --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/flake/call-flake.nix | 2 +- src/libexpr/flake/flake.cc | 52 ++--- src/libexpr/flake/flake.hh | 2 +- src/libexpr/flake/flakeref.cc | 22 +- src/libexpr/flake/flakeref.hh | 10 +- src/libexpr/flake/lockfile.cc | 40 ++-- src/libexpr/flake/lockfile.hh | 4 +- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchGit.cc | 10 +- src/libexpr/primops/fetchMercurial.cc | 14 +- src/libexpr/primops/fetchTree.cc | 71 ++++-- src/libfetchers/attrs.cc | 14 +- src/libfetchers/attrs.hh | 11 +- src/libfetchers/fetchers.cc | 224 ++++++++++++++++--- src/libfetchers/fetchers.hh | 116 +++++----- src/libfetchers/git.cc | 310 +++++++++++--------------- src/libfetchers/github.cc | 303 +++++++++++-------------- src/libfetchers/indirect.cc | 154 +++++-------- src/libfetchers/mercurial.cc | 210 +++++++---------- src/libfetchers/path.cc | 184 ++++++--------- src/libfetchers/registry.cc | 56 ++--- src/libfetchers/registry.hh | 17 +- src/libfetchers/tarball.cc | 143 +++++------- src/libfetchers/tree-info.cc | 60 ----- src/libfetchers/tree-info.hh | 33 --- src/nix/flake.cc | 32 +-- src/nix/installables.cc | 2 +- src/nix/profile.cc | 2 +- src/nix/registry.cc | 6 +- 30 files changed, 963 insertions(+), 1145 deletions(-) delete mode 100644 src/libfetchers/tree-info.cc delete mode 100644 src/libfetchers/tree-info.hh diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 8665abe8c..6b48ead1f 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 8ee17b8f4..2084e3fb3 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -11,7 +11,7 @@ let sourceInfo = if key == lockFile.root then rootSrc - else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]); + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {}); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 78b58cdfa..9741d98c5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -19,7 +19,7 @@ static FlakeRef maybeLookupFlake( const FlakeRef & flakeRef, bool allowLookup) { - if (!flakeRef.input->isDirect()) { + if (!flakeRef.input.isDirect()) { if (allowLookup) return flakeRef.resolve(store); else @@ -49,16 +49,15 @@ static FlakeRef lookupInFlakeCache( static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, - std::optional treeInfo, bool allowLookup, FlakeCache & flakeCache) { /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { + if (originalRef.input.isDirect() && originalRef.input.isImmutable() && originalRef.input.hasAllInfo()) { try { - auto storePath = treeInfo->computeStorePath(*state.store); + auto storePath = originalRef.input.computeStorePath(*state.store); state.store->ensurePath(storePath); @@ -74,7 +73,6 @@ static std::tuple fetchOrSubstituteTree( Tree { .actualPath = actualPath, .storePath = std::move(storePath), - .info = *treeInfo, }, originalRef, originalRef @@ -99,8 +97,7 @@ static std::tuple fetchOrSubstituteTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - if (treeInfo) - assert(tree.storePath == treeInfo->computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); return {std::move(tree), resolvedRef, lockedRef}; } @@ -202,12 +199,11 @@ static std::map parseFlakeInputs( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - std::optional treeInfo, bool allowLookup, FlakeCache & flakeCache) { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, treeInfo, allowLookup, flakeCache); + state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); @@ -278,7 +274,7 @@ static Flake getFlake( Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { FlakeCache flakeCache; - return getFlake(state, originalRef, {}, allowLookup, flakeCache); + return getFlake(state, originalRef, allowLookup, flakeCache); } /* Compute an in-memory lock file for the specified top-level flake, @@ -292,7 +288,7 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); + auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); // FIXME: symlink attack auto oldLockFile = LockFile::read( @@ -393,7 +389,7 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = std::make_shared( - oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -409,7 +405,7 @@ LockedFlake lockFlake( if (hasChildUpdate) { auto inputFlake = getFlake( - state, oldLock->lockedRef, oldLock->info, false, flakeCache); + state, oldLock->lockedRef, false, flakeCache); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); } else { /* No need to fetch this flake, we can be @@ -440,11 +436,11 @@ LockedFlake lockFlake( /* We need to create a new lock file entry. So fetch this input. */ - if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) + if (!lockFlags.allowMutable && !input.ref.input.isImmutable()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { - auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); + auto inputFlake = getFlake(state, input.ref, lockFlags.useRegistries, flakeCache); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -454,7 +450,7 @@ LockedFlake lockFlake( file. That is, overrides are sticky unless you use --no-write-lock-file. */ auto childNode = std::make_shared( - inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info); + inputFlake.lockedRef, input2.ref); node->inputs.insert_or_assign(id, childNode); @@ -479,9 +475,9 @@ LockedFlake lockFlake( else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, input.ref, {}, lockFlags.useRegistries, flakeCache); + state, input.ref, lockFlags.useRegistries, flakeCache); node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, input.ref, sourceInfo.info, false)); + std::make_shared(lockedRef, input.ref, false)); } } } @@ -534,7 +530,7 @@ LockedFlake lockFlake( printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input->getSourcePath()) { + if (auto sourcePath = topRef.input.getSourcePath()) { if (!newLockFile.isImmutable()) { if (settings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); @@ -555,7 +551,7 @@ LockedFlake lockFlake( newLockFile.write(path); - topRef.input->markChangedFile( + topRef.input.markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", lockFlags.commitLockFile ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", @@ -567,19 +563,19 @@ LockedFlake lockFlake( also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; FlakeCache dummyCache; - flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); if (lockFlags.commitLockFile && - flake.lockedRef.input->getRev() && - prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) - warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); /* Make sure that we picked up the change, i.e. the tree should usually be dirty now. Corner case: we could have reverted from a dirty to a clean tree! */ if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input->isImmutable()) + && !flake.lockedRef.input.isImmutable()) throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else @@ -625,7 +621,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va { auto flakeRefS = state.forceStringNoCtx(*args[0], pos); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input->isImmutable()) + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); callFlake(state, @@ -650,8 +646,8 @@ Fingerprint LockedFlake::getFingerprint() const return hashString(htSHA256, fmt("%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), - flake.sourceInfo->info.revCount.value_or(0), - flake.sourceInfo->info.lastModified.value_or(0), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), lockFile)); } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 59a1adb3b..ebf81362c 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -104,7 +104,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr input, + const fetchers::Input & input, Value & v); } diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index a70261a41..615269218 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -15,7 +15,7 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege std::string FlakeRef::to_string() const { - auto url = input->toURL(); + auto url = input.toURL(); if (subdir != "") url.query.insert_or_assign("dir", subdir); return url.to_string(); @@ -23,7 +23,7 @@ std::string FlakeRef::to_string() const fetchers::Attrs FlakeRef::toAttrs() const { - auto attrs = input->toAttrs(); + auto attrs = input.toAttrs(); if (subdir != "") attrs.emplace("dir", subdir); return attrs; @@ -37,13 +37,13 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) bool FlakeRef::operator ==(const FlakeRef & other) const { - return *input == *other.input && subdir == other.subdir; + return input == other.input && subdir == other.subdir; } FlakeRef FlakeRef::resolve(ref store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); - return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } FlakeRef parseFlakeRef( @@ -98,7 +98,7 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(inputFromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL), ""), percentDecode(std::string(match[6]))); } @@ -143,7 +143,7 @@ std::pair parseFlakeRefWithFragment( } return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } @@ -155,7 +155,7 @@ std::pair parseFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment); + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); } else { @@ -163,7 +163,7 @@ std::pair parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } } @@ -183,14 +183,14 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::inputFromAttrs(attrs2), + fetchers::Input::fromAttrs(std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input->fetchTree(store); - return {std::move(tree), FlakeRef(lockedInput, subdir)}; + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 72cbb2908..f4eb825a6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -14,17 +14,15 @@ typedef std::string FlakeId; struct FlakeRef { - std::shared_ptr input; + fetchers::Input input; Path subdir; bool operator==(const FlakeRef & other) const; - FlakeRef(const std::shared_ptr & input, const Path & subdir) - : input(input), subdir(subdir) - { - assert(input); - } + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } // FIXME: change to operator <<. std::string to_string() const; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 5c58d6080..68e587650 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -5,35 +5,40 @@ namespace nix::flake { -FlakeRef flakeRefFromJson(const nlohmann::json & json) -{ - return FlakeRef::fromAttrs(jsonToAttrs(json)); -} - FlakeRef getFlakeRef( const nlohmann::json & json, - const char * attr) + const char * attr, + const char * info) { auto i = json.find(attr); - if (i != json.end()) - return flakeRefFromJson(*i); + if (i != json.end()) { + auto attrs = jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } throw Error("attribute '%s' missing in lock file", attr); } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked")) - , originalRef(getFlakeRef(json, "original")) - , info(TreeInfo::fromJson(json)) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input->isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", lockedRef); + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const { - return info.computeStorePath(store); + return lockedRef.input.computeStorePath(store); } std::shared_ptr Node::findInput(const InputPath & path) @@ -53,7 +58,7 @@ std::shared_ptr Node::findInput(const InputPath & path) LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); - if (version != 5) + if (version < 5 || version > 6) throw Error("lock file '%s' has unsupported version %d", path, version); std::unordered_map> nodeMap; @@ -119,7 +124,6 @@ nlohmann::json LockFile::toJson() const if (auto lockedNode = std::dynamic_pointer_cast(node)) { n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); - n["info"] = lockedNode->info.toJson(); if (!lockedNode->isFlake) n["flake"] = false; } @@ -129,7 +133,7 @@ nlohmann::json LockFile::toJson() const }; nlohmann::json json; - json["version"] = 5; + json["version"] = 6; json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); @@ -176,7 +180,7 @@ bool LockFile::isImmutable() const for (auto & i : nodes) { if (i == root) continue; auto lockedNode = std::dynamic_pointer_cast(i); - if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false; + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; } return true; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index c34939ebc..ba47f9b89 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -31,15 +31,13 @@ struct Node : std::enable_shared_from_this struct LockedNode : Node { FlakeRef lockedRef, originalRef; - TreeInfo info; bool isFlake = true; LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - const TreeInfo & info, bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1993fa6c1..1ac5217ba 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -689,7 +689,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 1a8798fcc..a6539e888 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -56,23 +56,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); + if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit{true}); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); // Backward compatibility: set 'revCount' to 0 for a dirty tree. mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); + input2.getRevCount().value_or(0)); mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); v.attrs->sort(); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 0a1ba49d5..90030ea76 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -59,23 +59,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f9dfb1164..a1ad0a7b9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -13,31 +13,36 @@ namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr input, + const fetchers::Input & input, Value & v) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + // FIXME: support arbitrary input attributes. - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + auto narHash = input.getNarHash(); + assert(narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + narHash->to_string(SRI)); + + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - if (tree.info.lastModified) { - mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified); + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); } v.attrs->sort(); @@ -47,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V { settings.requireExperimentalFeature("flakes"); - std::shared_ptr input; + fetchers::Input input; PathSet context; state.forceValue(*args[0]); @@ -62,7 +67,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); else if (attr.value->type == tInt) attrs.emplace(attr.name, attr.value->integer); else @@ -73,18 +78,42 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (!attrs.count("type")) throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); - input = fetchers::inputFromAttrs(attrs); + input = fetchers::Input::fromAttrs(std::move(attrs)); } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false)); - if (!evalSettings.pureEval && !input->isDirect()) + if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input->isImmutable()) + if (evalSettings.pureEval && !input.isImmutable()) throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (input.hasAllInfo()) { + auto storePath = input.computeStorePath(*state.store); + + try { + state.store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + input.to_string(), state.store->printStorePath(storePath)); + + auto actualPath = state.store->toRealPath(storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(actualPath); + + emitTreeAttrs(state, fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, input, v); + + return; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", input.to_string(), e.what()); + } + } + + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); @@ -137,7 +166,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + if (auto v = std::get_if(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; + if (auto v = std::get_if>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map attrsToQuery(const Attrs & attrs) { std::map query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template struct Explicit { T t; + + bool operator ==(const Explicit & other) const + { + return t == other.t; + } }; -typedef std::variant> Attr; +typedef std::variant> Attr; typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 83268b4bf..e4852d662 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,82 +5,234 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +std::unique_ptr>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr && inputScheme) +void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); + if (!inputSchemes) inputSchemes = std::make_unique>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr inputFromURL(const std::string & url) +Input Input::fromAttrs(Attrs && attrs) { - return inputFromURL(parseURL(url)); -} - -std::unique_ptr inputFromAttrs(const Attrs & attrs) -{ - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - attrs.emplace("type", type()); return attrs; } -std::pair> Input::fetchTree(ref store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair Input::fetch(ref store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto narHash2 = getNarHash()) { + if (narHash != *narHash2) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + // FIXME: check lastModified, revCount + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } -std::shared_ptr Input::applyOverrides( +Input Input::applyOverrides( std::optional ref, std::optional rev) const +{ + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(true, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + return Hash(*s, htSHA256); + return {}; +} + +std::optional Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash(*s, htSHA1); + return {}; +} + +std::optional Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) { if (ref) - throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); if (rev) - throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); - return shared_from_this(); + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b75dcffa5..c43cfe50c 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,89 +12,100 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; }; -struct Input : std::enable_shared_from_this +struct InputScheme; + +struct Input { - std::optional narHash; // FIXME: implement + friend class InputScheme; - virtual std::string type() const = 0; + std::shared_ptr scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; - virtual ~Input() { } +public: + static Input fromURL(const std::string & url); - virtual bool operator ==(const Input & other) const { return false; } + static Input fromURL(const ParsedURL & url); - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ - virtual bool isDirect() const { return true; } + static Input fromAttrs(Attrs && attrs); - /* Check whether this is an "immutable" input, that is, - one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } + ParsedURL toURL() const; - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional getRef() const { return {}; } - - virtual std::optional getRev() const { return {}; } - - virtual ParsedURL toURL() const = 0; - - std::string to_string() const - { - return toURL().to_string(); - } + std::string to_string() const; Attrs toAttrs() const; - std::pair> fetchTree(ref store) const; + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + bool isDirect() const { return direct; } - virtual std::shared_ptr applyOverrides( + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const { return immutable; } + + bool hasAllInfo() const; + + bool operator ==(const Input & other) const; + + bool contains(const Input & other) const; + + std::pair fetch(ref store) const; + + Input applyOverrides( std::optional ref, std::optional rev) const; - virtual std::optional getSourcePath() const { return {}; } + void clone(const Path & destDir) const; - virtual void markChangedFile( + std::optional getSourcePath() const; + + void markChangedFile( std::string_view file, - std::optional commitMsg) const - { assert(false); } + std::optional commitMsg) const; - virtual void clone(const Path & destDir) const - { - throw Error("do not know how to clone input '%s'", to_string()); - } + StorePath computeStorePath(Store & store) const; -private: - - virtual std::pair> fetchTreeInternal(ref store) const = 0; - - virtual Attrs toAttrsInternal() const = 0; + // Convience functions for common attributes. + std::string getType() const; + std::optional getNarHash() const; + std::optional getRef() const; + std::optional getRev() const; + std::optional getRevCount() const; + std::optional getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual std::optional inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; - virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; + virtual ParsedURL toURL(const Input & input); + + virtual bool hasAllInfo(const Input & input) = 0; + + virtual Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev); + + virtual void clone(const Input & input, const Path & destDir); + + virtual std::optional getSourcePath(const Input & input); + + virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + + virtual std::pair fetch(ref store, const Input & input) = 0; }; -std::unique_ptr inputFromURL(const ParsedURL & url); - -std::unique_ptr inputFromURL(const std::string & url); - -std::unique_ptr inputFromAttrs(const Attrs & attrs); - -void registerInputScheme(std::unique_ptr && fetcher); +void registerInputScheme(std::shared_ptr && fetcher); struct DownloadFileResult { @@ -110,7 +120,7 @@ DownloadFileResult downloadFile( const std::string & name, bool immutable); -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c9f9a4b23..4fcf3f542 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - bool shallow = false; - bool submodules = false; - - GitInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "git"; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; - } + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - bool isImmutable() const override - { - return (bool) rev || narHash; - } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; - } - - Attrs toAttrsInternal() const override - { Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - void clone(const Path & destDir) const override + std::optional inputFromAttrs(const Attrs & attrs) override { - auto [isLocal, actualUrl] = getActualUrl(); + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; + } + + bool hasAllInfo(const Input & input) override + { + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); + } + + Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override + { + auto [isLocal, actualUrl] = getActualUrl(input); Strings args = {"clone"}; args.push_back(actualUrl); - if (ref) { + if (auto ref = input.getRef()) { args.push_back("--branch"); args.push_back(*ref); } - if (rev) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); args.push_back(destDir); runProgram("git", true, args); } - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override + std::optional getSourcePath(const Input & input) override { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional getSourcePath() const override - { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); runProgram("git", true, @@ -136,23 +147,25 @@ struct GitInput : Input { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair getActualUrl() const + std::pair getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -163,39 +176,38 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, }, input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -254,35 +266,37 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } - }; + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); - return {std::move(tree), input}; + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input + }; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -290,8 +304,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -305,18 +319,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -339,7 +353,7 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -354,8 +368,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -365,7 +379,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -387,7 +401,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -396,7 +410,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -406,18 +420,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -436,60 +450,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e59c83be4..8d113967e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,37 +8,80 @@ namespace nix::fetchers { -struct GitArchiveInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional ref; - std::optional rev; + virtual std::string type() = 0; - virtual std::shared_ptr _clone() const = 0; - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString>(url.path, "/"); + + std::optional rev; + std::optional ref; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; + if (maybeGetStrAttr(attrs, "type") != type()) return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + throw Error("unsupported input attribute '%s'", name); + + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); + + Input input; + input.attrs = attrs; + return input; } - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; @@ -49,32 +92,44 @@ struct GitArchiveInput : Input }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } - virtual Hash getRevFromRef(nix::ref store, std::string_view ref) const = 0; - - virtual std::string getDownloadUrl() const = 0; - - std::pair> fetchTreeInternal(nix::ref store) const override + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); + auto input(_input); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); + } + if (ref) { + if (input.getRev()) + throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string()); + input.attrs.insert_or_assign("ref", *ref); + } + return input; + } - if (!rev) rev = getRevFromRef(store, ref); + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; - auto input = _clone(); - input->ref = {}; - input->rev = *rev; + virtual std::string getDownloadUrl(const Input & input) const = 0; + + std::pair fetch(ref store, const Input & _input) override + { + Input input(_input); + + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master"); + + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -82,131 +137,44 @@ struct GitArchiveInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { Tree{ .actualPath = store->toRealPath(res->second), .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, }, input }; } - auto url = input->getDownloadUrl(); + auto url = getDownloadUrl(input); + auto [tree, lastModified] = downloadTarball(store, url, "source", true); - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); return {std::move(tree), input}; } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = _clone(); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } }; -struct GitArchiveInputScheme : InputScheme +struct GitHubInputScheme : GitArchiveInputScheme { - std::string type; + std::string type() override { return "github"; } - GitArchiveInputScheme(std::string && type) : type(type) - { } - - virtual std::unique_ptr create() = 0; - - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != type) return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = create(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != type) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported input attribute '%s'", name); - - auto input = create(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -struct GitHubInput : GitArchiveInput -{ - std::string type() const override { return "github"; } - - std::shared_ptr _clone() const override - { return std::make_shared(*this); } - - Hash getRevFromRef(nix::ref store, std::string_view ref) const override + Hash getRevFromRef(nix::ref store, const Input & input) const override { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -216,13 +184,14 @@ struct GitHubInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") @@ -231,35 +200,23 @@ struct GitHubInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override + void clone(const Input & input, const Path & destDir) override { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); + Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; -struct GitHubInputScheme : GitArchiveInputScheme +struct GitLabInputScheme : GitArchiveInputScheme { - GitHubInputScheme() : GitArchiveInputScheme("github") { } + std::string type() override { return "gitlab"; } - std::unique_ptr create() override - { - return std::make_unique(); - } -}; - -struct GitLabInput : GitArchiveInput -{ - std::string type() const override { return "gitlab"; } - - std::shared_ptr _clone() const override - { return std::make_shared(*this); } - - Hash getRevFromRef(nix::ref store, std::string_view ref) const override + Hash getRevFromRef(nix::ref store, const Input & input) const override { auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -269,12 +226,13 @@ struct GitLabInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) std::string accessToken = settings.githubAccessToken.get(); @@ -284,21 +242,12 @@ struct GitLabInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override + void clone(const Input & input, const Path & destDir) override { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitLabInputScheme : GitArchiveInputScheme -{ - GitLabInputScheme() : GitArchiveInputScheme("gitlab") { } - - std::unique_ptr create() override - { - return std::make_unique(); + Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 380b69fe0..91dc83740 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -4,135 +4,99 @@ namespace nix::fetchers { std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -struct IndirectInput : Input -{ - std::string id; - std::optional rev; - std::optional ref; - - std::string type() const override { return "indirect"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && rev == other2->rev - && ref == other2->ref; - } - - bool isDirect() const override - { - return false; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - bool contains(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && (!ref || ref == other2->ref) - && (!rev || rev == other2->rev); - } - - ParsedURL toURL() const override - { - ParsedURL url; - url.scheme = "flake"; - url.path = id; - if (ref) { url.path += '/'; url.path += *ref; }; - if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("id", id); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - throw Error("indirect input '%s' cannot be fetched directly", to_string()); - } -}; - struct IndirectInputScheme : InputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "flake") return nullptr; + if (url.scheme != "flake") return {}; auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); + + std::optional rev; + std::optional ref; if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - input->rev = Hash(path[1], htSHA1); + rev = Hash(path[1], htSHA1); else if (std::regex_match(path[1], refRegex)) - input->ref = path[1]; + ref = path[1]; else throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); - input->ref = path[1]; + ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - input->rev = Hash(path[2], htSHA1); + rev = Hash(path[2], htSHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + // FIXME: forbid query params? - input->id = path[0]; - if (!std::regex_match(input->id, flakeRegex)) - throw BadURL("'%s' is not a valid flake ID", input->id); + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev") + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") throw Error("unsupported indirect input attribute '%s'", name); - auto input = std::make_unique(); - input->id = getStrAttr(attrs, "id"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; return input; } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + return input; + } + + std::pair fetch(ref store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5abb00172..49ed63243 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -10,80 +10,92 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - - MercurialInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "hg"; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; } - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); } - std::shared_ptr applyOverrides( + Input applyOverrides( + const Input & input, std::optional ref, - std::optional rev) const override + std::optional rev) override { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); return res; } - std::optional getSourcePath() const + std::optional getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); // FIXME: shut up if file is already tracked. @@ -95,26 +107,27 @@ struct MercurialInput : Input { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } - std::pair getActualUrl() const + std::pair getActualUrl(const Input & input) const { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -129,7 +142,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -158,53 +171,50 @@ struct MercurialInput : Input } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { Tree{ .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, }, input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -213,10 +223,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -245,9 +255,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -255,18 +265,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -285,54 +295,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 77fe87d59..cbbb0fa02 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,76 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; - - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional rev; - std::optional revCount; - std::optional lastModified; - - std::string type() const override { return "path"; } - - std::optional getRev() const override { return rev; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + if (url.scheme != "path") return {}; + + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); + + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return narHash || rev; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - if (!rev && narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - return attrs; + return true; } - std::optional getSourcePath() const override + std::optional getSourcePath(const Input & input) override { - return path; + return getStrAttr(input.attrs, "path"); } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { + // nothing to do } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & input) override { - auto input = std::make_shared(*this); + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -85,83 +95,13 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - input->narHash = store->queryPathInfo(*storePath)->narHash; - - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique(); - input->path = url.path; - - if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(value); - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(getStrAttr(attrs, "narHash")); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + }, + input + }; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index f6760d2d0..914a0e1e8 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -22,20 +22,7 @@ std::shared_ptr Registry::read( auto version = json.value("version", 0); - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); - } - } - - else if (version == 2) { + if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); Attrs extraAttrs; @@ -47,8 +34,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = inputFromAttrs(jsonToAttrs(i["from"])), - .to = inputFromAttrs(toAttrs), + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -72,8 +59,8 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from->toAttrs()); - obj["to"] = attrsToJson(entry.to->toAttrs()); + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) obj["to"].update(attrsToJson(entry.extraAttrs)); if (entry.exact) @@ -90,8 +77,8 @@ void Registry::write(const Path & path) } void Registry::add( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { entries.emplace_back( @@ -102,11 +89,11 @@ void Registry::add( }); } -void Registry::remove(const std::shared_ptr & input) +void Registry::remove(const Input & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*i->from == *input) + if (i->from == input) i = entries.erase(i); else ++i; @@ -145,8 +132,8 @@ std::shared_ptr getFlagRegistry() } void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); @@ -180,32 +167,33 @@ Registries getRegistries(ref store) return registries; } -std::pair, Attrs> lookupInRegistries( +std::pair lookupInRegistries( ref store, - std::shared_ptr input) + const Input & _input) { Attrs extraAttrs; int n = 0; + Input input(_input); restart: n++; - if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { - if (*entry.from == *input) { + if (entry.from == input) { input = entry.to; extraAttrs = entry.extraAttrs; goto restart; } } else { - if (entry.from->contains(*input)) { - input = entry.to->applyOverrides( - !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional(), - !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional()); + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional()); extraAttrs = entry.extraAttrs; goto restart; } @@ -213,8 +201,8 @@ std::pair, Attrs> lookupInRegistries( } } - if (!input->isDirect()) - throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); return {input, extraAttrs}; } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index c3ce948a8..1077af020 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -20,8 +20,7 @@ struct Registry struct Entry { - std::shared_ptr from; - std::shared_ptr to; + Input from, to; Attrs extraAttrs; bool exact = false; }; @@ -38,11 +37,11 @@ struct Registry void write(const Path & path); void add( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); - void remove(const std::shared_ptr & input); + void remove(const Input & input); }; typedef std::vector> Registries; @@ -54,12 +53,12 @@ Path getUserRegistryPath(); Registries getRegistries(ref store); void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); -std::pair, Attrs> lookupInRegistries( +std::pair lookupInRegistries( ref store, - std::shared_ptr input); + const Input & input); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..624f8b3fb 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -101,7 +101,7 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -116,12 +116,12 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + return { + Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), }, + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable); @@ -156,118 +156,75 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, + return { + Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), }, + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI)); - return attrs; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } + return {}; + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair fetch(ref store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index 42a19cbc8..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,60 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - -TreeInfo TreeInfo::fromJson(const nlohmann::json & json) -{ - TreeInfo info; - - auto i = json.find("info"); - if (i != json.end()) { - const nlohmann::json & i2(*i); - - auto j = i2.find("narHash"); - if (j != i2.end()) - info.narHash = Hash((std::string) *j); - else - throw Error("attribute 'narHash' missing in lock file"); - - j = i2.find("revCount"); - if (j != i2.end()) - info.revCount = *j; - - j = i2.find("lastModified"); - if (j != i2.end()) - info.lastModified = *j; - - return info; - } - - i = json.find("narHash"); - if (i != json.end()) { - info.narHash = Hash((std::string) *i); - return info; - } - - throw Error("attribute 'info' missing in lock file"); -} - -nlohmann::json TreeInfo::toJson() const -{ - nlohmann::json json; - assert(narHash); - json["narHash"] = narHash.to_string(SRI); - if (revCount) - json["revCount"] = *revCount; - if (lastModified) - json["lastModified"] = *lastModified; - return json; -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 3b62151c6..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional revCount; - std::optional lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; - - static TreeInfo fromJson(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 59bb40110..15a3e261a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -62,13 +62,13 @@ static void printFlakeInfo(const Store & store, const Flake & flake) if (flake.description) logger->stdout("Description: %s", *flake.description); logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) logger->stdout("Revision: %s", rev->to_string(Base16, false)); - if (flake.sourceInfo->info.revCount) - logger->stdout("Revisions: %s", *flake.sourceInfo->info.revCount); - if (flake.sourceInfo->info.lastModified) + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->stdout("Revisions: %s", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) logger->stdout("Last modified: %s", - std::put_time(std::localtime(&*flake.sourceInfo->info.lastModified), "%F %T")); + std::put_time(std::localtime(&*lastModified), "%F %T")); } static nlohmann::json flakeToJson(const Store & store, const Flake & flake) @@ -82,13 +82,12 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake) j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); - j["info"] = flake.sourceInfo->info.toJson(); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); - if (flake.sourceInfo->info.revCount) - j["revCount"] = *flake.sourceInfo->info.revCount; - if (flake.sourceInfo->info.lastModified) - j["lastModified"] = *flake.sourceInfo->info.lastModified; + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; j["path"] = store.printStorePath(flake.sourceInfo->storePath); return j; } @@ -172,7 +171,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, lockedNode ? lockedNode->lockedRef : flake.flake.lockedRef); - + if (firstVisit) recurse(*input.second, prefix + (last ? treeNull : treeLine)); } }; @@ -501,7 +500,7 @@ struct CmdFlakeClone : FlakeCommand if (destDir.empty()) throw Error("missing flag '--dest'"); - getFlakeRef().resolve(store).input->clone(destDir); + getFlakeRef().resolve(store).input.clone(destDir); } }; @@ -563,9 +562,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto lockedInput = std::dynamic_pointer_cast(input.second); assert(lockedInput); auto jsonObj3 = jsonObj2 ? jsonObj2->object(input.first) : std::optional(); - if (!dryRun) - lockedInput->lockedRef.input->fetchTree(store); - auto storePath = lockedInput->computeStorePath(*store); + auto storePath = + dryRun + ? lockedInput->lockedRef.input.computeStorePath(*store) + : lockedInput->lockedRef.input.fetch(store).first.storePath; if (jsonObj3) jsonObj3->attr("path", store->printStorePath(storePath)); sources.insert(std::move(storePath)); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index fde1ca7aa..86d3bfd20 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -194,7 +194,7 @@ void EvalCommand::completeFlakeRef(std::string_view prefix) /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(getStore())) { for (auto & entry : registry->entries) { - auto from = entry.from->to_string(); + auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index cc239052d..f39213b8f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -330,7 +330,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); if (element.source - && !element.source->originalRef.input->isImmutable() + && !element.source->originalRef.input.isImmutable() && matches(*store, element, i, matchers)) { Activity act(*logger, lvlChatty, actUnknown, diff --git a/src/nix/registry.cc b/src/nix/registry.cc index e9c76e6c6..16d7e511f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : "global", - entry.from->to_string(), - entry.to->to_string()); + entry.from.to_string(), + entry.to.to_string()); } } } @@ -107,7 +107,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand auto ref = parseFlakeRef(url); auto userRegistry = fetchers::getUserRegistry(); userRegistry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); + auto [tree, resolved] = ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs);