Merge pull request #2907 from NixOS/subdir

Subdirectory improvements

src/libexpr/primops/flake.cc

@@ -121,7 +121,7 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry)
     return json;
 }
 
-void writeLockFile(const LockFile & lockFile, const Path & path)
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
 {
     nlohmann::json json;
     json["version"] = 1;
@@ -133,8 +133,14 @@ void writeLockFile(const LockFile & lockFile, const Path & path)
     json["inputs"] = nlohmann::json::object();
     for (auto & x : lockFile.flakeEntries)
         json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second);
+    stream << json.dump(4); // '4' = indentation in json file
+    return stream;
+}
+
+void writeLockFile(const LockFile & lockFile, const Path & path)
+{
     createDirs(dirOf(path));
-    writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file
+    writeFile(path, fmt("%s\n", lockFile));
 }
 
 Path getUserRegistryPath()
@@ -476,9 +482,12 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc
     Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true));
     LockFile oldLockFile;
 
-    if (!recreateLockFile (handleLockFile)) {
+    if (!recreateLockFile(handleLockFile)) {
         // If recreateLockFile, start with an empty lockfile
-        oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack
+        // FIXME: symlink attack
+        oldLockFile = readLockFile(
+            state.store->toRealPath(flake.sourceInfo.storePath)
+            + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock");
     }
 
     LockFile lockFile(oldLockFile);

src/libexpr/primops/flakeref.cc

@@ -1,4 +1,5 @@
 #include "flakeref.hh"
+#include "store-api.hh"
 
 #include <regex>
 
@@ -30,10 +31,6 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
 const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
 const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
 const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
-// FIXME: support escaping in query string.
-// Note: '/' is not a valid query parameter, but so what...
-const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*";
-const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))";
 
 // 'dir' path elements cannot start with a '.'. We also reject
 // potentially dangerous characters like ';'.
@@ -41,7 +38,7 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
 const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
 
 
-FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
+FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative)
 {
     // FIXME: could combine this into one regex.
 
@@ -50,21 +47,46 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
         std::regex::ECMAScript);
 
     static std::regex githubRegex(
-        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?"
+        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
-        + paramsRegex + "?",
         std::regex::ECMAScript);
 
     static std::regex uriRegex(
         "((" + schemeRegex + "):" +
         "(?://(" + authorityRegex + "))?" +
-        "(" + pathRegex + "))" +
+        "(" + pathRegex + "))",
-        paramsRegex + "?",
         std::regex::ECMAScript);
 
     static std::regex refRegex2(refRegex, std::regex::ECMAScript);
 
     static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript);
 
+    auto [uri, params] = splitUriAndParams(uri_);
+
+    auto handleSubdir = [&](const std::string & name, const std::string & value) {
+        if (name == "dir") {
+            if (value != "" && !std::regex_match(value, subDirRegex2))
+                throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value);
+            subdir = value;
+            return true;
+        } else
+            return false;
+    };
+
+    auto handleGitParams = [&](const std::string & name, const std::string & value) {
+        if (name == "rev") {
+            if (!std::regex_match(value, revRegex))
+                throw BadFlakeRef("invalid Git revision '%s'", value);
+            rev = Hash(value, htSHA1);
+        } else if (name == "ref") {
+            if (!std::regex_match(value, refRegex2))
+                throw BadFlakeRef("invalid Git ref '%s'", value);
+            ref = value;
+        } else if (handleSubdir(name, value))
+            ;
+        else return false;
+        return true;
+    };
+
     std::cmatch match;
     if (std::regex_match(uri.c_str(), match, flakeRegex)) {
         IsAlias d;
@@ -88,17 +110,11 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
         else if (match[4].matched) {
             ref = match[4];
         }
-        for (auto & param : tokenizeString<Strings>(match[5], "&")) {
+        for (auto & param : params) {
-            auto n = param.find('=');
+            if (handleSubdir(param.first, param.second))
-            assert(n != param.npos);
+                ;
-            std::string name(param, 0, n);
+            else
-            std::string value(param, n + 1);
+                throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
-            if (name == "dir") {
-                if (value != "" && !std::regex_match(value, subDirRegex2))
-                    throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
-                subdir = value;
-            } else
-                throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
         }
         data = d;
     }
@@ -108,40 +124,44 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
     {
         IsGit d;
         d.uri = match[1];
-        for (auto & param : tokenizeString<Strings>(match[5], "&")) {
+        for (auto & param : params) {
-            auto n = param.find('=');
+            if (handleGitParams(param.first, param.second))
-            assert(n != param.npos);
+                ;
-            std::string name(param, 0, n);
+            else
-            std::string value(param, n + 1);
-            if (name == "rev") {
-                if (!std::regex_match(value, revRegex))
-                    throw Error("invalid Git revision '%s'", value);
-                rev = Hash(value, htSHA1);
-            } else if (name == "ref") {
-                if (!std::regex_match(value, refRegex2))
-                    throw Error("invalid Git ref '%s'", value);
-                ref = value;
-            } else if (name == "dir") {
-                if (value != "" && !std::regex_match(value, subDirRegex2))
-                    throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
-                subdir = value;
-            } else
                 // FIXME: should probably pass through unknown parameters
-                throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
+                throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
         }
         if (rev && !ref)
-            throw Error("flake URI '%s' lacks a Git ref", uri);
+            throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri);
         data = d;
     }
 
-    else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) {
+    else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == ".")))
+        && uri.find(':') == std::string::npos)
+    {
         IsPath d;
-        d.path = allowRelative ? absPath(uri) : canonPath(uri);
+        if (allowRelative) {
+            d.path = absPath(uri);
+            while (true) {
+                if (pathExists(d.path + "/.git")) break;
+                subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir);
+                d.path = dirOf(d.path);
+                if (d.path == "/")
+                    throw BadFlakeRef("path '%s' does not reference a Git repository", uri);
+            }
+        } else
+            d.path = canonPath(uri);
         data = d;
+        for (auto & param : params) {
+            if (handleGitParams(param.first, param.second))
+                ;
+            else
+                throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
+        }
     }
 
     else
-        throw Error("'%s' is not a valid flake reference", uri);
+        throw BadFlakeRef("'%s' is not a valid flake reference", uri);
 }
 
 std::string FlakeRef::to_string() const
@@ -165,10 +185,10 @@ std::string FlakeRef::to_string() const
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) {
-        assert(subdir == "");
+        string = refData->path;
         if (ref) addParam("ref", *ref);
         if (rev) addParam("rev", rev->gitRev());
-        return refData->path;
+        if (subdir != "") addParam("dir", subdir);
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) {
@@ -217,4 +237,15 @@ FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef.
     result.rev = std::nullopt;
     return result;
 }
+
+std::optional<FlakeRef> parseFlakeRef(
+    const std::string & uri, bool allowRelative)
+{
+    try {
+        return FlakeRef(uri, allowRelative);
+    } catch (BadFlakeRef & e) {
+        return {};
+    }
+}
+
 }

src/libexpr/primops/flakeref.hh

@@ -180,4 +180,9 @@ struct FlakeRef
 
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
+MakeError(BadFlakeRef, Error);
+
+std::optional<FlakeRef> parseFlakeRef(
+    const std::string & uri, bool allowRelative = false);
+
 }

src/libstore/build.cc

@@ -3170,7 +3170,7 @@ void DerivationGoal::registerOutputs()
                    valid. */
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        dest, h.to_string(), h2.to_string()));
+                        dest, h.to_string(SRI), h2.to_string(SRI)));
 
                 Path actualDest = worker.store.toRealPath(dest);
 

src/nix/installables.cc

@@ -314,6 +314,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                         Strings{"packages." + std::string(s, 8)}));
             }
 
+            else if (auto flakeRef = parseFlakeRef(s, true))
+                result.push_back(std::make_shared<InstallableFlake>(*this, std::move(*flakeRef),
+                        getDefaultFlakeAttrPaths()));
+
             else if ((colon = s.rfind(':')) != std::string::npos) {
                 auto flakeRef = std::string(s, 0, colon);
                 auto attrPath = std::string(s, colon + 1);

tests/flakes.sh

@@ -131,9 +131,12 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo
 [[ -e $TEST_ROOT/result/hello ]]
 
 # Test defaultPackage.
-nix build -o $TEST_ROOT/result --flake-registry $registry flake1:
+nix build -o $TEST_ROOT/result --flake-registry $registry flake1
 [[ -e $TEST_ROOT/result/hello ]]
 
+nix build -o $TEST_ROOT/result --flake-registry $registry $flake1Dir
+nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake1Dir
+
 # Building a flake with an unlocked dependency should fail in pure mode.
 (! nix eval "(builtins.getFlake "$flake2Dir")")