StorePath: Rewrite in C++

On nix-env -qa -f '<nixpkgs>', this reduces maximum RSS by 20970 KiB
and runtime by 0.8%. This is mostly because we're not parsing the hash
part as a hash anymore (just validating that it consists of base-32
characters).

Also, replace storePathToHash() by StorePath::hashPart().

Makefile

@@ -1,7 +1,6 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
-  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libutil/tests/local.mk \
   src/libstore/local.mk \

src/libexpr/local.mk

@@ -8,7 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 
 libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
 
-libexpr_LIBS = libutil libstore libfetchers libnixrust
+libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libfetchers/local.mk

@@ -8,4 +8,4 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LIBS = libutil libstore libnixrust
+libfetchers_LIBS = libutil libstore

src/libstore/binary-cache-store.cc

@@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    return storePathToHash(printStorePath(storePath)) + ".narinfo";
+    return std::string(storePath.hashPart()) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(printStorePath(narInfo->path));
+    std::string hashPart(narInfo->path.hashPart());
 
     {
         auto state_(state.lock());
@@ -164,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
             }
         }
 
-        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
     /* Compress the NAR. */

src/libstore/local-store.cc

@@ -619,7 +619,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+        state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
             PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
@@ -791,7 +791,7 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
 
 std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
-    if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
+    if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part");
 
     Path prefix = storeDir + "/" + hashPart;
 
@@ -942,7 +942,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
+        state_->pathInfoCache.erase(std::string(path.hashPart()));
     }
 }
 
@@ -994,7 +994,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             if (info.ca == "" || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
             else
-                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1255,7 +1255,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 if (info->ca == "" || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil libnixrust
+libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)

src/libstore/nar-info-disk-cache.cc

@@ -189,7 +189,7 @@ public:
                 return {oInvalid, 0};
 
             auto namePart = queryNAR.getStr(1);
-            auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart));
+            auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart));
             narInfo->url = queryNAR.getStr(2);
             narInfo->compression = queryNAR.getStr(3);
             if (!queryNAR.isNull(4))
@@ -198,9 +198,9 @@ public:
             narInfo->narHash = Hash(queryNAR.getStr(6));
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
-                narInfo->references.insert(StorePath::fromBaseName(r));
+                narInfo->references.insert(StorePath(r));
             if (!queryNAR.isNull(9))
-                narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9));
+                narInfo->deriver = StorePath(queryNAR.getStr(9));
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
             narInfo->ca = queryNAR.getStr(11);

src/libstore/nar-info.cc

@@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
             auto refs = tokenizeString<Strings>(value, " ");
             if (!references.empty()) corrupt();
             for (auto & r : refs)
-                references.insert(StorePath::fromBaseName(r));
+                references.insert(StorePath(r));
         }
         else if (name == "Deriver") {
             if (value != "unknown-deriver")
-                deriver = StorePath::fromBaseName(value);
+                deriver = StorePath(value);
         }
         else if (name == "System")
             system = value;

src/libstore/path.cc

@@ -2,38 +2,38 @@
 
 namespace nix {
 
-extern "C" {
-    rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir);
-    rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir);
-    rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName);
-    rust::String ffi_StorePath_to_string(const StorePath & _this);
-    StorePath ffi_StorePath_clone(const StorePath & _this);
-    rust::StringSlice ffi_StorePath_name(const StorePath & _this);
+MakeError(BadStorePath, Error);
+
+static void checkName(std::string_view path, std::string_view name)
+{
+    if (name.empty())
+        throw BadStorePath("store path '%s' has an empty name", path);
+    if (name.size() > 211)
+        throw BadStorePath("store path '%s' has a name longer than 211 characters", path);
+    for (auto c : name)
+        if (!((c >= '0' && c <= '9')
+                || (c >= 'a' && c <= 'z')
+                || (c >= 'A' && c <= 'Z')
+                || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '='))
+            throw BadStorePath("store path '%s' contains illegal character '%s'", path, c);
 }
 
-StorePath StorePath::make(std::string_view path, std::string_view storeDir)
+StorePath::StorePath(std::string_view _baseName)
+    : baseName(_baseName)
 {
-    return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap();
+    if (baseName.size() < HashLen + 1)
+        throw BadStorePath("'%s' is too short to be a valid store path", baseName);
+    for (auto c : hashPart())
+        if (c == 'e' || c == 'o' || c == 'u' || c == 't'
+            || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')))
+            throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c);
+    checkName(baseName, name());
 }
 
-StorePath StorePath::make(unsigned char hash[20], std::string_view name)
+StorePath::StorePath(const Hash & hash, std::string_view _name)
+    : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name)))
 {
-    return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap();
-}
-
-StorePath StorePath::fromBaseName(std::string_view baseName)
-{
-    return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap();
-}
-
-rust::String StorePath::to_string() const
-{
-    return ffi_StorePath_to_string(*this);
-}
-
-StorePath StorePath::clone() const
-{
-    return ffi_StorePath_clone(*this);
+    checkName(baseName, name());
 }
 
 bool StorePath::isDerivation() const
@@ -41,18 +41,14 @@ bool StorePath::isDerivation() const
     return hasSuffix(name(), drvExtension);
 }
 
-std::string_view StorePath::name() const
-{
-    return ffi_StorePath_name(*this);
-}
-
-StorePath StorePath::dummy(
-    StorePath::make(
-        (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x"));
+StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath Store::parseStorePath(std::string_view path) const
 {
-    return StorePath::make(path, storeDir);
+    auto p = canonPath(std::string(path));
+    if (dirOf(p) != storeDir)
+        throw BadStorePath("path '%s' is not in the Nix store", p);
+    return StorePath(baseNameOf(p));
 }
 
 std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
@@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 
 std::string Store::printStorePath(const StorePath & path) const
 {
-    auto s = storeDir + "/";
-    s += (std::string_view) path.to_string();
-    return s;
+    return (storeDir + "/").append(path.to_string());
 }
 
 PathSet Store::printStorePathSet(const StorePathSet & paths) const

src/libstore/path.hh

@@ -1,59 +1,78 @@
 #pragma once
 
-#include "rust-ffi.hh"
+#include "types.hh"
 
 namespace nix {
 
-/* See path.rs. */
-struct StorePath;
-
 class Store;
+struct Hash;
 
-extern "C" {
-    void ffi_StorePath_drop(void *);
-    bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
-    bool ffi_StorePath_eq(const StorePath & a, const StorePath & b);
-    unsigned char * ffi_StorePath_hash_data(const StorePath & p);
-}
-
-struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
+class StorePath
 {
+    std::string baseName;
+
+    StorePath(const StorePath & path)
+        : baseName(path.baseName)
+    { }
+
+public:
+
+    /* Size of the hash part of store paths, in base-32 characters. */
+    constexpr static size_t HashLen = 32; // i.e. 160 bits
+
     StorePath() = delete;
 
-    static StorePath make(std::string_view path, std::string_view storeDir);
+    StorePath(std::string_view baseName);
 
-    static StorePath make(unsigned char hash[20], std::string_view name);
+    StorePath(const Hash & hash, std::string_view name);
 
-    static StorePath fromBaseName(std::string_view baseName);
+    StorePath(StorePath && path)
+        : baseName(std::move(path.baseName))
+    { }
 
-    rust::String to_string() const;
+    StorePath & operator = (StorePath && path)
+    {
+        baseName = std::move(path.baseName);
+        return *this;
+    }
+
+    std::string_view to_string() const
+    {
+        return baseName;
+    }
 
     bool operator < (const StorePath & other) const
     {
-        return ffi_StorePath_less_than(*this, other);
+        return baseName < other.baseName;
     }
 
     bool operator == (const StorePath & other) const
     {
-        return ffi_StorePath_eq(*this, other);
+        return baseName == other.baseName;
     }
 
     bool operator != (const StorePath & other) const
     {
-        return !(*this == other);
+        return baseName != other.baseName;
     }
 
-    StorePath clone() const;
+    StorePath clone() const
+    {
+        return StorePath(*this);
+    }
 
     /* Check whether a file name ends with the extension for
        derivations. */
     bool isDerivation() const;
 
-    std::string_view name() const;
-
-    unsigned char * hashData() const
+    std::string_view name() const
     {
-        return ffi_StorePath_hash_data(*this);
+        return std::string_view(baseName).substr(HashLen + 1);
+    }
+
+    std::string_view hashPart() const
+    {
+        return std::string_view(baseName).substr(0, HashLen);
     }
 
     static StorePath dummy;
@@ -67,9 +86,6 @@ StorePathSet storePathsToSet(const StorePaths & paths);
 
 StorePathSet singleton(const StorePath & path);
 
-/* Size of the hash part of store paths, in base-32 characters. */
-const size_t storePathHashLen = 32; // i.e. 160 bits
-
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
@@ -107,7 +123,7 @@ namespace std {
 template<> struct hash<nix::StorePath> {
     std::size_t operator()(const nix::StorePath & path) const noexcept
     {
-        return * (std::size_t *) path.hashData();
+        return * (std::size_t *) path.to_string().data();
     }
 };
 

src/libstore/remote-fs-accessor.cc

@@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
 Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
+    return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
 }
 
 void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,

src/libstore/store-api.cc

@@ -59,14 +59,6 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
 }
 
 
-string storePathToHash(const Path & path)
-{
-    auto base = baseNameOf(path);
-    assert(base.size() >= storePathHashLen);
-    return string(base, 0, storePathHashLen);
-}
-
-
 /* Store paths have the following form:
 
    <store>/<h>-<name>
@@ -144,7 +136,7 @@ StorePath Store::makeStorePath(const string & type,
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
     string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
     auto h = compressHash(hashString(htSHA256, s), 20);
-    return StorePath::make(h.hash, name);
+    return StorePath(h, name);
 }
 
 
@@ -243,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
 
 bool Store::isValidPath(const StorePath & storePath)
 {
-    auto hashPart = storePathToHash(printStorePath(storePath));
+    std::string hashPart(storePath.hashPart());
 
     {
         auto state_(state.lock());
@@ -311,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath,
     std::string hashPart;
 
     try {
-        hashPart = storePathToHash(printStorePath(storePath));
+        hashPart = storePath.hashPart();
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);

src/libstore/store-api.hh

@@ -731,10 +731,6 @@ public:
 };
 
 
-/* Extract the hash part of the given store path. */
-string storePathToHash(const Path & path);
-
-
 /* Copy a path from one store to another. */
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);

src/libutil/local.mk

@@ -7,5 +7,3 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
-
-libutil_LIBS = libnixrust

src/libutil/rust-ffi.cc

@@ -1,3 +1,4 @@
+#if 0
 #include "logging.hh"
 #include "rust-ffi.hh"
 
@@ -20,3 +21,4 @@ std::ostream & operator << (std::ostream & str, const String & s)
 }
 
 }
+#endif

src/libutil/rust-ffi.hh

@@ -1,4 +1,5 @@
 #pragma once
+#if 0
 
 #include "serialise.hh"
 
@@ -185,3 +186,4 @@ struct Result
 };
 
 }
+#endif

src/nix/local.mk

@@ -17,7 +17,7 @@ nix_SOURCES := \
 
 nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
 
-nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust
+nix_LIBS = libexpr libmain libfetchers libstore libutil
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 

src/nix/make-content-addressable.cc

@@ -48,7 +48,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
         for (auto & path : paths) {
             auto pathS = store->printStorePath(path);
             auto oldInfo = store->queryPathInfo(path);
-            auto oldHashPart = storePathToHash(pathS);
+            std::string oldHashPart(path.hashPart());
 
             StringSink sink;
             store->narFromPath(path, sink);
@@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
                 printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
 
             auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink);
+                RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
                 rsink2((unsigned char *) sink.s->data(), sink.s->size());
                 rsink2.flush();
             });

src/nix/verify.cc

@@ -90,7 +90,7 @@ struct CmdVerify : StorePathsCommand
                     if (info->ca == "")
                         hashSink = std::make_unique<HashSink>(info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(store->printStorePath(info->path)));
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                     store->narFromPath(info->path, *hashSink);
 

src/nix/why-depends.cc

@@ -76,7 +76,7 @@ struct CmdWhyDepends : SourceExprCommand
         auto packagePath = toStorePath(store, Build, package);
         auto dependency = parseInstallable(*this, store, _dependency, false);
         auto dependencyPath = toStorePath(store, NoBuild, dependency);
-        auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath));
+        auto dependencyPathHash = dependencyPath.hashPart();
 
         StorePathSet closure;
         store->computeFSClosure({packagePath}, closure, false, false);
@@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand
                 auto & node2 = graph.at(ref);
                 if (node2.dist == inf) continue;
                 refs.emplace(node2.dist, &node2);
-                hashes.insert(storePathToHash(store->printStorePath(node2.path)));
+                hashes.insert(std::string(node2.path.hashPart()));
             }
 
             /* For each reference, find the files and symlinks that
@@ -211,7 +211,7 @@ struct CmdWhyDepends : SourceExprCommand
                                     p2,
                                     hilite(filterPrintable(
                                             std::string(contents, pos2, pos - pos2 + hash.size() + margin)),
-                                        pos - pos2, storePathHashLen,
+                                        pos - pos2, StorePath::HashLen,
                                         getColour(hash))));
                         }
                     }
@@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand
                         auto pos = target.find(hash);
                         if (pos != std::string::npos)
                             hits[hash].emplace_back(fmt("%s -> %s\n", p2,
-                                    hilite(target, pos, storePathHashLen, getColour(hash))));
+                                    hilite(target, pos, StorePath::HashLen, getColour(hash))));
                     }
                 }
             };
@@ -235,7 +235,7 @@ struct CmdWhyDepends : SourceExprCommand
 
             RunPager pager;
             for (auto & ref : refs) {
-                auto hash = storePathToHash(store->printStorePath(ref.second->path));
+                std::string hash(ref.second->path.hashPart());
 
                 bool last = all ? ref == *refs.rbegin() : true;
 

src/resolve-system-dependencies/local.mk

@@ -8,6 +8,6 @@ resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix
 
 resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain
 
-resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust
+resolve-system-dependencies_LIBS := libstore libmain libutil
 
 resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc