From 9ccbd55c5b55b5530e61fd20476d9f20fd45e074 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2016 16:52:28 +0100 Subject: [PATCH] BinaryCacheStore: Implement addToStore() So now you can do $ NIX_REMOTE=file:///tmp/binary-cache nix-instantiate '' -A hello and lots of other operations. --- src/libstore/binary-cache-store.cc | 50 ++++++++++++++++++++++++++++++ src/libstore/binary-cache-store.hh | 16 +++++----- src/libutil/archive.cc | 15 ++++++--- src/libutil/archive.hh | 5 +++ 4 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6d40c70a3..dc086fe9c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -24,6 +24,10 @@ BinaryCacheStore::BinaryCacheStore(std::shared_ptr localStore, auto key = PublicKey(readFile(publicKeyFile)); publicKeys->emplace(key.name, key); } + + StringSink sink; + sink << narVersionMagic1; + narMagic = sink.s; } void BinaryCacheStore::init() @@ -55,6 +59,8 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; + assert(nar.compare(0, narMagic.size(), narMagic) == 0); + auto narInfo = make_ref(info); narInfo->narSize = nar.size(); @@ -261,6 +267,50 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, localStore->querySubstitutablePathInfos(left, infos); } +Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); + + if (repair || !isValidPath(info.path)) + addToCache(info, sink.s); + + return info.path; +} + +Path BinaryCacheStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + addToCache(info, sink.s); + } + + return info.path; +} + void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) { for (auto & storePath : paths) { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 0b343b357..2235d6d67 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -68,6 +68,8 @@ private: Stats stats; + std::string narMagic; + std::string narInfoFileFor(const Path & storePath); void addToCache(const ValidPathInfo & info, const string & nar); @@ -96,10 +98,10 @@ public: { notImpl(); } Path queryDeriver(const Path & path) override - { notImpl(); } + { return ""; } PathSet queryValidDerivers(const Path & path) override - { notImpl(); } + { return {}; } PathSet queryDerivationOutputs(const Path & path) override { notImpl(); } @@ -111,19 +113,17 @@ public: { notImpl(); } PathSet querySubstitutablePaths(const PathSet & paths) override - { notImpl(); } + { return {}; } void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) override - { notImpl(); } + PathFilter & filter = defaultPathFilter, bool repair = false) override; Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) override - { notImpl(); } + const PathSet & references, bool repair = false) override; void exportPath(const Path & path, bool sign, Sink & sink) override; @@ -156,7 +156,7 @@ public: { notImpl(); } PathSet queryFailedPaths() override - { return PathSet(); } + { return {}; } void clearFailedPaths(const PathSet & paths) override { } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6ee798143..5363496c2 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -29,7 +29,7 @@ bool useCaseHack = false; #endif -static string archiveVersion1 = "nix-archive-1"; +const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; @@ -113,11 +113,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) void dumpPath(const Path & path, Sink & sink, PathFilter & filter) { - sink << archiveVersion1; + sink << narVersionMagic1; dump(path, sink, filter); } +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + static SerialisationError badArchive(string s) { return SerialisationError("bad archive: " + s); @@ -214,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "executable" && type == tpRegular) { - readString(source); + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); sink.isExecutable(); } @@ -275,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source) /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ } - if (version != archiveVersion1) + if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); parse(sink, source, ""); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index c216e9768..90117f5ff 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -55,6 +55,8 @@ extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void dumpString(const std::string & s, Sink & sink); + struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -76,4 +78,7 @@ void restorePath(const Path & path, Source & source); extern bool useCaseHack; +extern const std::string narVersionMagic1; + + }