From cf198952d02aae5585e9bb895577b5b4e7b25707 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Jun 2016 15:15:21 +0200 Subject: [PATCH] HttpBinaryCacheStore: Fix caching of WantMassQuery Also, test HttpBinaryCacheStore in addition to LocalBinaryCacheStore. --- src/libstore/http-binary-cache-store.cc | 6 ++- src/libstore/local-binary-cache-store.cc | 10 ++--- src/libstore/nar-info-disk-cache.cc | 26 +++++++----- src/libstore/nar-info-disk-cache.hh | 3 +- src/libstore/s3-binary-cache-store.cc | 2 +- tests/binary-cache.sh | 52 ++++++++++++++++-------- 6 files changed, 60 insertions(+), 39 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9587ac547..da80b636c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -39,7 +39,7 @@ public: void init() override { // FIXME: do this lazily? - if (!diskCache->cacheExists(cacheUri)) { + if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { @@ -95,7 +95,9 @@ static RegisterStoreImplementation regStore([]( -> std::shared_ptr { if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://") return 0; + std::string(uri, 0, 8) != "https://" && + (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") + ) return 0; auto store = std::make_shared(params, uri); store->init(); return store; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index bdc80cf90..91d2650fe 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -17,9 +17,6 @@ public: : BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { - /* For testing the NAR info cache. */ - if (getEnv("_NIX_CACHE_FILE_URLS") == "1") - diskCache = getNarInfoDiskCache(); } void init() override; @@ -57,9 +54,6 @@ void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); BinaryCacheStore::init(); - - if (diskCache && !diskCache->cacheExists(getUri())) - diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); } static void atomicWrite(const Path & path, const std::string & s) @@ -96,7 +90,9 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - if (std::string(uri, 0, 7) != "file://") return 0; + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || + std::string(uri, 0, 7) != "file://") + return 0; auto store = std::make_shared(params, std::string(uri, 7)); store->init(); return store; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index b05f2661e..0751dda19 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -57,6 +57,8 @@ public: { int id; Path storeDir; + bool wantMassQuery; + int priority; }; struct State @@ -126,24 +128,28 @@ public: state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir}; + state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; } - bool cacheExists(const std::string & uri) override + bool cacheExists(const std::string & uri, + bool & wantMassQuery, int & priority) override { auto state(_state.lock()); auto i = state->caches.find(uri); - if (i != state->caches.end()) return true; - - auto queryCache(state->queryCache.use()(uri)); - - if (queryCache.next()) { - state->caches[uri] = Cache{(int) queryCache.getInt(0), queryCache.getStr(1)}; - return true; + if (i == state->caches.end()) { + auto queryCache(state->queryCache.use()(uri)); + if (!queryCache.next()) return false; + state->caches.emplace(uri, + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2), (int) queryCache.getInt(3)}); } - return false; + auto & cache(getCache(*state, uri)); + + wantMassQuery = cache.wantMassQuery; + priority = cache.priority; + + return true; } std::pair> lookupNarInfo( diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index f86d720a9..88d909732 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -13,7 +13,8 @@ public: virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; - virtual bool cacheExists(const std::string & uri) = 0; + virtual bool cacheExists(const std::string & uri, + bool & wantMassQuery, int & priority) = 0; virtual std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 33cc9659a..ed95620bb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -71,7 +71,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore void init() override { - if (!diskCache->cacheExists(getUri())) { + if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { /* Create the bucket if it doesn't already exists. */ // FIXME: HeadBucket would be more appropriate, but doesn't return diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 48bc62d31..96cab6ad4 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -9,34 +9,50 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix-push --dest $cacheDir $outPath -# By default, a binary cache doesn't support "nix-env -qas", but does -# support installation. -clearStore -clearCacheCache +basicTests() { -export _NIX_CACHE_FILE_URLS=1 + # By default, a binary cache doesn't support "nix-env -qas", but does + # support installation. + clearStore + clearCacheCache -nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" + nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" -nix-store --option binary-caches "file://$cacheDir" -r $outPath + nix-store --option binary-caches "file://$cacheDir" -r $outPath -[ -x $outPath/program ] + [ -x $outPath/program ] -# But with the right configuration, "nix-env -qas" should also work. -clearStore -clearCacheCache -echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + # But with the right configuration, "nix-env -qas" should also work. + clearStore + clearCacheCache + echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info -nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" -x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) -[ -z "$x" ] + x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) + [ -z "$x" ] -nix-store --option binary-caches "file://$cacheDir" -r $outPath + nix-store --option binary-caches "file://$cacheDir" -r $outPath -nix-store --check-validity $outPath -nix-store -qR $outPath | grep input-2 + nix-store --check-validity $outPath + nix-store -qR $outPath | grep input-2 + + echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info +} + + +# Test LocalBinaryCacheStore. +basicTests + + +# Test HttpBinaryCacheStore. +export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +basicTests + + +unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.