privatevoid.net/depot
1
1
Fork 0
Code Issues 23 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

Merge pull request #70 from privatevoid-net/pr-flake-update

Browse source 
`flake.lock`: Update
This commit is contained in:
hercules-ci[bot] 2023-03-31 15:54:28 +00:00 committed by GitHub
commit f3df93e78e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 38 additions and 342 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cluster/services/hercules-ci-multi-agent/orgs/max.nix
View file

@ -2,7 +2,6 @@
{
services.hercules-ci-agents.max = {
package = lib.mkForce depot.packages.hercules-ci-agent;
settings = {
clusterJoinTokenPath = config.age.secrets.hci-token-max.path;
binaryCachesPath = config.age.secrets.hci-cache-config-max.path;

1
cluster/services/hercules-ci-multi-agent/orgs/nixpak.nix
View file

@ -2,7 +2,6 @@
{
services.hercules-ci-agents.nixpak = {
package = lib.mkForce depot.packages.hercules-ci-agent;
settings = {
clusterJoinTokenPath = config.age.secrets.hci-token-nixpak.path;
binaryCachesPath = config.age.secrets.hci-cache-config-nixpak.path;

1
cluster/services/hercules-ci-multi-agent/orgs/private-void.nix
View file

@ -7,7 +7,6 @@
group = "hci-private-void";
};
services.hercules-ci-agents.private-void = {
package = lib.mkForce depot.packages.hercules-ci-agent;
settings = {
clusterJoinTokenPath = config.age.secrets.hci-token-private-void.path;
binaryCachesPath = config.age.secrets.hci-cache-config-private-void.path;

54
flake.lock
View file

@ -175,11 +175,11 @@
]
},
"locked": {
"lastModified": 1679549291,
"narHash": "sha256-I3kKsVZ6iuLpvSsR0ikKYMBjAMfnuQbeqt2rvHJRulc=",
"lastModified": 1680172861,
"narHash": "sha256-QMyI338xRxaHFDlCXdLCtgelGQX2PdlagZALky4ZXJ8=",
"owner": "DavHau",
"repo": "drv-parts",
"rev": "025af1da98fd81546bf2f81b1902d1df0b0e5f0c",
"rev": "ced8a52f62b0a94244713df2225c05c85b416110",
"type": "github"
},
"original": {
@ -227,11 +227,11 @@
]
},
"locked": {
"lastModified": 1678379998,
"narHash": "sha256-TZdfNqftHhDuIFwBcN9MUThx5sQXCTeZk9je5byPKRw=",
"lastModified": 1679737941,
"narHash": "sha256-srSD9CwsVPnUMsIZ7Kt/UegkKUEBcTyU1Rev7mO45S0=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "c13d60b89adea3dc20704c045ec4d50dd964d447",
"rev": "3502ee99d6dade045bdeaf7b0cd8ec703484c25c",
"type": "github"
},
"original": {
@ -302,17 +302,15 @@
"nix-darwin": [
"blank"
],
"nixpkgs": [
"nixpkgs"
],
"nixpkgs": "nixpkgs",
"pre-commit-hooks-nix": "pre-commit-hooks-nix"
},
"locked": {
"lastModified": 1679065768,
"narHash": "sha256-VYzLS/51fPx8wId7QeLJWHC9oL4QRbOJwu96G4wd/qA=",
"lastModified": 1680111004,
"narHash": "sha256-ZQogvVfGuVGhBsEbbaJ2tx/0khhMuvt8LnEU4fFC/qw=",
"owner": "hercules-ci",
"repo": "hercules-ci-agent",
"rev": "3a118a985ce57c2a0d8ce567b1f007d57933c540",
"rev": "c47745ed440c80d35abae09b96b17009db2e6648",
"type": "github"
},
"original": {
@ -334,11 +332,11 @@
]
},
"locked": {
"lastModified": 1678923240,
"narHash": "sha256-qoa7OvbboojbwZnCaL5+r2pFNhLkxRUsD9+RaoM9zjw=",
"lastModified": 1680111600,
"narHash": "sha256-qWYXr9gOeU1BaFgZOIDA5tcCWiZTTOVZRHTEl1Znb5w=",
"owner": "hercules-ci",
"repo": "hercules-ci-effects",
"rev": "0eec846f63e69f935ae1c06280b3b444f98d95e5",
"rev": "6c1ee352e9c65302969fc510828a98bea38500f6",
"type": "github"
},
"original": {
@ -459,16 +457,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1679577639,
"narHash": "sha256-7u7bsNP0ApBnLgsHVROQ5ytoMqustmMVMgtaFS/P7EU=",
"lastModified": 1680125544,
"narHash": "sha256-mlqo1r+TZUOuypWdrZHluxWL+E5WzXlUXNZ9Y0WLDFU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "8f1bcd72727c5d4cd775545595d068be410f2a7e",
"rev": "9a6aabc4740790ef3bbb246b86d029ccf6759658",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11-small",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
@ -489,6 +487,22 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1680183455,
"narHash": "sha256-lhlRyEZE1CZ2B5h4goEbS3Env7G6ORojj0/+AAGOooM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "646930ff24273c6d91fb3ea37531261553043b50",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11-small",
"repo": "nixpkgs",
"type": "github"
}
},
"poetry2nix": {
"flake": false,
"locked": {
@ -561,7 +575,7 @@
"nar-serve": "nar-serve",
"nix-filter": "nix-filter",
"nix-super": "nix-super",
"nixpkgs": "nixpkgs",
"nixpkgs": "nixpkgs_2",
"repin-flake-utils": "repin-flake-utils"
}
}

1
flake.nix
View file

@ -94,7 +94,6 @@
hercules-ci-agent = {
url = "github:hercules-ci/hercules-ci-agent";
inputs = {
nixpkgs.follows = "nixpkgs";
flake-parts.follows = "flake-parts";
nix-darwin.follows = "blank";
};

2
packages/build-support/drv-parts/backends/buildPythonPackage/implementation.nix
View file

@ -1,4 +1,4 @@
{ config, dependencySets, drv-parts, lib, ... }:
{ config, packageSets, drv-parts, lib, ... }:
let
inherit (config) deps;

2
packages/build-support/drv-parts/dependency-sets/default.nix
View file

@ -1,7 +1,7 @@
{ pkgs, inputs', self', ... }:
{
drv-parts.dependencySets = {
drv-parts.packageSets = {
inherit pkgs inputs' self';
inherit (pkgs) python3Packages;
};

45
packages/patched-inputs.nix
View file

@ -14,51 +14,6 @@
agenix = packages.agenix.agenix.override { nix = nix-super; };
# hci-agent's build code does some funny shenanigans
hercules-ci-agent = let
original = packages.hercules-ci-agent.hercules-ci-agent;
patchedNix = (patch original.nix "patches/extra/hercules-ci-agent/nix").overrideAttrs (old: rec {
name = "nix-${version}";
version = "${original.nix.version}_hci2";
postUnpack = ''
${old.postUnpack or ""}
echo -n "${version}" > .version
'';
});
forcePatchNix = old: {
buildInputs = (lib.remove original.nix old.buildInputs) ++ [ patchedNix ];
passthru = old.passthru // {
nix = patchedNix;
};
};
patchDeps = lib.const rec {
hercules-ci-cnix-store = packages.hercules-ci-agent.internal-hercules-ci-cnix-store.override (lib.const {
nix = patchedNix;
});
hercules-ci-cnix-expr = packages.hercules-ci-agent.internal-hercules-ci-cnix-expr.override (lib.const {
nix = patchedNix;
inherit hercules-ci-cnix-store;
});
cachix = (pkgs.haskellPackages.cachix.override (lib.const {
nix = patchedNix;
inherit hercules-ci-cnix-store;
})).overrideAttrs (o: {
postPatch = ''
${o.postPatch or ""}
# jailbreak pkgconfig deps
cp cachix.cabal cachix.cabal.backup
sed -i cachix.cabal -e 's/\(nix-[a-z]*\) *(==[0-9.]* *|| *>[0-9.]*) *&& *<[0-9.]*/\1/g'
sed -i cachix.cabal -e 's/pkgconfig-depends:.*/pkgconfig-depends: nix-main, nix-store/'
echo
echo Applied:
diff -U5 cachix.cabal.backup cachix.cabal ||:
echo
rm cachix.cabal.backup
'';
});
};
in (original.override patchDeps).overrideAttrs forcePatchNix;
hci = packages.hercules-ci-agent.hercules-ci-cli;
};
};

4
packages/servers/reflex-cache/project.nix
View file

@ -2,7 +2,7 @@
{
perSystem = { config, drv-backends, lib, pkgs, ... }: let
deps = with config.drv-parts.dependencySets.python3Packages; [
deps = with config.drv-parts.packageSets.python3Packages; [
poetry-core
requests-unixsocket
py-multibase
@ -19,7 +19,7 @@
env.PYTHON = pythonForDev.interpreter;
commands.reflex.command = "${pythonForDev.interpreter} -m reflex_cache.main";
};
drvs.reflex-cache = { dependencySets, ... }: {
drvs.reflex-cache = { packageSets, ... }: {
imports = [
drv-backends.buildPythonPackage
];

70
patches/extra/hercules-ci-agent/nix/GHPR_7489-PDT-TDE-3114-prevent-a-race-condition-when-creating-the-S3-cache.patch
View file

@ -1,70 +0,0 @@
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index f4ea739b05d..3e0689534b6 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -166,16 +166,37 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
return i->second;
}
+ std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
+ {
+ auto i = state.caches.find(uri);
+ if (i == state.caches.end()) {
+ auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
+ if (!queryCache.next())
+ return std::nullopt;
+ state.caches.emplace(uri,
+ Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+ }
+ return getCache(state, uri);
+ }
+
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
+ SQLiteTxn txn(state->db);
+
+ // To avoid the race, we have to check if maybe someone hasn't yet created
+ // the cache for this URI in the meantime.
+ auto cache(queryCacheRaw(*state, uri));
- // FIXME: race
+ if (cache)
+ return;
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+
+ txn.commit();
});
}
@@ -183,21 +204,12 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
{
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
auto state(_state.lock());
-
- auto i = state->caches.find(uri);
- if (i == state->caches.end()) {
- auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl));
- if (!queryCache.next())
- return std::nullopt;
- state->caches.emplace(uri,
- Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
- }
-
- auto & cache(getCache(*state, uri));
-
+ auto cache(queryCacheRaw(*state, uri));
+ if (!cache)
+ return std::nullopt;
return CacheInfo {
- .wantMassQuery = cache.wantMassQuery,
- .priority = cache.priority
+ .wantMassQuery = cache->wantMassQuery,
+ .priority = cache->priority
};
});
}

199
patches/extra/hercules-ci-agent/nix/GHPR_7616-Fix-foreign-key-error-inserting-into-NARs.patch
View file

@ -1,199 +0,0 @@
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 73bcd6e817d..1479822a9f7 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -56,7 +56,7 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v
void init() override
{
// FIXME: do this lazily?
- if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
+ if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
} else {
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 3e0689534b6..8c7d8126f3c 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -68,15 +68,25 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
struct Cache
{
int id;
+ // column: timestamp
+ time_t lastCacheInfoFetch;
Path storeDir;
bool wantMassQuery;
int priority;
+
+ bool dbRowUpToDate(const Cache &other) {
+ assert(id == other.id);
+ return lastCacheInfoFetch == other.lastCacheInfoFetch
+ && storeDir == other.storeDir
+ && wantMassQuery == other.wantMassQuery
+ && priority == other.priority;
+ }
};
struct State
{
SQLite db;
- SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR,
+ SQLiteStmt insertCache, updateCache, queryCache, insertNAR, insertMissingNAR,
queryNAR, insertRealisation, insertMissingRealisation,
queryRealisation, purgeCache;
std::map<std::string, Cache> caches;
@@ -98,10 +108,13 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
state->db.exec(schema);
state->insertCache.create(state->db,
- "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+ "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+
+ state->updateCache.create(state->db,
+ "update BinaryCaches set url = ?, timestamp = ?, storeDir = ?, wantMassQuery = ?, priority = ? WHERE id = ?");
state->queryCache.create(state->db,
- "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
+ "select id, timestamp, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
state->insertNAR.create(state->db,
"insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
@@ -166,19 +179,33 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
return i->second;
}
+private:
+
+ bool cacheIsUpToDate(const Cache &cache) {
+ return cache.lastCacheInfoFetch > (time(0) - cacheInfoTtl);
+ }
+
+ // May return an outdated record; check cacheIsUpToDate()!
std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
{
auto i = state.caches.find(uri);
if (i == state.caches.end()) {
- auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
+ auto queryCache(state.queryCache.use()(uri));
if (!queryCache.next())
return std::nullopt;
- state.caches.emplace(uri,
- Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+ auto cache = Cache {
+ .id = (int) queryCache.getInt(0),
+ .lastCacheInfoFetch = (time_t) queryCache.getInt(1),
+ .storeDir = queryCache.getStr(2),
+ .wantMassQuery = queryCache.getInt(3) != 0,
+ .priority = (int) queryCache.getInt(4),
+ };
+ state.caches.emplace(uri, cache);
}
return getCache(state, uri);
}
+public:
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
{
retrySQLite<void>([&]() {
@@ -189,23 +216,42 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
// the cache for this URI in the meantime.
auto cache(queryCacheRaw(*state, uri));
- if (cache)
+ if (cache && cacheIsUpToDate(*cache))
return;
- state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
- assert(sqlite3_changes(state->db) == 1);
- state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+ Cache ret {
+ // id: to be set in the conditional below
+ .id = -1,
+ // lastCacheInfoFetch: always the current time, because this is
+ // only called when a cached Cache wasn't available, so the parameters
+ // are always fresh.
+ .lastCacheInfoFetch = time(0),
+ .storeDir = storeDir,
+ .wantMassQuery = wantMassQuery,
+ .priority = priority,
+ };
+ if (cache) {
+ ret.id = cache->id;
+ state->updateCache.use()(uri)(ret.lastCacheInfoFetch)(storeDir)(wantMassQuery)(priority)(ret.id).exec();
+ assert(sqlite3_changes(state->db) == 1);
+ } else {
+ state->insertCache.use()(uri)(ret.lastCacheInfoFetch)(storeDir)(wantMassQuery)(priority).exec();
+ assert(sqlite3_changes(state->db) == 1);
+ ret.id = (int) state->db.getLastInsertedRowId();
+ }
+
+ state->caches[uri] = ret;
txn.commit();
});
}
- std::optional<CacheInfo> cacheExists(const std::string & uri) override
+ std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override
{
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
auto state(_state.lock());
auto cache(queryCacheRaw(*state, uri));
- if (!cache)
+ if (!cache || !cacheIsUpToDate(*cache))
return std::nullopt;
return CacheInfo {
.wantMassQuery = cache->wantMassQuery,
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index 2dcaa76a490..c185ca5e4f0 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -22,7 +22,7 @@ public:
int priority;
};
- virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
+ virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 844553ad309..8d76eee9977 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
void init() override
{
- if (auto cacheInfo = diskCache->cacheExists(getUri())) {
+ if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
} else {
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 353dff9fa02..a6fb53b1d0c 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -41,6 +41,10 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex
throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf));
}
+static void traceSQL (void *x, const char *sql){
+ fprintf(stderr, "SQL<[%s]>\n", sql);
+};
+
SQLite::SQLite(const Path & path, bool create)
{
// useSQLiteWAL also indicates what virtual file system we need. Using
@@ -58,6 +62,11 @@ SQLite::SQLite(const Path & path, bool create)
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
SQLiteError::throw_(db, "setting timeout");
+ if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") {
+ // To debug sqlite statements; trace all of them
+ sqlite3_trace(db, &traceSQL, nullptr);
+ }
+
exec("pragma foreign_keys = 1");
}