fetchTree/fetchGit: re-enable shallow fetching

Add several tests for git fetching:
- shallow-cache-separation: can fetch the same repo shallowly and non-shallowly
- shallow-ignore-ref: ensure that ref gets ignored when shallow=true is set
- ssh-shallow: can fetch a git repo via ssh using shallow=1
This commit is contained in:
DavHau 2024-01-19 15:59:15 +07:00
parent d762caff46
commit bc00fa4647
6 changed files with 168 additions and 8 deletions

View file

@ -387,13 +387,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// then use code that was removed in this commit (see blame) // then use code that was removed in this commit (see blame)
auto dir = this->path; auto dir = this->path;
Strings gitArgs;
if (shallow) {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
}
else {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec };
}
runProgram(RunOptions { runProgram(RunOptions {
.program = "git", .program = "git",
.searchPath = true, .searchPath = true,
// FIXME: git stderr messes up our progress indicator, so // FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr. // we're using --quiet for now. Should process its stderr.
.args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec }, .args = gitArgs,
.input = {}, .input = {},
.isInteractive = true .isInteractive = true
}); });

View file

@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time)
return lutimes(path.c_str(), times) == 0; return lutimes(path.c_str(), times) == 0;
} }
Path getCachePath(std::string_view key) Path getCachePath(std::string_view key, bool shallow)
{ {
return getCacheDir() + "/nix/gitv3/" + return getCacheDir()
hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); + "/nix/gitv3/"
+ hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false)
+ (shallow ? "-shallow" : "");
} }
// Returns the name of the HEAD branch. // Returns the name of the HEAD branch.
@ -92,7 +94,8 @@ std::optional<std::string> readHead(const Path & path)
// Persist the HEAD ref from the remote repo in the local cached repo. // Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
{ {
Path cacheDir = getCachePath(actualUrl); // set shallow=false as HEAD will never be queried for a shallow repo
Path cacheDir = getCachePath(actualUrl, false);
try { try {
runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) { } catch (ExecError &e) {
@ -107,7 +110,8 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
{ {
// Create a cache path to store the branch of the HEAD ref. Append something // Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself. // in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl); // set shallow=false as HEAD will never be queried for a shallow repo
Path cacheDir = getCachePath(actualUrl, false);
Path headRefFile = cacheDir + "/HEAD"; Path headRefFile = cacheDir + "/HEAD";
time_t now = time(0); time_t now = time(0);
@ -508,7 +512,7 @@ struct GitInputScheme : InputScheme
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
} else { } else {
Path cacheDir = getCachePath(repoInfo.url); Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
repoDir = cacheDir; repoDir = cacheDir;
repoInfo.gitDir = "."; repoInfo.gitDir = ".";

View file

@ -0,0 +1,57 @@
{
description = "can fetch the same repo shallowly and non-shallowly";
script = ''
# create branch1 off of main
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1' \
\
&& {repo.git} push origin --all
""")
# save the revision
mainRev = client.succeed(f"""
{repo.git} rev-parse main
""").strip()
# fetch shallowly
revCountShallow = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = true;
}}).revCount
'
""").strip()
# ensure the revCount is 0
assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}"
# fetch non-shallowly
revCountNonShallow = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = false;
}}).revCount
'
""").strip()
# ensure the revCount is 1
assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}"
# fetch shallowly again
revCountShallow2 = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = true;
}}).revCount
'
""").strip()
# ensure the revCount is 0
assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}"
'';
}

View file

@ -0,0 +1,40 @@
{
description = "ensure that ref gets ignored when shallow=true is set";
script = ''
# create branch1 off of main
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1' \
\
&& {repo.git} checkout -b branch1 main \
&& echo bangkok > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit2' \
\
&& {repo.git} push origin --all
""")
# save the revisions
mainRev = client.succeed(f"""
{repo.git} rev-parse main
""").strip()
branch1Rev = client.succeed(f"""
{repo.git} rev-parse branch1
""").strip()
# Ensure that ref gets ignored when fetching shallowly.
# This would fail if the ref was respected, as branch1Rev is not on main.
client.succeed(f"""
nix eval --impure --raw --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{branch1Rev}";
ref = "main";
shallow = true;
}})
'
""")
'';
}

View file

@ -0,0 +1,52 @@
{
description = "can fetch a git repo via ssh using shallow=1";
script = ''
# add a file to the repo
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1'
""")
# memoize the revision
rev1 = client.succeed(f"""
{repo.git} rev-parse HEAD
""").strip()
# push to the server
client.succeed(f"""
{repo.git} push origin-ssh main
""")
fetchGit_expr = f"""
builtins.fetchGit {{
url = "{repo.remote_ssh}";
rev = "{rev1}";
shallow = true;
}}
"""
# fetch the repo via nix
fetched1 = client.succeed(f"""
nix eval --impure --raw --expr '({fetchGit_expr}).outPath'
""")
# check if the committed file is there
client.succeed(f"""
test -f {fetched1}/thailand
""")
# check if the revision is the same
rev1_fetched = client.succeed(f"""
nix eval --impure --raw --expr '({fetchGit_expr}).rev'
""").strip()
assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
# check if revCount is 1
revCount1 = client.succeed(f"""
nix eval --impure --expr '({fetchGit_expr}).revCount'
""").strip()
print(f"revCount1: {revCount1}")
assert revCount1 == '0', f"rev count is not 0 but {revCount1}"
'';
}