Checkpoint

This commit is contained in:
Eelco Dolstra 2022-02-15 16:38:22 +01:00
parent e827e6288f
commit 3ec83565b1
11 changed files with 116 additions and 39 deletions

View file

@ -2070,14 +2070,14 @@ std::string EvalState::copyPathToStore(PathSet & context, const Path & path)
auto path2 = unpackPath(path); auto path2 = unpackPath(path);
#if 0 #if 0
auto p = settings.readOnlyMode auto p = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), canonPath(path)).first ? store->computeStorePathForPath(path2.baseName(), canonPath(path)).first
: store->addToStore(std::string(baseNameOf(path)), canonPath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); : store->addToStore(path2.baseName(), canonPath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
#endif #endif
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
path2.accessor->dumpPath(path2.path, sink); path2.accessor->dumpPath(path2.path, sink);
}); });
// FIXME: readOnlyMode // FIXME: readOnlyMode
auto p = store->addToStoreFromDump(*source, std::string(baseNameOf(path)), FileIngestionMethod::Recursive, htSHA256, repair); auto p = store->addToStoreFromDump(*source, path2.baseName(), FileIngestionMethod::Recursive, htSHA256, repair);
dstPath = store->printStorePath(p); dstPath = store->printStorePath(p);
allowPath(p); allowPath(p);
srcToStore.insert_or_assign(path, std::move(p)); srcToStore.insert_or_assign(path, std::move(p));

View file

@ -209,7 +209,7 @@ static Flake getFlake(
.originalRef = originalRef, .originalRef = originalRef,
.resolvedRef = resolvedRef, .resolvedRef = resolvedRef,
.lockedRef = lockedRef, .lockedRef = lockedRef,
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) //.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
}; };
if (!pathExists(flakeFile)) if (!pathExists(flakeFile))
@ -326,6 +326,7 @@ LockedFlake lockFlake(
state.store->setOptions(); state.store->setOptions();
} }
#if 0
try { try {
// FIXME: symlink attack // FIXME: symlink attack
@ -669,6 +670,9 @@ LockedFlake lockFlake(
e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
throw; throw;
} }
#endif
throw UnimplementedError("lockFlake");
} }
void callFlake(EvalState & state, void callFlake(EvalState & state,
@ -683,13 +687,17 @@ void callFlake(EvalState & state,
vLocks->mkString(lockedFlake.lockFile.to_string()); vLocks->mkString(lockedFlake.lockFile.to_string());
#if 0
emitTreeAttrs( emitTreeAttrs(
state, state,
*lockedFlake.flake.sourceInfo, //*lockedFlake.flake.sourceInfo,
lockedFlake.flake.lockedRef.input, lockedFlake.flake.lockedRef.input,
*vRootSrc, *vRootSrc,
false, false,
lockedFlake.flake.forceDirty); lockedFlake.flake.forceDirty);
#endif
throw UnimplementedError("callFlake");
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
@ -756,7 +764,8 @@ Fingerprint LockedFlake::getFingerprint() const
// flake.sourceInfo.storePath for the fingerprint. // flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256, return hashString(htSHA256,
fmt("%s;%s;%d;%d;%s", fmt("%s;%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(), "FIXME",
//flake.sourceInfo->storePath.to_string(),
flake.lockedRef.subdir, flake.lockedRef.subdir,
flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0), flake.lockedRef.input.getLastModified().value_or(0),

View file

@ -63,7 +63,6 @@ struct Flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher FlakeRef lockedRef; // the specific local store result of invoking the fetcher
bool forceDirty = false; // pretend that 'lockedRef' is dirty bool forceDirty = false; // pretend that 'lockedRef' is dirty
std::optional<std::string> description; std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs; FlakeInputs inputs;
ConfigFile config; // 'nixConfig' attribute ConfigFile config; // 'nixConfig' attribute
~Flake(); ~Flake();
@ -139,7 +138,7 @@ void callFlake(
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const fetchers::Tree & tree, const SourcePath & path,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
bool emptyRevFallback = false, bool emptyRevFallback = false,

View file

@ -16,15 +16,14 @@ Path EvalState::packPath(const SourcePath & path)
SourcePath EvalState::unpackPath(const Path & path) SourcePath EvalState::unpackPath(const Path & path)
{ {
printError("UNPACK %s", path);
if (hasPrefix(path, marker)) { if (hasPrefix(path, marker)) {
auto s = path.substr(marker.size()); auto s = path.substr(marker.size());
auto slash = s.find('/'); auto slash = s.find('/');
assert(slash != s.npos);
auto n = std::stoi(s.substr(0, slash)); auto n = std::stoi(s.substr(0, slash));
printError("GOT %d", n);
auto i = inputAccessors.find(n); auto i = inputAccessors.find(n);
assert(i != inputAccessors.end()); assert(i != inputAccessors.end());
return {i->second, s.substr(slash)}; return {i->second, slash != std::string::npos ? s.substr(slash) : "/"};
} else { } else {
printError("FIXME: %s", path); printError("FIXME: %s", path);
return rootPath(path); return rootPath(path);

View file

@ -13,25 +13,32 @@ namespace nix {
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const fetchers::Tree & tree, const SourcePath & path,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
bool emptyRevFallback, bool emptyRevFallback,
bool forceDirty) bool forceDirty)
{ {
assert(input.isLocked()); // FIXME?
//assert(input.isLocked());
auto attrs = state.buildBindings(8); auto attrs = state.buildBindings(8);
#if 0
auto storePath = state.store->printStorePath(tree.storePath); auto storePath = state.store->printStorePath(tree.storePath);
attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); attrs.alloc(state.sOutPath).mkString(storePath, {storePath});
#endif
attrs.alloc(state.sOutPath).mkPath(state.packPath(path));
// FIXME: support arbitrary input attributes. // FIXME: support arbitrary input attributes.
#if 0
auto narHash = input.getNarHash(); auto narHash = input.getNarHash();
assert(narHash); assert(narHash);
attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
#endif
if (input.getType() == "git") if (input.getType() == "git")
attrs.alloc("submodules").mkBool( attrs.alloc("submodules").mkBool(
@ -169,11 +176,17 @@ static void fetchTree(
if (evalSettings.pureEval && !input.isLocked()) if (evalSettings.pureEval && !input.isLocked())
throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos); throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos);
auto [tree, input2] = input.fetch(state.store); auto [accessor, input2] = input.lazyFetch(state.store);
state.allowPath(tree.storePath); //state.allowPath(tree.storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); emitTreeAttrs(
state,
{accessor, "/"},
input2,
v,
params.emptyRevFallback,
false);
} }
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)

View file

@ -172,6 +172,19 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
return {std::move(tree), input}; return {std::move(tree), input};
} }
std::pair<ref<InputAccessor>, Input> Input::lazyFetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
try {
return scheme->lazyFetch(store, *this);
} catch (Error & e) {
e.addTrace({}, "while fetching the input '%s'", to_string());
throw;
}
}
Input Input::applyOverrides( Input Input::applyOverrides(
std::optional<std::string> ref, std::optional<std::string> ref,
std::optional<Hash> rev) const std::optional<Hash> rev) const
@ -289,4 +302,11 @@ void InputScheme::clone(const Input & input, const Path & destDir)
throw Error("do not know how to clone input '%s'", input.to_string()); throw Error("do not know how to clone input '%s'", input.to_string());
} }
std::pair<ref<InputAccessor>, Input> InputScheme::lazyFetch(ref<Store> store, const Input & input)
{
auto [storePath, input2] = fetch(store, input);
return {makeFSInputAccessor(store->toRealPath(storePath)), input2};
}
} }

View file

@ -73,6 +73,11 @@ public:
the Nix store and the locked input. */ the Nix store and the locked input. */
std::pair<Tree, Input> fetch(ref<Store> store) const; std::pair<Tree, Input> fetch(ref<Store> store) const;
/* Return an InputAccessor that allows access to files in the
input without copying it to the store. Also return a possibly
unlocked input. */
std::pair<ref<InputAccessor>, Input> lazyFetch(ref<Store> store) const;
Input applyOverrides( Input applyOverrides(
std::optional<std::string> ref, std::optional<std::string> ref,
std::optional<Hash> rev) const; std::optional<Hash> rev) const;
@ -132,6 +137,8 @@ struct InputScheme
virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0; virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
virtual std::pair<ref<InputAccessor>, Input> lazyFetch(ref<Store> store, const Input & input);
virtual ref<InputAccessor> getAccessor() virtual ref<InputAccessor> getAccessor()
{ {
throw UnimplementedError("getAccessor"); throw UnimplementedError("getAccessor");

View file

@ -14,7 +14,7 @@ InputAccessor::InputAccessor()
// FIXME: merge with archive.cc. // FIXME: merge with archive.cc.
const std::string narVersionMagic1 = "nix-archive-1"; const std::string narVersionMagic1 = "nix-archive-1";
static string caseHackSuffix = "~nix~case~hack~"; static std::string caseHackSuffix = "~nix~case~hack~";
void InputAccessor::dumpPath( void InputAccessor::dumpPath(
const Path & path, const Path & path,
@ -51,12 +51,12 @@ void InputAccessor::dumpPath(
/* If we're on a case-insensitive system like macOS, undo /* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */ the case hack applied by restorePath(). */
std::map<string, string> unhacked; std::map<std::string, std::string> unhacked;
for (auto & i : readDirectory(path)) for (auto & i : readDirectory(path))
if (/* archiveSettings.useCaseHack */ false) { // FIXME if (/* archiveSettings.useCaseHack */ false) { // FIXME
string name(i.first); std::string name(i.first);
size_t pos = i.first.find(caseHackSuffix); size_t pos = i.first.find(caseHackSuffix);
if (pos != string::npos) { if (pos != std::string::npos) {
debug(format("removing case hack suffix from '%s'") % (path + "/" + i.first)); debug(format("removing case hack suffix from '%s'") % (path + "/" + i.first));
name.erase(pos); name.erase(pos);
} }
@ -227,4 +227,10 @@ ref<MemoryInputAccessor> makeMemoryInputAccessor()
return make_ref<MemoryInputAccessorImpl>(); return make_ref<MemoryInputAccessorImpl>();
} }
std::string_view SourcePath::baseName() const
{
// FIXME
return path == "" || path == "/" ? "source" : baseNameOf(path);
}
} }

View file

@ -57,6 +57,8 @@ struct SourcePath
{ {
ref<InputAccessor> accessor; ref<InputAccessor> accessor;
Path path; Path path;
std::string_view baseName() const;
}; };
std::ostream & operator << (std::ostream & str, const SourcePath & path); std::ostream & operator << (std::ostream & str, const SourcePath & path);

View file

@ -81,20 +81,20 @@ struct PathInputScheme : InputScheme
// nothing to do // nothing to do
} }
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override Path getAbsPath(ref<Store> store, const Input & input)
{ {
Input input(_input);
std::string absPath;
auto path = getStrAttr(input.attrs, "path"); auto path = getStrAttr(input.attrs, "path");
if (path[0] != '/') { if (path[0] == '/')
return path;
if (!input.parent) if (!input.parent)
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
auto parent = canonPath(*input.parent); auto parent = canonPath(*input.parent);
// the path isn't relative, prefix it // the path isn't relative, prefix it
absPath = nix::absPath(path, parent); auto absPath = nix::absPath(path, parent);
// for security, ensure that if the parent is a store path, it's inside it // for security, ensure that if the parent is a store path, it's inside it
if (store->isInStore(parent)) { if (store->isInStore(parent)) {
@ -102,8 +102,15 @@ struct PathInputScheme : InputScheme
if (!isDirOrInDir(absPath, storePath)) if (!isDirOrInDir(absPath, storePath))
throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
} }
} else
absPath = path; return absPath;
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
Input input(_input);
auto absPath = getAbsPath(store, input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
@ -125,6 +132,14 @@ struct PathInputScheme : InputScheme
return {std::move(*storePath), input}; return {std::move(*storePath), input};
} }
std::pair<ref<InputAccessor>, Input> lazyFetch(ref<Store> store, const Input & input) override
{
auto absPath = getAbsPath(store, input);
auto input2(input);
input2.attrs.emplace("path", absPath);
return {makeFSInputAccessor(absPath), std::move(input2)};
}
}; };
static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); }); static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });

View file

@ -182,7 +182,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
j["revCount"] = *revCount; j["revCount"] = *revCount;
if (auto lastModified = flake.lockedRef.input.getLastModified()) if (auto lastModified = flake.lockedRef.input.getLastModified())
j["lastModified"] = *lastModified; j["lastModified"] = *lastModified;
#if 0
j["path"] = store->printStorePath(flake.sourceInfo->storePath); j["path"] = store->printStorePath(flake.sourceInfo->storePath);
#endif
j["locks"] = lockedFlake.lockFile.toJSON(); j["locks"] = lockedFlake.lockFile.toJSON();
logger->cout("%s", j.dump()); logger->cout("%s", j.dump());
} else { } else {
@ -196,9 +198,11 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
logger->cout( logger->cout(
ANSI_BOLD "Description:" ANSI_NORMAL " %s", ANSI_BOLD "Description:" ANSI_NORMAL " %s",
*flake.description); *flake.description);
#if 0
logger->cout( logger->cout(
ANSI_BOLD "Path:" ANSI_NORMAL " %s", ANSI_BOLD "Path:" ANSI_NORMAL " %s",
store->printStorePath(flake.sourceInfo->storePath)); store->printStorePath(flake.sourceInfo->storePath));
#endif
if (auto rev = flake.lockedRef.input.getRev()) if (auto rev = flake.lockedRef.input.getRev())
logger->cout( logger->cout(
ANSI_BOLD "Revision:" ANSI_NORMAL " %s", ANSI_BOLD "Revision:" ANSI_NORMAL " %s",
@ -881,6 +885,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
StorePathSet sources; StorePathSet sources;
throw UnimplementedError("flake archive");
#if 0
sources.insert(flake.flake.sourceInfo->storePath); sources.insert(flake.flake.sourceInfo->storePath);
if (jsonRoot) if (jsonRoot)
jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath));
@ -911,6 +917,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
copyPaths(*store, *dstStore, sources); copyPaths(*store, *dstStore, sources);
} }
#endif
} }
}; };