max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-22 05:56:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into lazy-trees

Browse source
This commit is contained in:
Eelco Dolstra 2022-08-10 12:38:31 +02:00
commit dd1dac0f78
23 changed files with 697 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -22,7 +22,7 @@ perl/Makefile.config
/doc/manual/src/SUMMARY.md
/doc/manual/src/command-ref/new-cli
/doc/manual/src/command-ref/conf-file.md
/doc/manual/src/expressions/builtins.md
/doc/manual/src/language/builtins.md
# /scripts/
/scripts/nix-profile.sh

2
README.md
View file

@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download
## Building And Developing
See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to
See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to
build nix from source with nix-build or how to get a development environment.
## Additional Resources

9
configure.ac
View file

@ -299,15 +299,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
AC_CHECK_FUNCS([strsignal posix_fallocate sysconf])
# This is needed if bzip2 is a static library, and the Nix libraries
# are dynamic.
case "${host_os}" in
darwin*)
LDFLAGS="-all_load $LDFLAGS"
;;
esac
AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),
sandbox_shell=$withval)
AC_SUBST(sandbox_shell)

6
doc/manual/local.mk
View file

@ -1,5 +1,9 @@
ifeq ($(doc_generate),yes)
MANUAL_SRCS := \
$(call rwildcard, $(d)/src, *.md) \
$(call rwildcard, $(d)/src, */*.md)
# Generate man pages.
man-pages := $(foreach n, \
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
@ -97,7 +101,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
done
@touch $@
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(call rwildcard, $(d)/src, *.md)
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual
endif

8
doc/manual/src/SUMMARY.md.in
View file

@ -59,6 +59,14 @@
@manpages@
- [Files](command-ref/files.md)
- [nix.conf](command-ref/conf-file.md)
<!--
- [Architecture](architecture/architecture.md)
- [Store](architecture/store/store.md)
- [Closure](architecture/store/store/closure.md)
- [Build system terminology](architecture/store/store/build-system-terminology.md)
- [Store Path](architecture/store/path.md)
- [File System Object](architecture/store/fso.md)
-->
- [Glossary](glossary.md)
- [Contributing](contributing/contributing.md)
- [Hacking](contributing/hacking.md)

79
doc/manual/src/architecture/architecture.md Normal file
View file

@ -0,0 +1,79 @@
# Architecture
*(This chapter is unstable and a work in progress. Incoming links may rot.)*
This chapter describes how Nix works.
It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools.
## Overview
Nix consists of [hierarchical layers][layer-architecture].
```
+-----------------------------------------------------------------+
| Nix |
| [ commmand line interface ]------, |
| | | |
| evaluates | |
| | manages |
| V | |
| [ configuration language ] | |
| | | |
| +-----------------------------|-------------------V-----------+ |
| | store evaluates to | |
| | | | |
| | referenced by V builds | |
| | [ build input ] ---> [ build plan ] ---> [ build result ] | |
| | | |
| +-------------------------------------------------------------+ |
+-----------------------------------------------------------------+
```
At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers.
Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language.
It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*.
The command line and Nix language are what users interact with most.
> **Note**
> The Nix language itself does not have a notion of *packages* or *configurations*.
> As far as we are concerned here, the inputs and results of a build plan are just data.
Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them.
It can also execute build plans to produce new data.
A build plan is a series of *build tasks*.
Each build task has a special build input which is used as *build instructions*.
The result of a build task can be input to another build task.
```
+-----------------------------------------------------------------------------------------+
| store |
| ................................................. |
| : build plan : |
| : : |
| [ build input ]-----instructions-, : |
| : | : |
| : v : |
| [ build input ]----------->[ build task ]--instructions-, : |
| : | : |
| : | : |
| : v : |
| : [ build task ]----->[ build result ] |
| [ build input ]-----instructions-, ^ : |
| : | | : |
| : v | : |
| [ build input ]----------->[ build task ]---------------' : |
| : ^ : |
| : | : |
| [ build input ]------------------' : |
| : : |
| : : |
| :...............................................: |
| |
+-----------------------------------------------------------------------------------------+
```
[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers
[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming

69
doc/manual/src/architecture/store/fso.md Normal file
View file

@ -0,0 +1,69 @@
# File System Object
The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object).
Every file system object is one of the following:
- File: an executable flag, and arbitrary data for contents
- Directory: mapping of names to child file system objects
- [Symbolic link][symlink]: may point anywhere.
We call a store object's outermost file system object the *root*.
data FileSystemObject
= File { isExecutable :: Bool, contents :: Bytes }
| Directory { entries :: Map FileName FileSystemObject }
| SymLink { target :: Path }
Examples:
- a directory with contents
/nix/store/<hash>-hello-2.10
├── bin
│   └── hello
└── share
├── info
│   └── hello.info
└── man
└── man1
└── hello.1.gz
- a directory with relative symlink and other contents
/nix/store/<hash>-go-1.16.9
├── bin -> share/go/bin
├── nix-support/
└── share/
- a directory with absolute symlink
/nix/store/d3k...-nodejs
└── nix_node -> /nix/store/f20...-nodejs-10.24.
A bare file or symlink can be a root file system object.
Examples:
/nix/store/<hash>-hello-2.10.tar.gz
/nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc
Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended.
Examples:
- an arbitrarily symlinked file may change or not exist at all
/nix/store/<hash>-foo
└── foo -> /home/foo
- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted
/nix/store/<hash>-bar
└── bar -> /nix/store/abc...-foo
Nix file system objects do not support [hard links][hardlink]:
each file system object which is not the root has exactly one parent and one name.
However, as store objects are immutable, an underlying file system can use hard links for optimization.
[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link
[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link

105
doc/manual/src/architecture/store/path.md Normal file
View file

@ -0,0 +1,105 @@
# Store Path
Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*.
Store paths are pairs of
- a 20-byte [digest](#digest) for identification
- a symbolic name for people to read.
Example:
- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
- name: `firefox-33.1`
It is rendered to a file system path as the concatenation of
- [store directory](#store-directory)
- path-separator (`/`)
- [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
- hyphen (`-`)
- name
Example:
/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
|--------| |------------------------------| |----------|
store directory digest name
## Store Directory
Every [store](./store.md) has a store directory.
If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path).
This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
> **Note**
> The store directory defaults to `/nix/store`, but is in principle arbitrary.
It is important which store a given store object belongs to:
Files in the store object can contain store paths, and processes may read these paths.
Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries.
Therefore one can only copy store objects to a different store if
- the source and target stores' directories match
or
- the store object in question has no references, that is, contains no store paths.
One cannot copy a store object to a store with a different store directory.
Instead, it has to be rebuilt, together with all its dependencies.
It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums.
# Digest
In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*.
Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing).
> **Historical Note**
> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes.
> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility.
[digest]: https://en.m.wiktionary.org/wiki/digest#Noun
[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function
[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1
[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256
### Reference scanning
When a new store object is built, Nix scans its file contents for store paths to construct its set of references.
The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data.
Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives.
This way, scanning files captures run time dependencies without the user having to declare them explicitly.
Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation.
> **Note**
> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure.
This depends on the specifics of the software to build and run.
>
> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them.
## Input Addressing
Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan.
To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes.
Nix has a custom serialisation format called Nix Archive (NAR)
Store object references of this sort can *not* be validated from the content of the store object.
Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest.
## Content Addressing
Content addressing means that the digest derives from the store object's contents, namely its file system objects and references.
If one knows content addressing was used, one can recalculate the reference and thus verify the store object.
Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance.
Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md).

151
doc/manual/src/architecture/store/store.md Normal file
View file

@ -0,0 +1,151 @@
# Store
A Nix store is a collection of *store objects* with references between them.
It supports operations to manipulate that collection.
The following concept map is a graphical outline of this chapter.
Arrows indicate suggested reading order.
```
,--------------[ store ]----------------,
| | |
v v v
[ store object ] [ closure ]--, [ operations ]
| | | | | |
v | | v v |
[ files and processes ] | | [ garbage collection ] |
/ \ | | |
v v | v v
[ file system object ] [ store path ] | [ derivation ]--->[ building ]
| ^ | | |
v | v v |
[ digest ]----' [ reference scanning ]<------------'
/ \
v v
[ input addressing ] [ content addressing ]
```
## Store Object
A store object can hold
- arbitrary *data*
- *references* to other store objects.
Store objects can be build inputs, build results, or build tasks.
Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted.
## Reference
A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]:
The only way to obtain references is by adding or building store objects.
A reference will always point to exactly one store object.
## Operations
A Nix store can *add*, *retrieve*, and *delete* store objects.
[ data ]
|
V
[ store ] ---> add ----> [ store' ]
|
V
[ reference ]
<!-- -->
[ reference ]
|
V
[ store ] ---> get
|
V
[ store object ]
<!-- -->
[ reference ]
|
V
[ store ] --> delete --> [ store' ]
It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks.
[ reference ]
|
V
[ store ] --> build --(maybe)--> [ store' ]
|
V
[ reference ]
As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects.
[ store ] --> collect garbage --> [ store' ]
## Files and Processes
Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths.
That allows processes to resolve references contained in files and thus access the contents of store objects.
Store objects are therefore implemented as the pair of
- a [file system object](fso.md) for data
- a set of [store paths](path.md) for references.
[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file
[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor
The following diagram shows a radical simplification of how Nix interacts with the operating system:
It uses files as build inputs, and build outputs are files again.
On the operating system, files can be run as processes, which in turn operate on files.
A build function also amounts to an operating system process (not depicted).
```
+-----------------------------------------------------------------+
| Nix |
| [ commmand line interface ]------, |
| | | |
| evaluates | |
| | manages |
| V | |
| [ configuration language ] | |
| | | |
| +-----------------------------|-------------------V-----------+ |
| | store evaluates to | |
| | | | |
| | referenced by V builds | |
| | [ build input ] ---> [ build plan ] ---> [ build result ] | |
| | ^ | | |
| +---------|----------------------------------------|----------+ |
+-----------|----------------------------------------|------------+
| |
file system object store path
| |
+-----------|----------------------------------------|------------+
| operating system +------------+ | |
| '------------ | | <-----------' |
| | file | |
| ,-- | | <-, |
| | +------------+ | |
| execute as | | read, write, execute |
| | +------------+ | |
| '-> | process | --' |
| +------------+ |
+-----------------------------------------------------------------+
```
There exist different types of stores, which all follow this model.
Examples:
- store on the local file system
- remote store accessible via SSH
- binary cache store accessible via HTTP
To make store objects accessible to processes, stores ultimately have to expose store objects through the file system.

32
doc/manual/src/architecture/store/store/build-system-terminology.md Normal file
View file

@ -0,0 +1,32 @@
# A [Rosetta stone][rosetta-stone] for build system terminology
The Nix store's design is comparable to other build systems.
Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change.
The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future.
| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language |
| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ |
| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value |
| build instructions | builder | ([depends on action type][bazel-actions]) | function | function |
| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] |
| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] |
| build | build | build | application of `Build` | evaluation |
| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap |
All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018).
[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone
[bazel]: https://bazel.build/start/bazel-intro
[bazel-artifact]: https://bazel.build/reference/glossary#artifact
[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html
[bazel-action]: https://bazel.build/reference/glossary#action
[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph
[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph
[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache
[thunk]: https://en.m.wikipedia.org/wiki/Thunk
[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph
[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming
[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf
[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf
[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf

29
doc/manual/src/architecture/store/store/closure.md Normal file
View file

@ -0,0 +1,29 @@
# Closure
Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store.
The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*.
Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity:
- A newly added store object cannot have references, unless it is a build task.
- Build results must only refer to store objects in the closure of the build inputs.
Building a store object will add appropriate references, according to the build task.
- Store objects being copied must refer to objects already in the destination store.
Recursive copying must either proceed in dependency order or be atomic.
- We can only safely delete store objects which are not reachable from any reference still in use.
<!-- more details in section on garbage collection, link to it once it exists -->
[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity
[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)
[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object
[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type
[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier

18
doc/manual/src/installation/installing-binary.md
View file

@ -148,7 +148,8 @@ and `/etc/zshrc` which you may remove.
This will remove all the build users that no longer serve a purpose.
4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store
volume on `/nix`, which looks like this,
volume on `/nix`, which looks like
`UUID=<uuid> /nix apfs rw,noauto,nobrowse,suid,owners` or
`LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic
mounting of the Nix Store volume.
@ -175,6 +176,18 @@ and `/etc/zshrc` which you may remove.
This will remove the Nix Store volume and everything that was added to the
store.
If the output indicates that the command couldn't remove the volume, you should
make sure you don't have an _unmounted_ Nix Store volume. Look for a
"Nix Store" volume in the output of the following command:
```console
diskutil list
```
If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil
deleteVolume command, but replace `/nix` with the store volume's `diskXsY`
identifier.
> **Note**
>
> After you complete the steps here, you will still have an empty `/nix`
@ -191,8 +204,7 @@ and `/etc/zshrc` which you may remove.
<!-- Note: anchors above to catch permalinks to old explanations -->
We believe we have ironed out how to cleanly support the read-only root
on modern macOS. New installs will do this automatically, and you can
also re-run a new installer to convert your existing setup.
on modern macOS. New installs will do this automatically.
This section previously detailed the situation, options, and trade-offs,
but it now only outlines what the installer does. You don't need to know

2
scripts/install-multi-user.sh
View file

@ -640,7 +640,7 @@ place_channel_configuration() {
check_selinux() {
if command -v getenforce > /dev/null 2>&1; then
if ! [ "$(getenforce)" = "Disabled" ]; then
if [ "$(getenforce)" = "Enforcing" ]; then
failure <<EOF
Nix does not work with selinux enabled yet!
see https://github.com/NixOS/nix/issues/2374

3
src/libstore/build/derivation-goal.cc
View file

@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst)
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
if (rename(src.c_str(), dst.c_str()))
throw SysError("renaming '%1%' to '%2%'", src, dst);
renameFile(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);

8
src/libstore/build/local-derivation-goal.cc
View file

@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst)
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
if (rename(src.c_str(), dst.c_str()))
throw SysError("renaming '%1%' to '%2%'", src, dst);
renameFile(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
if (buildMode != bmCheck && status.known->isValid()) continue;
auto p = worker.store.printStorePath(status.known->path);
if (pathExists(chrootRootDir + p))
rename((chrootRootDir + p).c_str(), p.c_str());
renameFile((chrootRootDir + p), p);
}
return diskFull;
@ -2625,8 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
if (rename(path.c_str(), dst.c_str()))
throw SysError("renaming '%s' to '%s'", path, dst);
renameFile(path, dst);
}
}

3
src/libstore/builtins/unpack-channel.cc
View file

@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
auto entries = readDirectory(out);
if (entries.size() != 1)
throw Error("channel tarball '%s' contains more than one file", src);
if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
throw SysError("renaming channel directory");
renameFile((out + "/" + entries[0].name), (out + "/" + channelName));
}
}

4
src/libstore/gc.cc
View file

@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target)
createSymlink(target, tempLink);
/* Atomically replace the old one. */
if (rename(tempLink.c_str(), link.c_str()) == -1)
throw SysError("cannot rename '%1%' to '%2%'",
tempLink , link);
renameFile(tempLink, link);
}

3
src/libstore/local-binary-cache-store.cc
View file

@ -57,8 +57,7 @@ protected:
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
renameFile(tmp, path2);
del.cancel();
}

6
src/libstore/local-store.cc
View file

@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
writeFile(realPath, dumpSource);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))
throw Error("renaming '%s' to '%s'", tempPath, realPath);
moveFile(tempPath, realPath);
}
/* For computing the nar hash. In recursive SHA-256 mode, this
@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
writeFile(tmpFile, compress("bzip2", log));
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
renameFile(tmpFile, logPath);
}
std::optional<std::string> LocalStore::getVersion()

6
src/libstore/optimise-store.cc
View file

@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
try {
renameFile(tempLink, path);
} catch (SysError & e) {
if (unlink(tempLink.c_str()) == -1)
printError("unable to unlink '%1%'", tempLink);
if (errno == EMLINK) {
@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
debug("'%s' has reached maximum number of links", linkPath);
return;
}
throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
throw;
}
stats.filesLinked++;

172
src/libutil/filesystem.cc Normal file
View file

@ -0,0 +1,172 @@
#include <sys/time.h>
#include <filesystem>
#include "finally.hh"
#include "util.hh"
#include "types.hh"
namespace fs = std::filesystem;
namespace nix {
static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
int & counter)
{
tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
if (includePid)
return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
else
return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
}
Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter, mode_t mode)
{
static int globalCounter = 0;
int localCounter = 0;
int & counter(useGlobalCounter ? globalCounter : localCounter);
while (1) {
checkInterrupt();
Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
if (mkdir(tmpDir.c_str(), mode) == 0) {
#if __FreeBSD__
/* Explicitly set the group of the directory. This is to
work around around problems caused by BSD's group
ownership semantics (directories inherit the group of
the parent). For instance, the group of /tmp on
FreeBSD is "wheel", so all directories created in /tmp
will be owned by "wheel"; but if the user is not in
"wheel", then "tar" will fail to unpack archives that
have the setgid bit set on directories. */
if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
throw SysError("setting group of directory '%1%'", tmpDir);
#endif
return tmpDir;
}
if (errno != EEXIST)
throw SysError("creating directory '%1%'", tmpDir);
}
}
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
// FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
closeOnExec(fd.get());
return {std::move(fd), tmpl};
}
void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target);
if (mtime) {
struct timeval times[2];
times[0].tv_sec = *mtime;
times[0].tv_usec = 0;
times[1].tv_sec = *mtime;
times[1].tv_usec = 0;
if (lutimes(link.c_str(), times))
throw SysError("setting time of symlink '%s'", link);
}
}
void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
createSymlink(target, tmp, mtime);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;
}
renameFile(tmp, link);
break;
}
}
void setWriteTime(const fs::path & p, const struct stat & st)
{
struct timeval times[2];
times[0] = {
.tv_sec = st.st_atime,
.tv_usec = 0,
};
times[1] = {
.tv_sec = st.st_mtime,
.tv_usec = 0,
};
if (lutimes(p.c_str(), times) != 0)
throw SysError("changing modification time of '%s'", p);
}
void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
{
// TODO: Rewrite the `is_*` to use `symlink_status()`
auto statOfFrom = lstat(from.path().c_str());
auto fromStatus = from.symlink_status();
// Mark the directory as writable so that we can delete its children
if (andDelete && fs::is_directory(fromStatus)) {
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
}
if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
} else if (fs::is_directory(fromStatus)) {
fs::create_directory(to);
for (auto & entry : fs::directory_iterator(from.path())) {
copy(entry, to / entry.path().filename(), andDelete);
}
} else {
throw Error("file '%s' has an unsupported type", from.path());
}
setWriteTime(to, statOfFrom);
if (andDelete) {
if (!fs::is_symlink(fromStatus))
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
fs::remove(from.path());
}
}
void renameFile(const Path & oldName, const Path & newName)
{
fs::rename(oldName, newName);
}
void moveFile(const Path & oldName, const Path & newName)
{
try {
renameFile(oldName, newName);
} catch (fs::filesystem_error & e) {
auto oldPath = fs::path(oldName);
auto newPath = fs::path(newName);
// For the move to be as atomic as possible, copy to a temporary
// directory
fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
Finally removeTemp = [&]() { fs::remove(temp); };
auto tempCopyTarget = temp / "copy-target";
if (e.code().value() == EXDEV) {
fs::remove(newPath);
warn("Can’t rename %s as %s, copying instead", oldName, newName);
copy(fs::directory_entry(oldPath), tempCopyTarget, true);
renameFile(tempCopyTarget, newPath);
}
}
}
}

100
src/libutil/util.cc
View file

@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed)
}
static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
int & counter)
{
tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
if (includePid)
return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
else
return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
}
Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter, mode_t mode)
{
static int globalCounter = 0;
int localCounter = 0;
int & counter(useGlobalCounter ? globalCounter : localCounter);
while (1) {
checkInterrupt();
Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
if (mkdir(tmpDir.c_str(), mode) == 0) {
#if __FreeBSD__
/* Explicitly set the group of the directory. This is to
work around around problems caused by BSD's group
ownership semantics (directories inherit the group of
the parent). For instance, the group of /tmp on
FreeBSD is "wheel", so all directories created in /tmp
will be owned by "wheel"; but if the user is not in
"wheel", then "tar" will fail to unpack archives that
have the setgid bit set on directories. */
if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
throw SysError("setting group of directory '%1%'", tmpDir);
#endif
return tmpDir;
}
if (errno != EEXIST)
throw SysError("creating directory '%1%'", tmpDir);
}
}
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
// FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
closeOnExec(fd.get());
return {std::move(fd), tmpl};
}
std::string getUserName()
{
auto pw = getpwuid(geteuid());
@ -577,6 +522,7 @@ Path getHome()
{
static Path homeDir = []()
{
std::optional<std::string> unownedUserHomeDir = {};
auto homeDir = getEnv("HOME");
if (homeDir) {
// Only use $HOME if doesn't exist or is owned by the current user.
@ -588,8 +534,7 @@ Path getHome()
homeDir.reset();
}
} else if (st.st_uid != geteuid()) {
warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir);
homeDir.reset();
unownedUserHomeDir.swap(homeDir);
}
}
if (!homeDir) {
@ -600,6 +545,9 @@ Path getHome()
|| !pw || !pw->pw_dir || !pw->pw_dir[0])
throw Error("cannot determine user's home directory");
homeDir = pw->pw_dir;
if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
}
}
return *homeDir;
}();
@ -681,44 +629,6 @@ Paths createDirs(const Path & path)
}
void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target);
if (mtime) {
struct timeval times[2];
times[0].tv_sec = *mtime;
times[0].tv_usec = 0;
times[1].tv_sec = *mtime;
times[1].tv_usec = 0;
if (lutimes(link.c_str(), times))
throw SysError("setting time of symlink '%s'", link);
}
}
void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
createSymlink(target, tmp, mtime);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;
}
if (rename(tmp.c_str(), link.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmp, link);
break;
}
}
void readFull(int fd, char * buf, size_t count)
{
while (count) {

11
src/libutil/util.hh
View file

@ -168,6 +168,17 @@ void createSymlink(const Path & target, const Path & link,
void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
void renameFile(const Path & src, const Path & dst);
/**
* Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
* are on a different filesystem.
*
* Beware that this might not be atomic because of the copy that happens behind
* the scenes
*/
void moveFile(const Path & src, const Path & dst);
/* Wrappers arount read()/write() that read/write exactly the
requested number of bytes. */