Merge remote-tracking branch 'upstream/master' into overlayfs-store

This commit is contained in:
John Ericson 2023-06-14 10:25:57 -04:00
commit 32147c8c7b
16 changed files with 367 additions and 45 deletions

View file

@ -80,6 +80,38 @@ my $s3_us = Net::Amazon::S3->new(
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
sub getStorePath {
my ($jobName, $output) = @_;
my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'";
}
sub copyManual {
my $manual = getStorePath("build.x86_64-linux", "doc");
print "$manual\n";
my $manualNar = "$tmpDir/$releaseName-manual.nar.xz";
print "$manualNar\n";
unless (-e $manualNar) {
system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0
or die "unable to fetch $manual\n";
rename("$manualNar.tmp", $manualNar) or die;
}
unless (-e "$tmpDir/manual") {
system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0
or die "unable to unpack $manualNar\n";
rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die;
system("rm -rf '$tmpDir/manual.tmp'") == 0 or die;
}
system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
or die "syncing manual to S3\n";
}
copyManual;
sub downloadFile { sub downloadFile {
my ($jobName, $productNr, $dstName) = @_; my ($jobName, $productNr, $dstName) = @_;
@ -179,9 +211,20 @@ if ($isLatest) {
system("docker manifest push nixos/nix:latest") == 0 or die; system("docker manifest push nixos/nix:latest") == 0 or die;
} }
# Upload nix-fallback-paths.nix.
write_file("$tmpDir/fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
"}\n");
# Upload release files to S3. # Upload release files to S3.
for my $fn (glob "$tmpDir/*") { for my $fn (glob "$tmpDir/*") {
my $name = basename($fn); my $name = basename($fn);
next if $name eq "manual";
my $dstKey = "$releaseDir/" . $name; my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) { unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
@ -189,8 +232,7 @@ for my $fn (glob "$tmpDir/*") {
my $configuration = (); my $configuration = ();
$configuration->{content_type} = "application/octet-stream"; $configuration->{content_type} = "application/octet-stream";
if ($fn =~ /.sha256|install/) { if ($fn =~ /.sha256|install|\.nix$/) {
# Text files
$configuration->{content_type} = "text/plain"; $configuration->{content_type} = "text/plain";
} }
@ -199,24 +241,6 @@ for my $fn (glob "$tmpDir/*") {
} }
} }
# Print new nix-fallback-paths.nix.
if ($isLatest) {
sub getStorePath {
my ($jobName) = @_;
my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
}
print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" .
"{\n" .
" x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
"}\n";
}
# Update the "latest" symlink. # Update the "latest" symlink.
$channelsBucket->add_key( $channelsBucket->add_key(
"nix-latest/install", "", "nix-latest/install", "",

View file

@ -880,7 +880,7 @@ configure_shell_profile() {
fi fi
done done
task "Setting up shell profiles for Fish with with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}" task "Setting up shell profiles for Fish with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}"
for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do
if [ ! -d "$fish_prefix" ]; then if [ ! -d "$fish_prefix" ]; then
# this specific prefix (ie: /etc/fish) is very likely to exist # this specific prefix (ie: /etc/fish) is very likely to exist

View file

@ -3,6 +3,8 @@
#include "../../toml11/toml.hpp" #include "../../toml11/toml.hpp"
#include <sstream>
namespace nix { namespace nix {
static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)
@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
case toml::value_t::offset_datetime: case toml::value_t::offset_datetime:
case toml::value_t::local_date: case toml::value_t::local_date:
case toml::value_t::local_time: case toml::value_t::local_time:
// We fail since Nix doesn't have date and time types {
throw std::runtime_error("Dates and times are not supported"); if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
auto attrs = state.buildBindings(2);
attrs.alloc("_type").mkString("timestamp");
std::ostringstream s;
s << t;
attrs.alloc("value").mkString(s.str());
v.mkAttrs(attrs);
} else {
throw std::runtime_error("Dates and times are not supported");
}
}
break;; break;;
case toml::value_t::empty: case toml::value_t::empty:
v.mkNull(); v.mkNull();

View file

@ -1452,7 +1452,7 @@ void LocalDerivationGoal::startDaemon()
(struct sockaddr *) &remoteAddr, &remoteAddrLen); (struct sockaddr *) &remoteAddr, &remoteAddrLen);
if (!remote) { if (!remote) {
if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINTR || errno == EAGAIN) continue;
if (errno == EINVAL) break; if (errno == EINVAL || errno == ECONNABORTED) break;
throw SysError("accepting connection"); throw SysError("accepting connection");
} }
@ -1482,8 +1482,22 @@ void LocalDerivationGoal::startDaemon()
void LocalDerivationGoal::stopDaemon() void LocalDerivationGoal::stopDaemon()
{ {
if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) {
throw SysError("shutting down daemon socket"); // According to the POSIX standard, the 'shutdown' function should
// return an ENOTCONN error when attempting to shut down a socket that
// hasn't been connected yet. This situation occurs when the 'accept'
// function is called on a socket without any accepted connections,
// leaving the socket unconnected. While Linux doesn't seem to produce
// an error for sockets that have only been accepted, more
// POSIX-compliant operating systems like OpenBSD, macOS, and others do
// return the ENOTCONN error. Therefore, we handle this error here to
// avoid raising an exception for compliant behaviour.
if (errno == ENOTCONN) {
daemonSocket.close();
} else {
throw SysError("shutting down daemon socket");
}
}
if (daemonThread.joinable()) if (daemonThread.joinable())
daemonThread.join(); daemonThread.join();
@ -1494,7 +1508,8 @@ void LocalDerivationGoal::stopDaemon()
thread.join(); thread.join();
daemonWorkerThreads.clear(); daemonWorkerThreads.clear();
daemonSocket = -1; // release the socket.
daemonSocket.close();
} }

View file

@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
&& (std::string_view(SYSTEM) == "x86_64-linux" && (std::string_view(SYSTEM) == "x86_64-linux"
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|| system == "armv7l-linux" || system == "armv7l-linux"
|| system == "armv6l-linux") || system == "armv6l-linux"
|| system == "armv5tel-linux")
{ {
if (personality(PER_LINUX32) == -1) if (personality(PER_LINUX32) == -1)
throw SysError("cannot set 32-bit personality"); throw SysError("cannot set 32-bit personality");

View file

@ -915,12 +915,11 @@ public:
this, {}, "hashed-mirrors", this, {}, "hashed-mirrors",
R"( R"(
A list of web servers used by `builtins.fetchurl` to obtain files by A list of web servers used by `builtins.fetchurl` to obtain files by
hash. The default is `http://tarballs.nixos.org/`. Given a hash type hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
*ht* and a base-16 hash *h*, Nix will try to download the file from download the file from *hashed-mirror*/*ht*/*h*. This allows files to
*hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if be downloaded even if they have disappeared from their original URI.
they have disappeared from their original URI. For example, given For example, given an example mirror `http://tarballs.nixos.org/`,
the default mirror `http://tarballs.nixos.org/`, when building the when building the derivation
derivation
```nix ```nix
builtins.fetchurl { builtins.fetchurl {

View file

@ -13,6 +13,14 @@
namespace nix { namespace nix {
std::string UDSRemoteStoreConfig::doc()
{
return
#include "uds-remote-store.md"
;
}
UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore(const Params & params)
: StoreConfig(params) : StoreConfig(params)
, LocalFSStoreConfig(params) , LocalFSStoreConfig(params)

View file

@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
const std::string name() override { return "Local Daemon Store"; } const std::string name() override { return "Local Daemon Store"; }
std::string doc() override std::string doc() override;
{
return
#include "uds-remote-store.md"
;
}
}; };
class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

View file

@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
std::string_view description; std::string_view description;
}; };
constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{ constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
{ {
.tag = Xp::CaDerivations, .tag = Xp::CaDerivations,
.name = "ca-derivations", .name = "ca-derivations",
@ -214,6 +214,13 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
derivations that are themselves derivations outputs. derivations that are themselves derivations outputs.
)", )",
}, },
{
.tag = Xp::ParseTomlTimestamps,
.name = "parse-toml-timestamps",
.description = R"(
Allow parsing of timestamps in builtins.fromTOML.
)",
},
{ {
.tag = Xp::ReadOnlyLocalStore, .tag = Xp::ReadOnlyLocalStore,
.name = "read-only-local-store", .name = "read-only-local-store",
@ -264,7 +271,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag)
return xpFeatureDetails[(size_t)tag].name; return xpFeatureDetails[(size_t)tag].name;
} }
nlohmann::json documentExperimentalFeatures() nlohmann::json documentExperimentalFeatures()
{ {
StringMap res; StringMap res;
for (auto & xpFeature : xpFeatureDetails) for (auto & xpFeature : xpFeatureDetails)

View file

@ -30,6 +30,7 @@ enum struct ExperimentalFeature
DiscardReferences, DiscardReferences,
DaemonTrustOverride, DaemonTrustOverride,
DynamicDerivations, DynamicDerivations,
ParseTomlTimestamps,
ReadOnlyLocalStore, ReadOnlyLocalStore,
}; };

View file

@ -24,6 +24,7 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <sys/un.h> #include <sys/un.h>
#include <sys/select.h>
#include <errno.h> #include <errno.h>
#include <pwd.h> #include <pwd.h>
#include <grp.h> #include <grp.h>

View file

@ -0,0 +1,130 @@
builtins.fromTOML ''
key = "value"
bare_key = "value"
bare-key = "value"
1234 = "value"
"127.0.0.1" = "value"
"character encoding" = "value"
"ʎǝʞ" = "value"
'key2' = "value"
'quoted "value"' = "value"
name = "Orange"
physical.color = "orange"
physical.shape = "round"
site."google.com" = true
# This is legal according to the spec, but cpptoml doesn't handle it.
#a.b.c = 1
#a.d = 2
str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
int1 = +99
int2 = 42
int3 = 0
int4 = -17
int5 = 1_000
int6 = 5_349_221
int7 = 1_2_3_4_5
hex1 = 0xDEADBEEF
hex2 = 0xdeadbeef
hex3 = 0xdead_beef
oct1 = 0o01234567
oct2 = 0o755
bin1 = 0b11010110
flt1 = +1.0
flt2 = 3.1415
flt3 = -0.01
flt4 = 5e+22
flt5 = 1e6
flt6 = -2E-2
flt7 = 6.626e-34
flt8 = 9_224_617.445_991_228_313
bool1 = true
bool2 = false
odt1 = 1979-05-27T07:32:00Z
odt2 = 1979-05-27T00:32:00-07:00
odt3 = 1979-05-27T00:32:00.999999-07:00
odt4 = 1979-05-27 07:32:00Z
ldt1 = 1979-05-27T07:32:00
ldt2 = 1979-05-27T00:32:00.999999
ld1 = 1979-05-27
lt1 = 07:32:00
lt2 = 00:32:00.999999
arr1 = [ 1, 2, 3 ]
arr2 = [ "red", "yellow", "green" ]
arr3 = [ [ 1, 2 ], [3, 4, 5] ]
arr4 = [ "all", 'strings', """are the same""", ''''type'''']
arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
arr7 = [
1, 2, 3
]
arr8 = [
1,
2, # this is ok
]
[table-1]
key1 = "some string"
key2 = 123
[table-2]
key1 = "another string"
key2 = 456
[dog."tater.man"]
type.name = "pug"
[a.b.c]
[ d.e.f ]
[ g . h . i ]
[ j . "ʞ" . 'l' ]
[x.y.z.w]
name = { first = "Tom", last = "Preston-Werner" }
point = { x = 1, y = 2 }
animal = { type.name = "pug" }
[[products]]
name = "Hammer"
sku = 738594937
[[products]]
[[products]]
name = "Nail"
sku = 284758393
color = "gray"
[[fruit]]
name = "apple"
[fruit.physical]
color = "red"
shape = "round"
[[fruit.variety]]
name = "red delicious"
[[fruit.variety]]
name = "granny smith"
[[fruit]]
name = "banana"
[[fruit.variety]]
name = "plantain"
''

View file

@ -0,0 +1 @@
{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; }

View file

@ -0,0 +1 @@
--extra-experimental-features parse-toml-timestamps

View file

@ -0,0 +1,130 @@
builtins.fromTOML ''
key = "value"
bare_key = "value"
bare-key = "value"
1234 = "value"
"127.0.0.1" = "value"
"character encoding" = "value"
"ʎǝʞ" = "value"
'key2' = "value"
'quoted "value"' = "value"
name = "Orange"
physical.color = "orange"
physical.shape = "round"
site."google.com" = true
# This is legal according to the spec, but cpptoml doesn't handle it.
#a.b.c = 1
#a.d = 2
str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
int1 = +99
int2 = 42
int3 = 0
int4 = -17
int5 = 1_000
int6 = 5_349_221
int7 = 1_2_3_4_5
hex1 = 0xDEADBEEF
hex2 = 0xdeadbeef
hex3 = 0xdead_beef
oct1 = 0o01234567
oct2 = 0o755
bin1 = 0b11010110
flt1 = +1.0
flt2 = 3.1415
flt3 = -0.01
flt4 = 5e+22
flt5 = 1e6
flt6 = -2E-2
flt7 = 6.626e-34
flt8 = 9_224_617.445_991_228_313
bool1 = true
bool2 = false
odt1 = 1979-05-27T07:32:00Z
odt2 = 1979-05-27T00:32:00-07:00
odt3 = 1979-05-27T00:32:00.999999-07:00
odt4 = 1979-05-27 07:32:00Z
ldt1 = 1979-05-27T07:32:00
ldt2 = 1979-05-27T00:32:00.999999
ld1 = 1979-05-27
lt1 = 07:32:00
lt2 = 00:32:00.999999
arr1 = [ 1, 2, 3 ]
arr2 = [ "red", "yellow", "green" ]
arr3 = [ [ 1, 2 ], [3, 4, 5] ]
arr4 = [ "all", 'strings', """are the same""", ''''type'''']
arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
arr7 = [
1, 2, 3
]
arr8 = [
1,
2, # this is ok
]
[table-1]
key1 = "some string"
key2 = 123
[table-2]
key1 = "another string"
key2 = 456
[dog."tater.man"]
type.name = "pug"
[a.b.c]
[ d.e.f ]
[ g . h . i ]
[ j . "ʞ" . 'l' ]
[x.y.z.w]
name = { first = "Tom", last = "Preston-Werner" }
point = { x = 1, y = 2 }
animal = { type.name = "pug" }
[[products]]
name = "Hammer"
sku = 738594937
[[products]]
[[products]]
name = "Nail"
sku = 284758393
color = "gray"
[[fruit]]
name = "apple"
[fruit.physical]
color = "red"
shape = "round"
[[fruit.variety]]
name = "red delicious"
[[fruit.variety]]
name = "granny smith"
[[fruit]]
name = "banana"
[[fruit.variety]]
name = "plantain"
''

View file

@ -1,8 +1,5 @@
source common.sh source common.sh
# FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
enableFeatures 'recursive-nix' enableFeatures 'recursive-nix'
restartDaemon restartDaemon