Merge remote-tracking branch 'upstream/master' into overlayfs-store

maintainers/upload-release.pl

@@ -80,6 +80,38 @@ my $s3_us = Net::Amazon::S3->new(
 
 my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
+sub getStorePath {
+    my ($jobName, $output) = @_;
+    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'";
+}
+
+sub copyManual {
+    my $manual = getStorePath("build.x86_64-linux", "doc");
+    print "$manual\n";
+
+    my $manualNar = "$tmpDir/$releaseName-manual.nar.xz";
+    print "$manualNar\n";
+
+    unless (-e $manualNar) {
+        system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0
+            or die "unable to fetch $manual\n";
+        rename("$manualNar.tmp", $manualNar) or die;
+    }
+
+    unless (-e "$tmpDir/manual") {
+        system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0
+            or die "unable to unpack $manualNar\n";
+        rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die;
+        system("rm -rf '$tmpDir/manual.tmp'") == 0 or die;
+    }
+
+    system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
+        or die "syncing manual to S3\n";
+}
+
+copyManual;
+
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
 
@@ -179,9 +211,20 @@ if ($isLatest) {
     system("docker manifest push nixos/nix:latest") == 0 or die;
 }
 
+# Upload nix-fallback-paths.nix.
+write_file("$tmpDir/fallback-paths.nix",
+    "{\n" .
+    "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+    "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+    "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+    "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+    "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+    "}\n");
+
 # Upload release files to S3.
 for my $fn (glob "$tmpDir/*") {
     my $name = basename($fn);
+    next if $name eq "manual";
     my $dstKey = "$releaseDir/" . $name;
     unless (defined $releasesBucket->head_key($dstKey)) {
         print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
@@ -189,8 +232,7 @@ for my $fn (glob "$tmpDir/*") {
         my $configuration = ();
         $configuration->{content_type} = "application/octet-stream";
 
-        if ($fn =~ /.sha256|install/) {
+        if ($fn =~ /.sha256|install|\.nix$/) {
-            # Text files
             $configuration->{content_type} = "text/plain";
         }
 
@@ -199,24 +241,6 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-# Print new nix-fallback-paths.nix.
-if ($isLatest) {
-    sub getStorePath {
-        my ($jobName) = @_;
-        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-        return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
-    }
-
-    print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" .
-               "{\n" .
-               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-               "}\n";
-}
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",

scripts/install-multi-user.sh

@@ -880,7 +880,7 @@ configure_shell_profile() {
         fi
     done
 
-    task "Setting up shell profiles for Fish with with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}"
+    task "Setting up shell profiles for Fish with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}"
     for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do
         if [ ! -d "$fish_prefix" ]; then
             # this specific prefix (ie: /etc/fish) is very likely to exist

src/libexpr/primops/fromTOML.cc

@@ -3,6 +3,8 @@
 
 #include "../../toml11/toml.hpp"
 
+#include <sstream>
+
 namespace nix {
 
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)
@@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
             case toml::value_t::offset_datetime:
             case toml::value_t::local_date:
             case toml::value_t::local_time:
-                // We fail since Nix doesn't have date and time types
+                {
+                    if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
+                        auto attrs = state.buildBindings(2);
+                        attrs.alloc("_type").mkString("timestamp");
+                        std::ostringstream s;
+                        s << t;
+                        attrs.alloc("value").mkString(s.str());
+                        v.mkAttrs(attrs);
+                    } else {
                         throw std::runtime_error("Dates and times are not supported");
+                    }
+                }
                 break;;
             case toml::value_t::empty:
                 v.mkNull();

src/libstore/build/local-derivation-goal.cc

@@ -1452,7 +1452,7 @@ void LocalDerivationGoal::startDaemon()
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             if (!remote) {
                 if (errno == EINTR || errno == EAGAIN) continue;
-                if (errno == EINVAL) break;
+                if (errno == EINVAL || errno == ECONNABORTED) break;
                 throw SysError("accepting connection");
             }
 
@@ -1482,8 +1482,22 @@ void LocalDerivationGoal::startDaemon()
 
 void LocalDerivationGoal::stopDaemon()
 {
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
+    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) {
+        // According to the POSIX standard, the 'shutdown' function should
+        // return an ENOTCONN error when attempting to shut down a socket that
+        // hasn't been connected yet. This situation occurs when the 'accept'
+        // function is called on a socket without any accepted connections,
+        // leaving the socket unconnected. While Linux doesn't seem to produce
+        // an error for sockets that have only been accepted, more
+        // POSIX-compliant operating systems like OpenBSD, macOS, and others do
+        // return the ENOTCONN error. Therefore, we handle this error here to
+        // avoid raising an exception for compliant behaviour.
+        if (errno == ENOTCONN) {
+            daemonSocket.close();
+        } else {
             throw SysError("shutting down daemon socket");
+        }
+    }
 
     if (daemonThread.joinable())
         daemonThread.join();
@@ -1494,7 +1508,8 @@ void LocalDerivationGoal::stopDaemon()
         thread.join();
     daemonWorkerThreads.clear();
 
-    daemonSocket = -1;
+    // release the socket.
+    daemonSocket.close();
 }
 
 

src/libstore/build/personality.cc

@@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
                 && (std::string_view(SYSTEM) == "x86_64-linux"
                     || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
             || system == "armv7l-linux"
-            || system == "armv6l-linux")
+            || system == "armv6l-linux"
+            || system == "armv5tel-linux")
         {
             if (personality(PER_LINUX32) == -1)
                 throw SysError("cannot set 32-bit personality");

src/libstore/globals.hh

@@ -915,12 +915,11 @@ public:
         this, {}, "hashed-mirrors",
         R"(
           A list of web servers used by `builtins.fetchurl` to obtain files by
-          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+          hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
-          *ht* and a base-16 hash *h*, Nix will try to download the file from
+          download the file from *hashed-mirror*/*ht*/*h*. This allows files to
-          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
+          be downloaded even if they have disappeared from their original URI.
-          they have disappeared from their original URI. For example, given
+          For example, given an example mirror `http://tarballs.nixos.org/`,
-          the default mirror `http://tarballs.nixos.org/`, when building the
+          when building the derivation
-          derivation
 
           ```nix
           builtins.fetchurl {

src/libstore/uds-remote-store.cc

@@ -13,6 +13,14 @@
 
 namespace nix {
 
+std::string UDSRemoteStoreConfig::doc()
+{
+    return
+        #include "uds-remote-store.md"
+        ;
+}
+
+
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
     , LocalFSStoreConfig(params)

src/libstore/uds-remote-store.hh

@@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
 
     const std::string name() override { return "Local Daemon Store"; }
 
-    std::string doc() override
+    std::string doc() override;
-    {
-        return
-          #include "uds-remote-store.md"
-          ;
-    }
 };
 
 class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

src/libutil/experimental-features.cc

@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -214,6 +214,13 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
                 derivations that are themselves derivations outputs.
         )",
     },
+    {
+        .tag = Xp::ParseTomlTimestamps,
+        .name = "parse-toml-timestamps",
+        .description = R"(
+            Allow parsing of timestamps in builtins.fromTOML.
+        )",
+    },
     {
         .tag = Xp::ReadOnlyLocalStore,
         .name = "read-only-local-store",

src/libutil/experimental-features.hh

@@ -30,6 +30,7 @@ enum struct ExperimentalFeature
     DiscardReferences,
     DaemonTrustOverride,
     DynamicDerivations,
+    ParseTomlTimestamps,
     ReadOnlyLocalStore,
 };
 

src/nix/daemon.cc

@@ -24,6 +24,7 @@
 #include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/un.h>
+#include <sys/select.h>
 #include <errno.h>
 #include <pwd.h>
 #include <grp.h>

tests/lang/eval-fail-fromTOML-timestamps.nix

@@ -0,0 +1,130 @@
+builtins.fromTOML ''
+  key = "value"
+  bare_key = "value"
+  bare-key = "value"
+  1234 = "value"
+
+  "127.0.0.1" = "value"
+  "character encoding" = "value"
+  "ʎǝʞ" = "value"
+  'key2' = "value"
+  'quoted "value"' = "value"
+
+  name = "Orange"
+
+  physical.color = "orange"
+  physical.shape = "round"
+  site."google.com" = true
+
+  # This is legal according to the spec, but cpptoml doesn't handle it.
+  #a.b.c = 1
+  #a.d = 2
+
+  str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+  int1 = +99
+  int2 = 42
+  int3 = 0
+  int4 = -17
+  int5 = 1_000
+  int6 = 5_349_221
+  int7 = 1_2_3_4_5
+
+  hex1 = 0xDEADBEEF
+  hex2 = 0xdeadbeef
+  hex3 = 0xdead_beef
+
+  oct1 = 0o01234567
+  oct2 = 0o755
+
+  bin1 = 0b11010110
+
+  flt1 = +1.0
+  flt2 = 3.1415
+  flt3 = -0.01
+  flt4 = 5e+22
+  flt5 = 1e6
+  flt6 = -2E-2
+  flt7 = 6.626e-34
+  flt8 = 9_224_617.445_991_228_313
+
+  bool1 = true
+  bool2 = false
+
+  odt1 = 1979-05-27T07:32:00Z
+  odt2 = 1979-05-27T00:32:00-07:00
+  odt3 = 1979-05-27T00:32:00.999999-07:00
+  odt4 = 1979-05-27 07:32:00Z
+  ldt1 = 1979-05-27T07:32:00
+  ldt2 = 1979-05-27T00:32:00.999999
+  ld1 = 1979-05-27
+  lt1 = 07:32:00
+  lt2 = 00:32:00.999999
+
+  arr1 = [ 1, 2, 3 ]
+  arr2 = [ "red", "yellow", "green" ]
+  arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+  arr4 = [ "all", 'strings', """are the same""", ''''type'''']
+  arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+
+  arr7 = [
+    1, 2, 3
+  ]
+
+  arr8 = [
+    1,
+    2, # this is ok
+  ]
+
+  [table-1]
+  key1 = "some string"
+  key2 = 123
+
+
+  [table-2]
+  key1 = "another string"
+  key2 = 456
+
+  [dog."tater.man"]
+  type.name = "pug"
+
+  [a.b.c]
+  [ d.e.f ]
+  [ g .  h  . i ]
+  [ j . "ʞ" . 'l' ]
+  [x.y.z.w]
+
+  name = { first = "Tom", last = "Preston-Werner" }
+  point = { x = 1, y = 2 }
+  animal = { type.name = "pug" }
+
+  [[products]]
+  name = "Hammer"
+  sku = 738594937
+
+  [[products]]
+
+  [[products]]
+  name = "Nail"
+  sku = 284758393
+  color = "gray"
+
+  [[fruit]]
+    name = "apple"
+
+    [fruit.physical]
+      color = "red"
+      shape = "round"
+
+    [[fruit.variety]]
+      name = "red delicious"
+
+    [[fruit.variety]]
+      name = "granny smith"
+
+  [[fruit]]
+    name = "banana"
+
+    [[fruit.variety]]
+      name = "plantain"
+''

tests/lang/eval-okay-fromTOML-timestamps.exp

@@ -0,0 +1 @@
+{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; }

tests/lang/eval-okay-fromTOML-timestamps.flags

@@ -0,0 +1 @@
+--extra-experimental-features parse-toml-timestamps

tests/lang/eval-okay-fromTOML-timestamps.nix

@@ -0,0 +1,130 @@
+builtins.fromTOML ''
+  key = "value"
+  bare_key = "value"
+  bare-key = "value"
+  1234 = "value"
+
+  "127.0.0.1" = "value"
+  "character encoding" = "value"
+  "ʎǝʞ" = "value"
+  'key2' = "value"
+  'quoted "value"' = "value"
+
+  name = "Orange"
+
+  physical.color = "orange"
+  physical.shape = "round"
+  site."google.com" = true
+
+  # This is legal according to the spec, but cpptoml doesn't handle it.
+  #a.b.c = 1
+  #a.d = 2
+
+  str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+  int1 = +99
+  int2 = 42
+  int3 = 0
+  int4 = -17
+  int5 = 1_000
+  int6 = 5_349_221
+  int7 = 1_2_3_4_5
+
+  hex1 = 0xDEADBEEF
+  hex2 = 0xdeadbeef
+  hex3 = 0xdead_beef
+
+  oct1 = 0o01234567
+  oct2 = 0o755
+
+  bin1 = 0b11010110
+
+  flt1 = +1.0
+  flt2 = 3.1415
+  flt3 = -0.01
+  flt4 = 5e+22
+  flt5 = 1e6
+  flt6 = -2E-2
+  flt7 = 6.626e-34
+  flt8 = 9_224_617.445_991_228_313
+
+  bool1 = true
+  bool2 = false
+
+  odt1 = 1979-05-27T07:32:00Z
+  odt2 = 1979-05-27T00:32:00-07:00
+  odt3 = 1979-05-27T00:32:00.999999-07:00
+  odt4 = 1979-05-27 07:32:00Z
+  ldt1 = 1979-05-27T07:32:00
+  ldt2 = 1979-05-27T00:32:00.999999
+  ld1 = 1979-05-27
+  lt1 = 07:32:00
+  lt2 = 00:32:00.999999
+
+  arr1 = [ 1, 2, 3 ]
+  arr2 = [ "red", "yellow", "green" ]
+  arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+  arr4 = [ "all", 'strings', """are the same""", ''''type'''']
+  arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+
+  arr7 = [
+    1, 2, 3
+  ]
+
+  arr8 = [
+    1,
+    2, # this is ok
+  ]
+
+  [table-1]
+  key1 = "some string"
+  key2 = 123
+
+
+  [table-2]
+  key1 = "another string"
+  key2 = 456
+
+  [dog."tater.man"]
+  type.name = "pug"
+
+  [a.b.c]
+  [ d.e.f ]
+  [ g .  h  . i ]
+  [ j . "ʞ" . 'l' ]
+  [x.y.z.w]
+
+  name = { first = "Tom", last = "Preston-Werner" }
+  point = { x = 1, y = 2 }
+  animal = { type.name = "pug" }
+
+  [[products]]
+  name = "Hammer"
+  sku = 738594937
+
+  [[products]]
+
+  [[products]]
+  name = "Nail"
+  sku = 284758393
+  color = "gray"
+
+  [[fruit]]
+    name = "apple"
+
+    [fruit.physical]
+      color = "red"
+      shape = "round"
+
+    [[fruit.variety]]
+      name = "red delicious"
+
+    [[fruit.variety]]
+      name = "granny smith"
+
+  [[fruit]]
+    name = "banana"
+
+    [[fruit.variety]]
+      name = "plantain"
+''

tests/recursive.sh

@@ -1,8 +1,5 @@
 source common.sh
 
-# FIXME
-if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
-
 enableFeatures 'recursive-nix'
 restartDaemon