From 15833516a4bad0a4ae7786293b22df4bf650aa80 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 5 Feb 2019 16:42:45 -0500 Subject: [PATCH 1/8] Add armv6l-linux & armv7l-linux as cross jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a cheap way to get 32-bit ARM working. We don’t support it officially but lots of people have raspberry pis and similar hardware they want to install the Nix package manager on. --- release.nix | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/release.nix b/release.nix index 271645067..0ec742906 100644 --- a/release.nix +++ b/release.nix @@ -2,6 +2,7 @@ , nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, crossSystems ? [ "armv6l-linux" "armv7l-linux" ] }: let @@ -53,11 +54,12 @@ let }; - build = pkgs.lib.genAttrs systems (system: + build = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; with import ./release-common.nix { inherit pkgs; }; @@ -89,9 +91,12 @@ let }); - perlBindings = pkgs.lib.genAttrs systems (system: + perlBindings = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; releaseTools.nixBuild { name = "nix-perl"; @@ -112,9 +117,12 @@ let }); - binaryTarball = pkgs.lib.genAttrs systems (system: + binaryTarball = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - with import nixpkgs { inherit system; }; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; let toplevel = builtins.getAttr system jobs.build; From 1996af425ac8ddea1e8a591650e7d0caba2aa201 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 21:43:47 -0500 Subject: [PATCH 2/8] Use buildPackages for native dependencies Unfortunately, releaseTools.nixBuild does not separate native and non-native build inputs. As an alternative, we can just use buildPackages to get the native version of some packages like: - pkgconfig - git - curl - utillinux --- release-common.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release-common.nix b/release-common.nix index 4c5565985..f2aa57c2c 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,14 +50,16 @@ rec { buildDeps = [ curl bzip2 xz brotli editline - openssl pkgconfig sqlite boehmgc + openssl sqlite boehmgc boost + buildPackages.pkgconfig + # Tests - git - mercurial + buildPackages.git + buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { From e9072ded9749ab00cc397980e8a26f83d341efc0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 22:43:28 -0500 Subject: [PATCH 3/8] Use nativeBuildInputs --- release-common.nix | 16 +++++++++------- release.nix | 3 +++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/release-common.nix b/release-common.nix index f2aa57c2c..707d36f95 100644 --- a/release-common.nix +++ b/release-common.nix @@ -47,19 +47,21 @@ rec { autoreconfHook ]; + nativeBuildDeps = + [ buildPackages.pkgconfig + + # Tests + buildPackages.git + buildPackages.mercurial + ] ++ lib.optional stdenv.isLinux buildPackages.utillinuxMinimal; + buildDeps = [ curl bzip2 xz brotli editline openssl sqlite boehmgc boost - - buildPackages.pkgconfig - - # Tests - buildPackages.git - buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { diff --git a/release.nix b/release.nix index 0ec742906..9843efa29 100644 --- a/release.nix +++ b/release.nix @@ -24,6 +24,7 @@ let src = nix; inherit officialRelease; + nativeBuildInputs = nativeBuildDeps; buildInputs = tarballDeps ++ buildDeps; configureFlags = "--enable-gc"; @@ -67,6 +68,7 @@ let name = "nix"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; preConfigure = @@ -199,6 +201,7 @@ let name = "nix-build"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; dontInstall = false; From f6ea56dfac1f4df45a5fa9e2801bc632dee9eff7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 23:04:40 -0500 Subject: [PATCH 4/8] Get shellcheck from buildPackages --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 9843efa29..06db7bdc0 100644 --- a/release.nix +++ b/release.nix @@ -133,7 +133,7 @@ let in runCommand "nix-binary-tarball-${version}" - { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + { nativeBuildInputs = lib.optional (system != "aarch64-linux") buildPackages.shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' From 4fefe26717fa70828e3f524e43c76e3f7b7a09b0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 5 Feb 2021 18:22:34 -0600 Subject: [PATCH 5/8] Re-enable armv6l support This fixes the libatomic detection. --- configure.ac | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 2047ed8d2..685c471c5 100644 --- a/configure.ac +++ b/configure.ac @@ -152,7 +152,7 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="$LDFLAGS -latomic" fi PKG_PROG_PKG_CONFIG diff --git a/flake.nix b/flake.nix index 869b92cb7..7e02fd70d 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" ]; - crossSystems = [ "armv7l-linux" ]; + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); From c906d6530dc52d79f69e65e0447893084a0d4556 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jun 2021 00:12:03 -0500 Subject: [PATCH 6/8] Support cross-compiling binaryTarball --- flake.nix | 164 ++++++++++++++++++---------------- maintainers/upload-release.pl | 2 + scripts/install.in | 10 +++ 3 files changed, 99 insertions(+), 77 deletions(-) diff --git a/flake.nix b/flake.nix index 829cd2804..08b385ec2 100644 --- a/flake.nix +++ b/flake.nix @@ -135,10 +135,11 @@ substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: - '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + (system: let + tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; + in '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) systems @@ -176,6 +177,77 @@ }; + binaryTarball = buildPackages: nix: pkgs: let + inherit (pkgs) cacert; + installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; + in + + buildPackages.runCommand "nix-binary-tarball-${version}" + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${pkgs.system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -315,85 +387,23 @@ # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: + binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system}); - with nixpkgsFor.${system}; - - let - installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; }; - in - - runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''); + binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: { + name = crossSystem; + value = let + nixpkgsCross = import nixpkgs { + inherit system crossSystem; + overlays = [ self.overlay ]; + }; + in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; + }) crossSystems)); # The first half of the installation script. This is uploaded # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index c2933300f..77007f914 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -110,6 +110,8 @@ downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); downloadFile("installerScript", "1"); for my $fn (glob "$tmpDir/*") { diff --git a/scripts/install.in b/scripts/install.in index 39016d161..e801d4268 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -40,6 +40,16 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-linux@ system=aarch64-linux ;; + Linux.armv6l_linux) + hash=@tarballHash_armv6l-linux@ + path=@tarballPath_armv6l-linux@ + system=armv6l-linux + ;; + Linux.armv7l_linux) + hash=@tarballHash_armv7l-linux@ + path=@tarballPath_armv7l-linux@ + system=armv7l-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ From 580583e0b386df181a3c30519c821275362b10e0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jun 2021 00:14:54 -0500 Subject: [PATCH 7/8] Build cross-compilation in gha --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 08b385ec2..8b37a5658 100644 --- a/flake.nix +++ b/flake.nix @@ -404,7 +404,7 @@ # tarball for the user's system and calls the second half of the # installation script. installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; # Line coverage analysis. coverage = From 6c13a3f7354d02fb3a38604ed601b76c011e826f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 28 Jun 2021 15:08:17 -0500 Subject: [PATCH 8/8] Support binaryTarballCross in gha --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8b37a5658..c97130ac9 100644 --- a/flake.nix +++ b/flake.nix @@ -387,7 +387,7 @@ # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system}); + binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: { name = crossSystem; @@ -498,7 +498,10 @@ # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; } "touch $out"; - }); + } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: { + name = "binaryTarball-${crossSystem}"; + value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem}; + }) crossSystems)) else {})); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix;