Merge pull request #10855 from NixOS/meson-libutil

Build `nix-util` with Meson

.gitignore

@@ -11,6 +11,8 @@ perl/Makefile.config
 /svn-revision
 /libtool
 /config/config.*
+# Default meson build dir
+/build
 
 # /doc/manual/
 /doc/manual/*.1

flake.nix

@@ -160,21 +160,23 @@
             };
           });
 
-          nix =
+          nix-util = final.callPackage ./src/libutil/package.nix {
-            let
+            inherit
-              officialRelease = false;
+              fileset
-              versionSuffix =
+              stdenv
-                if officialRelease
+              officialRelease
-                then ""
+              versionSuffix
-                else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
+              ;
+          };
 
-            in final.callPackage ./package.nix {
+          nix =
+            final.callPackage ./package.nix {
               inherit
                 fileset
                 stdenv
+                officialRelease
                 versionSuffix
                 ;
-              officialRelease = false;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               libseccomp = final.libseccomp-nix;
@@ -203,7 +205,7 @@
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
 
-      hydraJobs = import ./build/hydra.nix {
+      hydraJobs = import ./maintainers/hydra.nix {
         inherit
           inputs
           binaryTarball
@@ -236,11 +238,29 @@
       } // devFlake.checks.${system} or {}
       );
 
-      packages = forAllSystems (system: rec {
+      packages = forAllSystems (system: {
-        inherit (nixpkgsFor.${system}.native) nix changelog-d;
+        inherit (nixpkgsFor.${system}.native)
-        default = nix;
+          changelog-d;
-      } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
+        default = self.packages.${system}.nix;
-        nix-static = nixpkgsFor.${system}.static.nix;
+      } // lib.concatMapAttrs
+        (pkgName: {}: {
+          "${pkgName}" = nixpkgsFor.${system}.native.${pkgName};
+          "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName};
+        } // lib.concatMapAttrs
+          (crossSystem: {}: {
+            "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName};
+          })
+          (lib.genAttrs crossSystems (_: { }))
+        // lib.concatMapAttrs
+          (stdenvName: {}: {
+            "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName};
+          })
+          (lib.genAttrs stdenvs (_: { })))
+        {
+          "nix" = { };
+          "nix-util" = { };
+        }
+        // lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         dockerImage =
           let
             pkgs = nixpkgsFor.${system}.native;
@@ -255,18 +275,7 @@
               ln -s ${image} $image
               echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
             '';
-      } // builtins.listToAttrs (map
+      });
-          (crossSystem: {
-            name = "nix-${crossSystem}";
-            value = nixpkgsFor.${system}.cross.${crossSystem}.nix;
-          })
-          crossSystems)
-        // builtins.listToAttrs (map
-          (stdenvName: {
-            name = "nix-${stdenvName}";
-            value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix;
-          })
-          stdenvs)));
 
       devShells = let
         makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs:
@@ -274,6 +283,11 @@
           modular = devFlake.getSystem stdenv.buildPlatform.system;
         in {
           pname = "shell-for-" + attrs.pname;
+
+          # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
+          version = lib.fileContents ./.version;
+          name = attrs.pname;
+
           installFlags = "sysconfdir=$(out)/etc";
           shellHook = ''
             PATH=$prefix/bin:$PATH
@@ -288,12 +302,19 @@
           src = null;
 
           env = {
+            # Needed for Meson to find Boost.
+            # https://github.com/NixOS/nixpkgs/issues/86131.
+            BOOST_INCLUDEDIR = "${lib.getDev pkgs.boost}/include";
+            BOOST_LIBRARYDIR = "${lib.getLib pkgs.boost}/lib";
             # For `make format`, to work without installing pre-commit
             _NIX_PRE_COMMIT_HOOKS_CONFIG =
               "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
           };
 
+          inherit (pkgs.nix-util) mesonFlags;
+
           nativeBuildInputs = attrs.nativeBuildInputs or []
+            ++ pkgs.nix-util.nativeBuildInputs
             ++ [
               modular.pre-commit.settings.package
               (pkgs.writeScriptBin "pre-commit-hooks-install"

maintainers/hydra.nix

@@ -32,6 +32,8 @@ let
 
       doBuild = false;
     };
+
+  forAllPackages = lib.genAttrs [ "nix" "nix-util" ];
 in
 {
   # Binary package for various platforms.
@@ -39,10 +41,12 @@ in
 
   shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
 
-  buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
+  buildStatic = forAllPackages (pkgName:
+    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName}));
 
-  buildCross = forAllCrossSystems (crossSystem:
+  buildCross = forAllPackages (pkgName:
-    lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}"));
+    forAllCrossSystems (crossSystem:
+      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName})));
 
   buildNoGc = forAllSystems (system:
     self.packages.${system}.nix.override { enableGC = false; }
@@ -76,7 +80,7 @@ in
   binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
     forAllCrossSystems (crossSystem:
       binaryTarball
-        self.packages.${system}."nix-${crossSystem}"
+        nixpkgsFor.${system}.cross.${crossSystem}.nix
         nixpkgsFor.${system}.cross.${crossSystem}));
 
   # The first half of the installation script. This is uploaded

meson.build

@@ -0,0 +1,9 @@
+# This is just a stub project to include all the others as subprojects
+# for development shell purposes
+
+project('nix-dev-shell', 'cpp',
+  version : files('.version'),
+  subproject_dir : 'src',
+)
+
+subproject('libutil')

package.nix

@@ -385,8 +385,7 @@ in {
 
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 
-  # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
+  # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
-  # to work with `strictDeps`.
   strictDeps = !withCoverageChecks;
 
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";

src/libutil/.version

@@ -0,0 +1 @@
+../../.version

src/libutil/linux/meson.build

@@ -0,0 +1,11 @@
+sources += files(
+  'cgroup.cc',
+  'namespaces.cc',
+)
+
+include_dirs += include_directories('.')
+
+headers += files(
+  'cgroup.hh',
+  'namespaces.hh',
+)

src/libutil/meson.build

@@ -0,0 +1,288 @@
+project('nix-util', 'cpp',
+  version : run_command('cat', './.version', check : true).stdout().strip(),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 0.64.0',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+deps_private = [ ]
+deps_public = [ ]
+deps_other = [ ]
+
+configdata = configuration_data()
+
+# Check for each of these functions, and create a define like `#define HAVE_LUTIMES 1`.
+check_funcs = [
+   # Optionally used for changing the mtime of symlinks.
+  'lutimes',
+  'pipe2',
+  'posix_fallocate',
+  'strsignal',
+  'sysconf',
+]
+foreach funcspec : check_funcs
+  define_name = 'HAVE_' + funcspec.underscorify().to_upper()
+  define_value = cxx.has_function(funcspec).to_int()
+  configdata.set(define_name, define_value)
+endforeach
+
+# Conditional to work around https://github.com/mesonbuild/meson/issues/13293
+if host_machine.system() != 'windows' and cxx.get_id() == 'gcc'
+  deps_private += dependency('threads')
+endif
+
+if host_machine.system() == 'windows'
+  socket = cxx.find_library('ws2_32')
+  deps_other += socket
+elif host_machine.system() == 'sunos'
+  socket = cxx.find_library('socket')
+  network_service_library = cxx.find_library('nsl')
+  deps_other += [socket, network_service_library]
+endif
+
+boost = dependency(
+  'boost',
+  modules : ['context', 'coroutine'],
+)
+# Actually public, but wrong type of dep for pkg config
+deps_other += boost
+
+openssl = dependency(
+  'libcrypto',
+  'openssl',
+  version : '>= 1.1.1',
+)
+deps_private += openssl
+
+libarchive = dependency('libarchive', version : '>= 3.1.2')
+deps_public += libarchive
+if get_option('default_library') == 'static'
+  # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
+  add_project_arguments('-lz', language : 'cpp')
+endif
+
+sodium = dependency('libsodium', 'sodium')
+deps_private += sodium
+
+brotli = [
+  dependency('libbrotlicommon'),
+  dependency('libbrotlidec'),
+  dependency('libbrotlienc'),
+]
+deps_private += brotli
+
+# cpuid only makes sense on x86_64
+cpuid_required = host_machine.cpu_family() == 'x86_64' ? get_option('cpuid') : false
+cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required)
+configdata.set('HAVE_LIBCPUID', cpuid.found().to_int())
+deps_private += cpuid
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+config_util_h = configure_file(
+  configuration : configdata,
+  output : 'config-util.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.h',
+  '-Wno-deprecated-declarations',
+  '-Wimplicit-fallthrough',
+  '-Werror=switch',
+  '-Werror=switch-enum',
+  '-Wdeprecated-copy',
+  '-Wignored-qualifiers',
+  # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
+  # at ~1% overhead in `nix search`.
+  #
+  # FIXME: remove when we get meson 1.4.0 which will default this to on for us:
+  # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions
+  '-D_GLIBCXX_ASSERTIONS=1',
+  language : 'cpp',
+)
+
+sources = files(
+  'archive.cc',
+  'args.cc',
+  'canon-path.cc',
+  'compression.cc',
+  'compute-levels.cc',
+  'config.cc',
+  'current-process.cc',
+  'english.cc',
+  'environment-variables.cc',
+  'error.cc',
+  'exit.cc',
+  'experimental-features.cc',
+  'file-content-address.cc',
+  'file-descriptor.cc',
+  'file-system.cc',
+  'fs-sink.cc',
+  'git.cc',
+  'hash.cc',
+  'hilite.cc',
+  'json-utils.cc',
+  'logging.cc',
+  'memory-source-accessor.cc',
+  'position.cc',
+  'posix-source-accessor.cc',
+  'references.cc',
+  'serialise.cc',
+  'signature/local-keys.cc',
+  'signature/signer.cc',
+  'source-accessor.cc',
+  'source-path.cc',
+  'suggestions.cc',
+  'tarfile.cc',
+  'terminal.cc',
+  'thread-pool.cc',
+  'unix-domain-socket.cc',
+  'url.cc',
+  'users.cc',
+  'util.cc',
+  'xml-writer.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_util_h] + files(
+  'abstract-setting-to-json.hh',
+  'ansicolor.hh',
+  'archive.hh',
+  'args.hh',
+  'args/root.hh',
+  'callback.hh',
+  'canon-path.hh',
+  'chunked-vector.hh',
+  'closure.hh',
+  'comparator.hh',
+  'compression.hh',
+  'compute-levels.hh',
+  'config-impl.hh',
+  'config.hh',
+  'current-process.hh',
+  'english.hh',
+  'environment-variables.hh',
+  'error.hh',
+  'exit.hh',
+  'experimental-features.hh',
+  'file-content-address.hh',
+  'file-descriptor.hh',
+  'file-path-impl.hh',
+  'file-path.hh',
+  'file-system.hh',
+  'finally.hh',
+  'fmt.hh',
+  'fs-sink.hh',
+  'git.hh',
+  'hash.hh',
+  'hilite.hh',
+  'json-impls.hh',
+  'json-utils.hh',
+  'logging.hh',
+  'lru-cache.hh',
+  'memory-source-accessor.hh',
+  'muxable-pipe.hh',
+  'pool.hh',
+  'position.hh',
+  'posix-source-accessor.hh',
+  'processes.hh',
+  'ref.hh',
+  'references.hh',
+  'regex-combinators.hh',
+  'repair-flag.hh',
+  'serialise.hh',
+  'signals.hh',
+  'signature/local-keys.hh',
+  'signature/signer.hh',
+  'source-accessor.hh',
+  'source-path.hh',
+  'split.hh',
+  'suggestions.hh',
+  'sync.hh',
+  'tarfile.hh',
+  'terminal.hh',
+  'thread-pool.hh',
+  'topo-sort.hh',
+  'types.hh',
+  'unix-domain-socket.hh',
+  'url-parts.hh',
+  'url.hh',
+  'users.hh',
+  'util.hh',
+  'variant-wrapper.hh',
+  'xml-writer.hh',
+)
+
+if host_machine.system() == 'linux'
+  subdir('linux')
+endif
+
+if host_machine.system() == 'windows'
+  subdir('windows')
+else
+  subdir('unix')
+endif
+
+if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  linker_export_flags = ['-Wl,--export-all-symbols']
+else
+  linker_export_flags = []
+endif
+
+libutil = library(
+  'nixutil',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = ['-lboost_context', '-lboost_coroutine']
+if host_machine.system() == 'windows'
+  # `libraries_private` cannot contain ad-hoc dependencies (from
+  # `find_library), so we need to do this manually
+  libraries_private += ['-lws2_32']
+endif
+
+import('pkgconfig').generate(
+  libutil,
+  filebase : 'nix-util',
+  name : 'Nix',
+  description : 'Nix Package Manager',
+  subdirs : ['nix'],
+  extra_cflags : ['-std=c++2a'],
+  requires : deps_public,
+  requires_private : deps_private,
+  # avoid absolute paths, use vendored ones
+  libraries_private : libraries_private,
+)
+
+nix_util = declare_dependency(
+  include_directories : include_dirs,
+  link_with : libutil,
+  compile_args : ['-std=c++2a'],
+  dependencies : [],
+)
+meson.override_dependency('nix-util', nix_util)

src/libutil/meson.options

@@ -0,0 +1,5 @@
+# vim: filetype=meson
+
+option('cpuid', type : 'feature',
+  description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)',
+)

src/libutil/package.nix

@@ -0,0 +1,144 @@
+{ lib
+, stdenv
+, releaseTools
+, fileset
+
+, meson
+, ninja
+, pkg-config
+
+, boost
+, brotli
+, libarchive
+, libcpuid
+, libsodium
+, nlohmann_json
+, openssl
+
+# Configuration Options
+
+, versionSuffix ? ""
+, officialRelease ? false
+
+# Check test coverage of Nix. Probably want to use with at least
+# one of `doCheck` or `doInstallCheck` enabled.
+, withCoverageChecks ? false
+}:
+
+let
+  version = lib.fileContents ./.version + versionSuffix;
+
+  mkDerivation =
+    if withCoverageChecks
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
+    else stdenv.mkDerivation;
+in
+
+mkDerivation (finalAttrs: {
+  pname = "nix-util";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions [
+      ./meson.build
+      ./meson.options
+      ./linux/meson.build
+      ./unix/meson.build
+      ./windows/meson.build
+      (fileset.fileFilter (file: file.hasExt "cc") ./.)
+      (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    ];
+  };
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    boost
+    brotli
+    libsodium
+    openssl
+  ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
+  ;
+
+  propagatedBuildInputs = [
+    libarchive
+    nlohmann_json
+  ];
+
+  disallowedReferences = [ boost ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix
+    ''
+      echo ${version} > .version
+    ''
+    # Copy libboost_context so we don't get all of Boost in our closure.
+
+    # https://github.com/NixOS/nixpkgs/issues/45462
+    + lib.optionalString (!stdenv.hostPlatform.isStatic) (''
+      mkdir -p $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+      rm -f $out/lib/*.a
+    '' + lib.optionalString stdenv.hostPlatform.isLinux ''
+      chmod u+w $out/lib/*.so.*
+      patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+    '' + lib.optionalString stdenv.hostPlatform.isDarwin ''
+      for LIB in $out/lib/*.dylib; do
+        chmod u+w $LIB
+        install_name_tool -id $LIB $LIB
+        install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+      done
+      install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+    ''
+  );
+
+  env = {
+    # Needed for Meson to find Boost.
+    # https://github.com/NixOS/nixpkgs/issues/86131.
+    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
+    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
+  } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  postInstall =
+    # Remove absolute path to boost libs
+    ''
+      sed -i "$out/lib/pkgconfig/nix-util.pc" -e 's, ${lib.getLib boost}[^ ]*,,g'
+    ''
+    + lib.optionalString stdenv.isDarwin ''
+      install_name_tool \
+      -change ${boost}/lib/libboost_context.dylib \
+      $out/lib/libboost_context.dylib \
+      $out/lib/libnixutil.dylib
+    '';
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
+  strictDeps = !withCoverageChecks;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = [ "fortify" ];
+})

src/libutil/unix/meson.build

@@ -0,0 +1,17 @@
+sources += files(
+  'environment-variables.cc',
+  'file-descriptor.cc',
+  'file-path.cc',
+  'file-system.cc',
+  'muxable-pipe.cc',
+  'processes.cc',
+  'signals.cc',
+  'users.cc',
+)
+
+include_dirs += include_directories('.')
+
+headers += files(
+  'monitor-fd.hh',
+  'signals-impl.hh',
+)

src/libutil/windows/meson.build

@@ -0,0 +1,19 @@
+sources += files(
+  'environment-variables.cc',
+  'file-descriptor.cc',
+  'file-path.cc',
+  'file-system.cc',
+  'muxable-pipe.cc',
+  'processes.cc',
+  'users.cc',
+  'windows-async-pipe.cc',
+  'windows-error.cc',
+)
+
+include_dirs += include_directories('.')
+
+headers += files(
+  'signals-impl.hh',
+  'windows-async-pipe.hh',
+  'windows-error.hh',
+)