Build nix-util with Meson

The idea is two-fold:

- Replace autotools with Meson

- Build each library in its own derivation

The interaction of these two features is that Meson's "subprojects"
feature (https://mesonbuild.com/Subprojects) allows us to have single
dev shell for building all libraries still, while also building things
separately. This allows us to break up the build without a huge
productivity lost.

I tested the Linux native build, and NetBSD and Windows cross builds.

Also do some clean ups of the Flake in the process of supporting new
jobs.

Special thanks to everyone that has worked on a Meson port so far,
@p01arst0rm and @Qyriad in particular.

Co-Authored-By: p01arst0rm <polar@ever3st.com>
Co-Authored-By: Artemis Tosini <lix@artem.ist>
Co-Authored-By: Artemis Tosini <me@artem.ist>
Co-Authored-By: Felix Uhl <felix.uhl@outlook.com>
Co-Authored-By: Jade Lovelace <lix@jade.fyi>
Co-Authored-By: Lunaphied <lunaphied@lunaphied.me>
Co-Authored-By: Maximilian Bosch <maximilian@mbosch.me>
Co-Authored-By: Pierre Bourdon <delroth@gmail.com>
Co-Authored-By: Qyriad <qyriad@qyriad.me>
Co-Authored-By: Rebecca Turner <rbt@sent.as>
Co-Authored-By: Winter <winter@winter.cafe>
Co-Authored-By: eldritch horrors <pennae@lix.systems>
Co-Authored-By: jade <lix@jade.fyi>
Co-Authored-By: julia <midnight@trainwit.ch>
Co-Authored-By: rebecca “wiggles” turner <rbt@sent.as>
Co-Authored-By: wiggles dog <rbt@sent.as>
Co-Authored-By: fricklerhandwerk <valentin@fricklerhandwerk.de>
Co-authored-By: Eli Schwartz <eschwartz93@gmail.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
This commit is contained in:
John Ericson 2024-06-04 09:28:27 -04:00
parent 394286cf5e
commit 28d2af4ea6
12 changed files with 553 additions and 33 deletions

2
.gitignore vendored
View file

@ -11,6 +11,8 @@ perl/Makefile.config
/svn-revision
/libtool
/config/config.*
# Default meson build dir
/build
# /doc/manual/
/doc/manual/*.1

View file

@ -160,21 +160,23 @@
};
});
nix =
let
officialRelease = false;
versionSuffix =
if officialRelease
then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
in final.callPackage ./package.nix {
nix-util = final.callPackage ./src/libutil/package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
};
nix =
final.callPackage ./package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
officialRelease = false;
boehmgc = final.boehmgc-nix;
libgit2 = final.libgit2-nix;
libseccomp = final.libseccomp-nix;
@ -203,7 +205,7 @@
# 'nix.perl-bindings' packages.
overlays.default = overlayFor (p: p.stdenv);
hydraJobs = import ./build/hydra.nix {
hydraJobs = import ./maintainers/hydra.nix {
inherit
inputs
binaryTarball
@ -236,11 +238,29 @@
} // devFlake.checks.${system} or {}
);
packages = forAllSystems (system: rec {
inherit (nixpkgsFor.${system}.native) nix changelog-d;
default = nix;
} // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
nix-static = nixpkgsFor.${system}.static.nix;
packages = forAllSystems (system: {
inherit (nixpkgsFor.${system}.native)
changelog-d;
default = self.packages.${system}.nix;
} // lib.concatMapAttrs
(pkgName: {}: {
"${pkgName}" = nixpkgsFor.${system}.native.${pkgName};
"${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName};
} // lib.concatMapAttrs
(crossSystem: {}: {
"${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName};
})
(lib.genAttrs crossSystems (_: { }))
// lib.concatMapAttrs
(stdenvName: {}: {
"${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName};
})
(lib.genAttrs stdenvs (_: { })))
{
"nix" = { };
"nix-util" = { };
}
// lib.optionalAttrs (builtins.elem system linux64BitSystems) {
dockerImage =
let
pkgs = nixpkgsFor.${system}.native;
@ -255,18 +275,7 @@
ln -s ${image} $image
echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
'';
} // builtins.listToAttrs (map
(crossSystem: {
name = "nix-${crossSystem}";
value = nixpkgsFor.${system}.cross.${crossSystem}.nix;
})
crossSystems)
// builtins.listToAttrs (map
(stdenvName: {
name = "nix-${stdenvName}";
value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix;
})
stdenvs)));
});
devShells = let
makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs:
@ -274,6 +283,11 @@
modular = devFlake.getSystem stdenv.buildPlatform.system;
in {
pname = "shell-for-" + attrs.pname;
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
version = lib.fileContents ./.version;
name = attrs.pname;
installFlags = "sysconfdir=$(out)/etc";
shellHook = ''
PATH=$prefix/bin:$PATH
@ -288,12 +302,19 @@
src = null;
env = {
# Needed for Meson to find Boost.
# https://github.com/NixOS/nixpkgs/issues/86131.
BOOST_INCLUDEDIR = "${lib.getDev pkgs.boost}/include";
BOOST_LIBRARYDIR = "${lib.getLib pkgs.boost}/lib";
# For `make format`, to work without installing pre-commit
_NIX_PRE_COMMIT_HOOKS_CONFIG =
"${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
};
inherit (pkgs.nix-util) mesonFlags;
nativeBuildInputs = attrs.nativeBuildInputs or []
++ pkgs.nix-util.nativeBuildInputs
++ [
modular.pre-commit.settings.package
(pkgs.writeScriptBin "pre-commit-hooks-install"

View file

@ -32,6 +32,8 @@ let
doBuild = false;
};
forAllPackages = lib.genAttrs [ "nix" "nix-util" ];
in
{
# Binary package for various platforms.
@ -39,10 +41,12 @@ in
shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
buildStatic = forAllPackages (pkgName:
lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName}));
buildCross = forAllCrossSystems (crossSystem:
lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}"));
buildCross = forAllPackages (pkgName:
forAllCrossSystems (crossSystem:
lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName})));
buildNoGc = forAllSystems (system:
self.packages.${system}.nix.override { enableGC = false; }
@ -76,7 +80,7 @@ in
binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
forAllCrossSystems (crossSystem:
binaryTarball
self.packages.${system}."nix-${crossSystem}"
nixpkgsFor.${system}.cross.${crossSystem}.nix
nixpkgsFor.${system}.cross.${crossSystem}));
# The first half of the installation script. This is uploaded

9
meson.build Normal file
View file

@ -0,0 +1,9 @@
# This is just a stub project to include all the others as subprojects
# for development shell purposes
project('nix-dev-shell', 'cpp',
version : files('.version'),
subproject_dir : 'src',
)
subproject('libutil')

View file

@ -385,8 +385,7 @@ in {
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
# to work with `strictDeps`.
# TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
strictDeps = !withCoverageChecks;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";

1
src/libutil/.version Symbolic link
View file

@ -0,0 +1 @@
../../.version

View file

@ -0,0 +1,11 @@
sources += files(
'cgroup.cc',
'namespaces.cc',
)
include_dirs += include_directories('.')
headers += files(
'cgroup.hh',
'namespaces.hh',
)

288
src/libutil/meson.build Normal file
View file

@ -0,0 +1,288 @@
project('nix-util', 'cpp',
version : run_command('cat', './.version', check : true).stdout().strip(),
default_options : [
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 0.64.0',
license : 'LGPL-2.1-or-later',
)
cxx = meson.get_compiler('cpp')
deps_private = [ ]
deps_public = [ ]
deps_other = [ ]
configdata = configuration_data()
# Check for each of these functions, and create a define like `#define HAVE_LUTIMES 1`.
check_funcs = [
# Optionally used for changing the mtime of symlinks.
'lutimes',
'pipe2',
'posix_fallocate',
'strsignal',
'sysconf',
]
foreach funcspec : check_funcs
define_name = 'HAVE_' + funcspec.underscorify().to_upper()
define_value = cxx.has_function(funcspec).to_int()
configdata.set(define_name, define_value)
endforeach
# Conditional to work around https://github.com/mesonbuild/meson/issues/13293
if host_machine.system() != 'windows' and cxx.get_id() == 'gcc'
deps_private += dependency('threads')
endif
if host_machine.system() == 'windows'
socket = cxx.find_library('ws2_32')
deps_other += socket
elif host_machine.system() == 'sunos'
socket = cxx.find_library('socket')
network_service_library = cxx.find_library('nsl')
deps_other += [socket, network_service_library]
endif
boost = dependency(
'boost',
modules : ['context', 'coroutine'],
)
# Actually public, but wrong type of dep for pkg config
deps_other += boost
openssl = dependency(
'libcrypto',
'openssl',
version : '>= 1.1.1',
)
deps_private += openssl
libarchive = dependency('libarchive', version : '>= 3.1.2')
deps_public += libarchive
if get_option('default_library') == 'static'
# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
add_project_arguments('-lz', language : 'cpp')
endif
sodium = dependency('libsodium', 'sodium')
deps_private += sodium
brotli = [
dependency('libbrotlicommon'),
dependency('libbrotlidec'),
dependency('libbrotlienc'),
]
deps_private += brotli
# cpuid only makes sense on x86_64
cpuid_required = host_machine.cpu_family() == 'x86_64' ? get_option('cpuid') : false
cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required)
configdata.set('HAVE_LIBCPUID', cpuid.found().to_int())
deps_private += cpuid
nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json
config_util_h = configure_file(
configuration : configdata,
output : 'config-util.h',
)
add_project_arguments(
# TODO(Qyriad): Yes this is how the autoconf+Make system did it.
# It would be nice for our headers to be idempotent instead.
'-include', 'config-util.h',
'-Wno-deprecated-declarations',
'-Wimplicit-fallthrough',
'-Werror=switch',
'-Werror=switch-enum',
'-Wdeprecated-copy',
'-Wignored-qualifiers',
# Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
# at ~1% overhead in `nix search`.
#
# FIXME: remove when we get meson 1.4.0 which will default this to on for us:
# https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions
'-D_GLIBCXX_ASSERTIONS=1',
language : 'cpp',
)
sources = files(
'archive.cc',
'args.cc',
'canon-path.cc',
'compression.cc',
'compute-levels.cc',
'config.cc',
'current-process.cc',
'english.cc',
'environment-variables.cc',
'error.cc',
'exit.cc',
'experimental-features.cc',
'file-content-address.cc',
'file-descriptor.cc',
'file-system.cc',
'fs-sink.cc',
'git.cc',
'hash.cc',
'hilite.cc',
'json-utils.cc',
'logging.cc',
'memory-source-accessor.cc',
'position.cc',
'posix-source-accessor.cc',
'references.cc',
'serialise.cc',
'signature/local-keys.cc',
'signature/signer.cc',
'source-accessor.cc',
'source-path.cc',
'suggestions.cc',
'tarfile.cc',
'terminal.cc',
'thread-pool.cc',
'unix-domain-socket.cc',
'url.cc',
'users.cc',
'util.cc',
'xml-writer.cc',
)
include_dirs = [include_directories('.')]
headers = [config_util_h] + files(
'abstract-setting-to-json.hh',
'ansicolor.hh',
'archive.hh',
'args.hh',
'args/root.hh',
'callback.hh',
'canon-path.hh',
'chunked-vector.hh',
'closure.hh',
'comparator.hh',
'compression.hh',
'compute-levels.hh',
'config-impl.hh',
'config.hh',
'current-process.hh',
'english.hh',
'environment-variables.hh',
'error.hh',
'exit.hh',
'experimental-features.hh',
'file-content-address.hh',
'file-descriptor.hh',
'file-path-impl.hh',
'file-path.hh',
'file-system.hh',
'finally.hh',
'fmt.hh',
'fs-sink.hh',
'git.hh',
'hash.hh',
'hilite.hh',
'json-impls.hh',
'json-utils.hh',
'logging.hh',
'lru-cache.hh',
'memory-source-accessor.hh',
'muxable-pipe.hh',
'pool.hh',
'position.hh',
'posix-source-accessor.hh',
'processes.hh',
'ref.hh',
'references.hh',
'regex-combinators.hh',
'repair-flag.hh',
'serialise.hh',
'signals.hh',
'signature/local-keys.hh',
'signature/signer.hh',
'source-accessor.hh',
'source-path.hh',
'split.hh',
'suggestions.hh',
'sync.hh',
'tarfile.hh',
'terminal.hh',
'thread-pool.hh',
'topo-sort.hh',
'types.hh',
'unix-domain-socket.hh',
'url-parts.hh',
'url.hh',
'users.hh',
'util.hh',
'variant-wrapper.hh',
'xml-writer.hh',
)
if host_machine.system() == 'linux'
subdir('linux')
endif
if host_machine.system() == 'windows'
subdir('windows')
else
subdir('unix')
endif
if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
# Windows DLLs are stricter about symbol visibility than Unix shared
# objects --- see https://gcc.gnu.org/wiki/Visibility for details.
# This is a temporary sledgehammer to export everything like on Unix,
# and not detail with this yet.
#
# TODO do not do this, and instead do fine-grained export annotations.
linker_export_flags = ['-Wl,--export-all-symbols']
else
linker_export_flags = []
endif
libutil = library(
'nixutil',
sources,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
link_args: linker_export_flags,
install : true,
)
install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = ['-lboost_context', '-lboost_coroutine']
if host_machine.system() == 'windows'
# `libraries_private` cannot contain ad-hoc dependencies (from
# `find_library), so we need to do this manually
libraries_private += ['-lws2_32']
endif
import('pkgconfig').generate(
libutil,
filebase : 'nix-util',
name : 'Nix',
description : 'Nix Package Manager',
subdirs : ['nix'],
extra_cflags : ['-std=c++2a'],
requires : deps_public,
requires_private : deps_private,
# avoid absolute paths, use vendored ones
libraries_private : libraries_private,
)
nix_util = declare_dependency(
include_directories : include_dirs,
link_with : libutil,
compile_args : ['-std=c++2a'],
dependencies : [],
)
meson.override_dependency('nix-util', nix_util)

View file

@ -0,0 +1,5 @@
# vim: filetype=meson
option('cpuid', type : 'feature',
description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)',
)

144
src/libutil/package.nix Normal file
View file

@ -0,0 +1,144 @@
{ lib
, stdenv
, releaseTools
, fileset
, meson
, ninja
, pkg-config
, boost
, brotli
, libarchive
, libcpuid
, libsodium
, nlohmann_json
, openssl
# Configuration Options
, versionSuffix ? ""
, officialRelease ? false
# Check test coverage of Nix. Probably want to use with at least
# one of `doCheck` or `doInstallCheck` enabled.
, withCoverageChecks ? false
}:
let
version = lib.fileContents ./.version + versionSuffix;
mkDerivation =
if withCoverageChecks
then
# TODO support `finalAttrs` args function in
# `releaseTools.coverageAnalysis`.
argsFun:
releaseTools.coverageAnalysis (let args = argsFun args; in args)
else stdenv.mkDerivation;
in
mkDerivation (finalAttrs: {
pname = "nix-util";
inherit version;
src = fileset.toSource {
root = ./.;
fileset = fileset.unions [
./meson.build
./meson.options
./linux/meson.build
./unix/meson.build
./windows/meson.build
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
];
};
outputs = [ "out" "dev" ];
nativeBuildInputs = [
meson
ninja
pkg-config
];
buildInputs = [
boost
brotli
libsodium
openssl
] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
;
propagatedBuildInputs = [
libarchive
nlohmann_json
];
disallowedReferences = [ boost ];
preConfigure =
# "Inline" .version so it's not a symlink, and includes the suffix
''
echo ${version} > .version
''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
+ lib.optionalString (!stdenv.hostPlatform.isStatic) (''
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
'' + lib.optionalString stdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
'' + lib.optionalString stdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''
);
env = {
# Needed for Meson to find Boost.
# https://github.com/NixOS/nixpkgs/issues/86131.
BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
} // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
enableParallelBuilding = true;
postInstall =
# Remove absolute path to boost libs
''
sed -i "$out/lib/pkgconfig/nix-util.pc" -e 's, ${lib.getLib boost}[^ ]*,,g'
''
+ lib.optionalString stdenv.isDarwin ''
install_name_tool \
-change ${boost}/lib/libboost_context.dylib \
$out/lib/libboost_context.dylib \
$out/lib/libnixutil.dylib
'';
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
strictDeps = !withCoverageChecks;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};
} // lib.optionalAttrs withCoverageChecks {
lcovFilter = [ "*/boost/*" "*-tab.*" ];
hardeningDisable = [ "fortify" ];
})

View file

@ -0,0 +1,17 @@
sources += files(
'environment-variables.cc',
'file-descriptor.cc',
'file-path.cc',
'file-system.cc',
'muxable-pipe.cc',
'processes.cc',
'signals.cc',
'users.cc',
)
include_dirs += include_directories('.')
headers += files(
'monitor-fd.hh',
'signals-impl.hh',
)

View file

@ -0,0 +1,19 @@
sources += files(
'environment-variables.cc',
'file-descriptor.cc',
'file-path.cc',
'file-system.cc',
'muxable-pipe.cc',
'processes.cc',
'users.cc',
'windows-async-pipe.cc',
'windows-error.cc',
)
include_dirs += include_directories('.')
headers += files(
'signals-impl.hh',
'windows-async-pipe.hh',
'windows-error.hh',
)