Merge pull request #9393 from hercules-ci/changelog-d

Automatically compile hand-written release notes with `changelog-d`

.gitignore

@@ -21,12 +21,14 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh

doc/manual/local.mk

@@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
@@ -144,6 +144,24 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
+
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -177,7 +195,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/rl-next/config

@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix

doc/manual/rl-next/mounted-ssh-store.md

@@ -0,0 +1,9 @@
+synopsis: Mounted SSH Store
+issues: #7890
+prs: #7912
+description: {
+
+Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+}

doc/manual/rl-next/nix-env-json-drv-path.md

@@ -0,0 +1,9 @@
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: #9257
+description: {
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+}
+
+

doc/manual/src/SUMMARY.md.in

@@ -114,7 +114,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

doc/manual/src/contributing/hacking.md

@@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+synopsis: Basically a title
+issues: #1234
+prs: #1238
+description: {
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+
+}
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.

doc/manual/src/release-notes/rl-next.md

@@ -1,6 +0,0 @@
-# Release X.Y (202?-??-??)
-
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

flake.lock

@@ -50,11 +50,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700342017,
+        "lastModified": 1700748986,
-        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "decdf666c833a325cb4417041a90681499e06a41",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -14,6 +14,9 @@
 
       officialRelease = false;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -173,6 +176,8 @@
           "--enable-internal-api-docs"
         ];
 
+        changelog-d = pkgs.buildPackages.changelog-d;
+
         nativeBuildDeps =
           [
             buildPackages.bison
@@ -190,7 +195,10 @@
             buildPackages.jq # Also for custom mdBook preprocessor.
             buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
+          # Official releases don't have rl-next, so we don't need to compile a changelog
+          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
+          ;
 
         buildDeps =
           [ curl
@@ -735,6 +743,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
+              # We want changelog-d in the shell even if the current build doesn't need it
+              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

maintainers/release-notes

@@ -0,0 +1,179 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#                                   ^^^^^^^
+#                Only used for bash. shell.nix goes to the flake.
+
+# --- CONFIGURATION ---
+
+# This does double duty for
+# - including rl-next
+# - marking where to insert new links (right after)
+SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}'
+
+# --- LIB ---
+
+log() {
+  echo 1>&2 "release-notes:" "$@"
+}
+logcmd() {
+  local cmd="$1"
+  shift
+  logcmd2 "$cmd" "${*@Q}" "$cmd" "$@"
+}
+logcmd2() {
+  local fakecmd="$1"
+  local fakeargs="$2"
+  shift
+  shift
+  printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m "
+  echo "$fakeargs" 1>&2
+  "$@"
+}
+die() {
+  # ANSI red
+  printf 1>&2 "release-notes: \033[31;1merror:\033[0m"
+  echo 1>&2 "" "$@"
+  exit 1
+}
+confirm() {
+  local answer
+  echo 1>&2 "$@" "[y/n]"
+  read -r answer
+  case "$answer" in
+    y|Y|yes|Yes|YES)
+      return 0
+      ;;
+    n|N|no|No|NO)
+      return 1
+      ;;
+    *)
+      echo 1>&2 "please answer y or n"
+      confirm "$@"
+      ;;
+  esac
+}
+report_done() {
+  logcmd2 "git" "show" git -c pager.show=false show
+  printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n"
+}
+
+# --- PARSE ARGS ---
+
+if [[ $# -gt 0 ]]; then
+  die "Release notes takes no arguments, but make sure to set VERSION."
+fi
+
+# --- CHECKS ---
+
+if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then
+  die "must run in repo root"
+  exit 1
+fi
+
+# repo must be clean
+if ! git diff --quiet; then
+  die "repo is dirty, please commit or stash changes"
+fi
+
+if ! git diff --quiet --cached; then
+  die "repo has staged changes, please commit or stash them"
+fi
+
+if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then
+  # would have been nice to catch this early, but won't be worth the extra infra
+  die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script."
+fi
+
+if [[ ! -n "${VERSION:-}" ]]; then
+  die "please set the VERSION environment variable before invoking this script"
+  exit 1
+fi
+
+# version_major_minor:      MAJOR.MINOR
+# version_full:             MAJOR.MINOR.PATCH
+# IS_PATCH:                 true if this is a patch release; append instead of create
+if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is minor'
+  IS_PATCH=false
+  version_full="$VERSION.0"
+  version_major_minor="$VERSION"
+elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
+  log 'is minor (.0)'
+  IS_PATCH=false
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')"
+elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is patch'
+  IS_PATCH=true
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+else
+  die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
+fi
+
+unset VERSION
+
+log "version_major_minor=$version_major_minor"
+log "version_full=$version_full"
+log "IS_PATCH=$IS_PATCH"
+
+basename=rl-${version_major_minor}.md
+file=doc/manual/src/release-notes/$basename
+
+if ! $IS_PATCH; then
+  if [[ -e $file ]]; then
+    die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1"
+  fi
+fi
+
+# --- DEFAULTS ---
+
+if [[ ! -n "${DATE:-}" ]]; then
+  DATE="$(date +%Y-%m-%d)"
+  log "DATE not set, using $DATE"
+fi
+
+case "$DATE" in
+  [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])
+    ;;
+  *)
+    die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
+    ;;
+esac
+
+# --- DO THE WORK ---
+
+# menu
+title="Release $version_major_minor ($DATE)"
+# section on page
+section_title="Release $version_full ($DATE)"
+
+(
+  # TODO add minor number, and append?
+  echo "# $section_title"
+  echo
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
+) | tee -a $file
+
+log "Wrote $file"
+
+if ! $IS_PATCH; then
+  NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+
+  # find the marker line, insert new link after it
+  escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+fi
+
+for f in doc/manual/rl-next/*.md; do
+  if [[ config != "$(basename $f)" ]]; then
+    logcmd git rm $f
+  fi
+done
+
+logcmd git add $file doc/manual/src/SUMMARY.md.in
+logcmd git status
+logcmd git commit -m "release notes: $version_full"
+
+report_done

maintainers/release-process.md

@@ -24,34 +24,23 @@ release:
 * In a checkout of the Nix repo, make sure you're on `master` and run
   `git pull`.
 
-* Move the contents of `doc/manual/src/release-notes/rl-next.md`
+* Compile the release notes by running
-  (except the first line) to
-  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
-  the contents of `.version` *without* the patch level, e.g. `2.12`
-  rather than `2.12.0`).
-
-* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
-
-  ```
-  # Release 2.12 (2022-12-06)
-  ```
-
-* Proof-read / edit / rearrange the release notes. Breaking changes
-  and highlights should go to the top.
-
-* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
-  (*not* `SUMMARY.md`), e.g.
-
-  ```
-  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
-  ```
-
-* Run
 
   ```console
   $ git checkout -b release-notes
-  $ git add doc/manual/src/release-notes/rl-$VERSION.md
+  $ VERSION=X.YY ./maintainers/release-notes
-  $ git commit -a -m 'Release notes'
+  ```
+
+  where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
+
+  A commit is created.
+
+* Proof-read / edit / rearrange the release notes if needed. Breaking changes
+  and highlights should go to the top.
+
+* Push.
+
+  ```console
   $ git push --set-upstream $REMOTE release-notes
   ```
 
@@ -67,15 +56,17 @@ release:
   $ git checkout -b $VERSION-maintenance
   ```
 
-* Mark the release as stable:
+* Mark the release as official:
 
   ```console
-  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix
   ```
 
   This removes the link to `rl-next.md` from the manual and sets
   `officialRelease = true` in `flake.nix`.
 
+* Commit
+
 * Push the release branch:
 
   ```console
@@ -159,6 +150,30 @@ release:
 
 ## Creating a point release
 
+* Checkout.
+
+  ```console
+  $ git checkout XX.YY-maintenance
+  ```
+
+* Determine the next patch version.
+
+  ```console
+  $ export VERSION=XX.YY.ZZ
+  ```
+
+* Update release notes.
+
+  ```console
+  $ ./maintainers/release-notes
+  ```
+
+* Push.
+
+  ```console
+  $ git push
+  ```
+
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.