Merge branch 'sieve-migration' into 'main'

sieve: move `cfg.sieveDirectory` into home directory of virtual users

See merge request simple-nixos-mailserver/nixos-mailserver!508

default.nix

@@ -1064,14 +1064,6 @@ in
       '';
     };
 
-    sieveDirectory = mkOption {
-      type = types.path;
-      default = "/var/sieve";
-      description = ''
-        Where to store the sieve scripts.
-      '';
-    };
-
     virusScanning = mkOption {
       type = types.bool;
       default = false;
@@ -1803,5 +1795,8 @@ in
     (mkRemovedOptionModule [ "mailserver" "fullTextSearch" "autoIndexExclude" ] ''
       Configure `fts_autoindex` on mail directories in `mailserver.mailboxes` instead.
     '')
+    (mkRemovedOptionModule [ "mailserver" "sieveDirectory" ] ''
+      The Sieve directory has been moved into the virtual Dovecot home directory of each user and can longer be configured.
+    '')
   ];
 }

docs/backup-guide.rst

@@ -13,11 +13,6 @@ solution does not preserve the owner of the files don’t forget to ``chown`` th
 to ``virtualMail:virtualMail`` if you copy them back (or whatever you specified
 as :option:`mailserver.storage.owner`, and :option:`mailserver.storage.group`).
 
-If you enabled ``enableManageSieve`` then you also may want to backup
-``/var/sieve`` or whatever you have specified as ``sieveDirectory``.
-The same considerations regarding file ownership apply as for the
-Maildir.
-
 To backup spam and ham training data, backup ``/var/lib/redis-rspamd``.
 
 Finally you can (optionally) make a backup of ``/var/dkim`` (or whatever you

docs/migrations.rst

@@ -13,6 +13,87 @@ apply to your setup.
 NixOS 26.05
 -----------
 
+#5 Sieve script directory migration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sieve scripts managed by users via ManageSieve were previously stored in
+``/var/sieve`` (or via the now-removed option
+``mailserver.sieveDirectory``). This setup partially mirrored the mail
+directory structure in ``/var/vmail`` (``mailserver.storage.path``),
+which proved to be fragile and error-prone.
+
+Thanks to a `prior migration`_, we can now migrate these directories into each
+user’s home directory, aligning with upstream recommendations — almost
+nine years later.
+
+.. _prior migration: #dovecot-mail-directory-migration
+
+This migration is only required if you have :option:`mailserver.enableManageSieve` enabled.
+
+1. If you are coming from ``25.11`` and are using LDAP, temporarily disable
+   :option:`mailserver.enableManageSieve` by setting it to ``false``, deploy,
+   and only then continue with this migration.
+
+   If you are not coming from ``25.11`` or are not using LDAP, continue with
+   this migration as is.
+
+2. Copy the migration script to your mailserver and make it executable:
+
+   .. code-block:: console
+
+      cd /tmp
+      wcurl https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/raw/main/migrations/nixos-mailserver-migration-05.py
+      chmod +x nixos-mailserver-migration-05.py
+
+3. Stop the ``postfix.service``.
+
+   .. code-block:: bash
+
+      systemctl stop postfix.service
+
+4. Create a backup or snapshot of your ``mailserver.sieveDirectory``, so
+   you can restore should anything go wrong.
+
+5. Run the migration script and pass your ``mailserver.sieveDirectory`` as argument:
+
+   The script should be run under the user who owns the ``mailserver.sieveDirectory``.
+   If run as root it will automatically switch into the appropriate user by itself.
+
+   The script will not modify your data unless called with ``--execute``.
+
+   The migration script finds all Sieve script directories in
+   ``/var/sieve/`` (or any other ``mailserver.sieveDirectory``), for
+   example that of ``bob`` at ``/var/vmail/bob@example.com``.
+
+   It then takes ``bob@example.com`` and looks up the home directory
+   for ``bob``. Finally, it starts suggesting the necessary move
+   operations to migrate the Sieve directory to
+   ``/var/vmail/example.com/bob/sieve`` and symlinks the active script
+   to ``/var/vmail/example.com/bob/.dovecot.sieve``.
+
+   Example:
+
+   .. code-block:: bash
+
+      ./nixos-mailserver-migration-05.py \
+         /var/sieve
+
+6. Review the script output.
+
+   The script can highlight various inconsistencies and problems, that should
+   be reviewed and acted upon.
+
+   If in doubt, join our community chat for help before applying any changes.
+
+7. Rerun the command with ``--execute`` or run the proposed commands manually.
+
+8. Update the ``mailserver.stateVersion`` to ``5``.
+
+9. The previous Sieve directory (``mailserver.sieveDirectory``) should now be safe to delete.
+
+10. If you temporarily disabled :option:`mailserver.enableManageSieve` in step 1,
+    re-enable it now by setting it back to ``true``.
+
 #4 Dovecot LDAP UUID-based home directories
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/release-notes.rst

@@ -49,6 +49,8 @@ NixOS 26.05
   - :option:`mailserver.borgbackup.enable`
   - :option:`mailserver.backup.enable`
   - :option:`mailserver.monitoring.enable`
+- Setups with :option:`mailserver.enableManageSieve` enabled require a
+  migration of the `Sieve script directories into Dovecot home directories`_.
 
 .. _setup guide: setup-guide.html#setup-the-server
 .. _DKIM key management: dkim.html
@@ -57,6 +59,7 @@ NixOS 26.05
 .. _AEAD: https://en.wikipedia.org/wiki/Authenticated_encryption
 .. _ECDHE: https://www.rfc-editor.org/rfc/rfc8422
 .. _UUID based home directories: migrations.html#dovecot-ldap-uuid-based-home-directories
+.. _Sieve script directories into Dovecot home directories: migrations.html#sieve-script-directory-migration
 
 NixOS 25.11
 -----------

docs/setup-example.nix

@@ -29,7 +29,7 @@
 
   mailserver = {
     enable = true;
-    stateVersion = 4;
+    stateVersion = 5;
     fqdn = "mail.example.com";
     domains = [ "example.com" ];
 

mail-server/assertions.nix

@@ -132,5 +132,15 @@ in
         '';
       }
     ]
+    ++ lib.optionals (config.mailserver.enableManageSieve) [
+      {
+        assertion = config.mailserver.stateVersion != null -> config.mailserver.stateVersion >= 5;
+        message = ''
+          NixOS Mailserver requires moving the Sieve script directories into Dovecot home directories.
+
+          Check https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#sieve-script-directory-migration for required migration steps.
+        '';
+      }
+    ]
   );
 }

mail-server/dovecot.nix

@@ -346,14 +346,28 @@ in
           "sieve_script default" = {
             # declarative
             type = "default";
-            path = "${cfg.sieveDirectory}/%{user}/default.sieve";
+            name = "default";
+            # TODO: Pre-compile Sieve scripts with 'sievec' (requires a Dovecot config in build sandbox)
+            path = "${
+              pkgs.runCommand "declarative-sieve-scripts" { } (
+                ''
+                  mkdir "$out"
+                ''
+                + lib.concatMapAttrsStringSep "\n" (_: value: ''
+                  mkdir "$out/${value.name}"
+                  cp -v "${builtins.toFile "default.sieve" value.sieveScript}" "$out/${value.name}/default.sieve"
+                '') (lib.filterAttrs (_: value: value.sieveScript != null) cfg.accounts)
+              )
+            }/%{user}/default.sieve";
           };
 
           "sieve_script personal" = {
             # managesieve
             type = "personal";
-            active_path = "${cfg.sieveDirectory}/%{user}/active.sieve";
+            # Upstream default, but we want to be explicit about it
-            path = "${cfg.sieveDirectory}/%{user}/scripts";
+            # https://doc.dovecot.org/main/core/plugins/sieve.html#script-storage-type-personal
+            active_path = "~/.dovecot.sieve";
+            path = "~/sieve";
           };
 
           sieve_extensions = {

mail-server/users.nix

@@ -16,59 +16,12 @@
 
 {
   config,
-  pkgs,
   lib,
   ...
 }:
 
 let
   cfg = config.mailserver;
-
-  virtualMailUsersActivationScript =
-    pkgs.writeScript "activate-virtual-mail-users"
-      # bash
-      ''
-        #!${pkgs.stdenv.shell}
-
-        set -euo pipefail
-
-        # Prevent world-readable paths, even temporarily.
-        umask 007
-
-        # Create directory to store user sieve scripts if it doesn't exist
-        if (! test -d "${cfg.sieveDirectory}"); then
-          mkdir "${cfg.sieveDirectory}"
-          chown "${cfg.storage.owner}:${cfg.storage.group}" "${cfg.sieveDirectory}"
-          chmod 770 "${cfg.sieveDirectory}"
-        fi
-
-        # Copy user's sieve script to the correct location (if it exists).  If it
-        # is null, remove the file.
-        ${lib.concatMapStringsSep "\n" (
-          { name, sieveScript }:
-          if lib.isString sieveScript then
-            ''
-              if (! test -d "${cfg.sieveDirectory}/${name}"); then
-                mkdir -p "${cfg.sieveDirectory}/${name}"
-                chown "${cfg.storage.owner}:${cfg.storage.group}" "${cfg.sieveDirectory}/${name}"
-                chmod 770 "${cfg.sieveDirectory}/${name}"
-              fi
-              cat << 'EOF' > "${cfg.sieveDirectory}/${name}/default.sieve"
-              ${sieveScript}
-              EOF
-              chown "${cfg.storage.owner}:${cfg.storage.group}" "${cfg.sieveDirectory}/${name}/default.sieve"
-            ''
-          else
-            ''
-              if (test -f "${cfg.sieveDirectory}/${name}/default.sieve"); then
-                rm "${cfg.sieveDirectory}/${name}/default.sieve"
-              fi
-              if (test -f "${cfg.sieveDirectory}/${name}.svbin"); then
-                rm "${cfg.sieveDirectory}/${name}/default.svbin"
-              fi
-            ''
-        ) (map (user: { inherit (user) name sieveScript; }) (lib.attrValues cfg.accounts))}
-      '';
 in
 {
   config = lib.mkIf cfg.enable {
@@ -107,14 +60,5 @@ in
       home = cfg.storage.path;
       createHome = true;
     };
-
-    systemd.services.activate-virtual-mail-users = {
-      wantedBy = [ "multi-user.target" ];
-      before = [ "dovecot.service" ];
-      serviceConfig = {
-        ExecStart = virtualMailUsersActivationScript;
-      };
-      enable = true;
-    };
   };
 }

migrations/nixos-mailserver-migration-05.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from pwd import getpwnam
+
+EXIT_OK = 0
+EXIT_ERROR = 1
+
+GREEN = "32"
+YELLOW = "33"
+RED = "31"
+BOLD = "1"
+
+NO_COLOR = "NO_COLOR" in os.environ
+
+
+def color(text, code):
+    if NO_COLOR:
+        return text
+    return f"\033[{code}m{text}\033[0m"
+
+
+def check_user(sieve_root: Path):
+    owner = sieve_root.owner()
+    owner_uid = getpwnam(owner).pw_uid
+
+    if os.geteuid() == owner_uid:
+        return
+
+    try:
+        print(f"Trying to switch effective user id to {owner_uid} ({owner})")
+        os.seteuid(owner_uid)
+        return
+    except PermissionError:
+        print(
+            f"Failed switching to virtual mail user. Please run this script under it, for example by using `sudo -u {owner}`)"
+        )
+    sys.exit(1)
+
+
+def doveadm_get_user_home(user: str) -> Path:
+    output = subprocess.check_output(
+        ["doveadm", "user", "-f", "home", user], text=True, stderr=subprocess.DEVNULL
+    )
+    homedir = Path(output.strip())
+    return homedir
+
+
+def move(src: Path, dst: Path, dry_run: bool = True) -> bool:
+    print(f'mv "{src}" "{dst}"')
+    if not dry_run:
+        try:
+            shutil.move(src, dst)
+        except OSError as exc:
+            print(f"Rename failed ({src=!s}, {dst=!s}): {exc}")
+            return False
+    return True
+
+
+def symlink(target: Path, link: Path, dry_run: bool = True) -> bool:
+    print(f'ln --symbolic --relative "{target}" "{link}"')
+    if not dry_run:
+        try:
+            target_relative = target.relative_to(link.parent)
+            link.symlink_to(target_relative)
+        except (OSError, ValueError) as exc:
+            print(f"Symlinking failed ({target=!s}, {link=!s}): {exc}")
+            return False
+    return True
+
+
+def main(sieve_root: Path, dry_run: bool = True):
+    print(
+        color(
+            f"\nFind accounts based on existing Sieve script directories in {sieve_root!s}",
+            BOLD,
+        )
+    )
+
+    skipped = 0
+    accounts = set()
+    for path in sieve_root.glob("*"):
+        if not path.is_dir():
+            print(f"- Not a directory ({path=!s}) (skipping)")
+            skipped += 1
+            continue
+        elif not set(path.glob("scripts/*.sieve")):
+            print(f"- No Sieve scripts in directory ({path=!s}) (skipping)")
+            skipped += 1
+            continue
+
+        account = path.name
+        accounts.add(account)
+        print(f"- Sieve directory found ({path=!s}, {account=})")
+
+    print(
+        color(
+            f"\nLookup home directory of accounts based on the remaining Sieve directories found in {sieve_root!s}",
+            BOLD,
+        )
+    )
+
+    lookup_failed = 0
+    homedirs = {}
+    for account in accounts:
+        try:
+            homedir = doveadm_get_user_home(account)
+        except subprocess.CalledProcessError as exc:
+            print(f"- Home directory lookup failed ({account=}): {exc}")
+            lookup_failed += 1
+            continue
+        print(f"- Home directory retrieved ({account=}, {homedir=!s})")
+        homedirs.update({account: homedir})
+
+    print(
+        color(
+            "\nEnumerate Sieve directories of accounts",
+            BOLD,
+        )
+    )
+
+    plan = {}
+    for account, homedir in homedirs.items():
+        sieve_src = sieve_root / account / "scripts"
+        sieve_dst = homedir / "sieve"
+        plan.update({sieve_src: sieve_dst})
+
+        active = sieve_root / account / "active.sieve"
+        link = homedir / ".dovecot.sieve"
+        # An account may have Sieve scripts but none enabled,
+        # e.g. an out-of-office auto-reply but currently in-office.
+        if not active.is_symlink():
+            print(
+                f"- Account has Sieve scripts but none enabled ({account=}, {active=!s})"
+            )
+            continue
+        active_relative = active.resolve().relative_to(sieve_src)
+        target = sieve_dst / active_relative
+        plan.update({target: link})
+        print(
+            f"- Account has Sieve scripts and one enabled ({account=}, {sieve_src=!s})"
+        )
+
+    print(color("\nThe following operations will be executed:", BOLD))
+    moved = 0
+    moves_failed = 0
+    for src, dst in plan.items():
+        if src.is_dir():
+            if not move(src=src, dst=dst, dry_run=dry_run):
+                moves_failed += 1
+            else:
+                moved += 1
+        else:
+            if not symlink(target=src, link=dst, dry_run=dry_run):
+                moves_failed += 1
+            else:
+                moved += 1
+
+    print(
+        color(
+            "\nMigration summary",
+            BOLD,
+        )
+    )
+
+    if any([skipped, lookup_failed, not accounts, moves_failed]):
+        print("""
+We strongly recommend reviewing and remediating all potential issues before
+running with `--execute`. Specific details can be found further up.""")
+
+    if moved:
+        print(f"""
+- {color(f"{moved} Sieve script directories were migrated successfully.", GREEN)} {"(dry run)" if dry_run else ""}""")
+
+    if skipped and accounts:
+        print(f"""
+- {color(f"{skipped} paths in {(sieve_root)!s} were skipped.", YELLOW)}
+  These were not a directory or did not contain a ./scripts directory.
+  They should be reviewed but can most likely be deleted.""")
+
+    if lookup_failed:
+        print(f"""
+- {color(f"{lookup_failed} account lookups failed.", YELLOW)}
+  This could be a problem, because we cannot migrate the Sieve script
+  directory into the home directory without finding the owner of the
+  directory. In practice this can happen if an account was deleted but
+  its Sieve script directory remained.""")
+
+    if not accounts:
+        print(f"""
+- {color("No Sieve script directories were found.", RED)}
+  Make sure you are passing the correct `sieve_root` argument. It must match
+  your `mailserver.sieveDirectory` setting. In practise this may also happen
+  if simply no account has Sieve scripts.""")
+
+    if moves_failed:
+        print(f"""
+- {color(f"{moves_failed} Sieve script directories could not be renamed", RED)}
+  No reason to panic, but the script tried to rename a Sieve script directory
+  and that triggered and error. Check further up what went wrong.""")
+
+    if dry_run:
+        print(f"\n{color('No changes were made.', YELLOW)}")
+        print("Run the script with `--execute` to apply the listed changes.")
+
+    sys.exit(EXIT_OK if moves_failed == 0 else EXIT_ERROR)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="""
+        NixOS Mailserver Migration #5: Sieve script directory migration
+        (https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#sieve-script-directory-migration)
+    """
+    )
+    parser.add_argument(
+        "sieve_root", type=Path, help="Path to the `mailserver.sieveDirectory`"
+    )
+    parser.add_argument(
+        "--execute", action="store_true", help="Actually perform changes"
+    )
+
+    args = parser.parse_args()
+
+    check_user(args.sieve_root)
+    main(
+        sieve_root=args.sieve_root,
+        dry_run=not args.execute,
+    )

tests/internal.nix

@@ -61,7 +61,7 @@ in
 
   nodes = {
     machine =
-      { pkgs, ... }:
+      { pkgs, lib, ... }:
       {
         imports = [
           ./../default.nix
@@ -111,6 +111,7 @@ in
             };
             "user3@example.com" = {
               passwordFile = "/run/passwords/user3";
+              sieveScript = lib.readFile ./lib/redirect.sieve;
             };
             "user4@example.com" = {
               hashedPasswordFile = hashedPasswordFileWithScheme;
@@ -292,6 +293,27 @@ in
               )
           )
 
+      with subtest("user's static Sieve script is being executed"):
+          # user3@example.com has a cfg.sieveScript that forwards every
+          # mail sent from user1@example.com back to user1@example.com
+          machine.succeed(
+              " ".join(
+                  [
+                      "mail-check send-and-read",
+                      "--smtp-port 587",
+                      "--smtp-starttls",
+                      "--smtp-host localhost",
+                      "--imap-host localhost",
+                      "--imap-username user1@example.com",
+                      "--from-addr user1@example.com",
+                      "--to-addr user3@example.com",
+                      "--src-password-file ${passwordFile}",
+                      "--dst-password-file ${passwordFile}",
+                      "--ignore-dkim-spf",
+                  ]
+              )
+          )
+
       with subtest("imap port 143 is closed and imaps is serving SSL"):
           machine.wait_for_closed_port(143)
           machine.wait_for_open_port(993)

tests/lib/redirect.sieve

@@ -0,0 +1,3 @@
+if address :is "from" "user1@example.com" {
+  redirect "user1@example.com";
+}