Merge branch 'ldap-updates' into 'master'

LDAP: UUID based homedirs, username based login, group attribute options, docs

Closes #323 and #342

See merge request simple-nixos-mailserver/nixos-mailserver!493

default.nix

@@ -319,7 +319,11 @@ in
           ]
         '';
         description = ''
-          URIs where your LDAP server can be reached
+          List of LDAP server URIs. Multiple can be specified.
+
+          Use `ldaps://` for implicit TLS or `ldap://` for a plain connection. See
+          also {option}`mailserver.ldap.startTls` to enable StartTLS on plain
+          connections.
         '';
       };
 
@@ -327,16 +331,16 @@ in
         type = types.bool;
         default = false;
         description = ''
-          Whether to enable StartTLS upon connection to the server.
+          Whether to enable StartTLS on ``ldap://`` connections.
         '';
       };
 
-      tlsCAFile = mkOption {
+      caFile = mkOption {
         type = types.path;
         default = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-        defaultText = literalMD "see [source](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/default.nix)";
+        defaultText = lib.literalExpression "\${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         description = ''
-          Certificate trust anchors used to verify the LDAP server certificate.
+          Bundle of CA certificates used to authenticate the LDAP server certificate.
         '';
       };
 
@@ -345,88 +349,133 @@ in
           type = types.str;
           example = "cn=mail,ou=accounts,dc=example,dc=com";
           description = ''
-            Distinguished name used by the mail server to do lookups
+            DN used to bind against the LDAP server.
-            against the LDAP servers.
+
+            The server uses this account to lookup and filter accounts.
           '';
         };
 
         passwordFile = mkOption {
-          type = types.str;
+          type = types.pathWith { inStore = false; };
           example = "/run/my-secret";
           description = ''
-            A file containing the password required to authenticate against the LDAP servers.
+            File containing the password required to bind against the LDAP server.
           '';
         };
       };
 
-      searchBase = mkOption {
+      base = mkOption {
         type = types.str;
         example = "ou=people,ou=accounts,dc=example,dc=com";
         description = ''
-          Base DN at below which to search for users accounts.
+          Base DN below which user accounts are searched for.
         '';
       };
 
-      searchScope = mkOption {
+      scope = mkOption {
         type = types.enum [
-          "sub"
           "base"
           "one"
+          "sub"
         ];
         default = "sub";
         description = ''
-          Search scope below which users accounts are looked for.
+          Search scope relative to the {option}`mailserver.ldap.base`.
+
+          - base: Only the exact Base DN
+          - one: Immediate child entries of the Base DN, but not the Base DN itself.
+          - sub: Base DN and all descendant entries at any depth.
+
+          In practice only `one` or `sub` are suitable for multiple LDAP users.
         '';
       };
 
-      dovecot = {
+      attributes = {
-        userAttrs = mkOption {
+        uuid = mkOption {
-          type = types.nullOr types.str;
+          type = types.str;
-          default = null;
+          default = "entryUUID";
+          example = "uuid";
           description = ''
-            LDAP attributes to be retrieved during userdb lookups.
+            The long-term stable LDAP attribute to reference accounts across
+            username changes. Used to determine a stable Dovecot home and
+            mail directory location.
 
-            See the users_attrs reference at
+            Typically the `entryUUID` attribute as defined by [RFC4530].
-            https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#user-attrs
+
-            in the Dovecot manual.
+            [RFC4530]: https://www.rfc-editor.org/rfc/rfc4530.html
           '';
         };
 
+        username = mkOption {
+          type = types.str;
+          default = "uid";
+          example = "name";
+          description = ''
+            The LDAP attribute referencing the username used to login with.
+
+            Typically the `uid` attribute which is part of the `inetOrgPerson` schema.
+          '';
+        };
+
+        password = mkOption {
+          type = types.str;
+          default = "userPassword";
+          example = "unix_password";
+          description = ''
+            The LDAP attribute referencing the account password used to login with.
+
+            Typically the `userPassword` attribute which is part of the `inetOrgPerson` schema.
+          '';
+        };
+
+        mail = mkOption {
+          type = types.str;
+          default = "mail";
+          example = "maildrop";
+          description = ''
+            The attribute name used for looking up accounts by mail address.
+
+            Typically this can be the `mail` attribute from the `inetOrgPerson`
+            schema, or the `maildrop` attribute from the unofficial Postfix
+            schema.
+          '';
+        };
+      };
+
+      dovecot = {
         userFilter = mkOption {
           type = types.str;
-          default = "mail=%{user}";
+          default = with cfg.ldap.attributes; "(|(${mail}=%{user})(${username}=%{user}))";
-          example = "(&(objectClass=inetOrgPerson)(mail=%{user}))";
+          defaultText = literalExpression ''
-          description = ''
+            with config.mailserver.ldap.attributes; "(|(''${mail}=%{user})(''${username}=%{user}))";
-            Filter for user lookups in Dovecot.
-
-            See the user_filter reference at
-            https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#user-filter
-            in the Dovecot manual.
           '';
-        };
+          example = "(|(mail=%{user})(uid=%{user}))";
-
-        passAttrs = mkOption {
-          type = types.str;
-          default = "userPassword=password";
           description = ''
-            LDAP attributes to be retrieved during passdb lookups.
+            LDAP filter used for LMTP delivery from Postfix and post-login
+            information construction, like the home directory.
 
-            See the pass_attrs reference at
+            See the [user_filter] reference at in the Dovecot manual.
-            https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#pass-attrs
+
-            in the Dovecot manual.
+            [user_filter]: https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#user-filter
           '';
         };
 
         passFilter = mkOption {
           type = types.nullOr types.str;
-          default = "mail=%{user}";
+          default = with cfg.ldap.attributes; "${username}=%{user}";
-          example = "(&(objectClass=inetOrgPerson)(mail=%{user}))";
+          defaultText = lib.literalExpression ''
+            with config.mailserver.ldap.attributes; "''${username}=%{user}";
+          '';
+          example =
+            with cfg.ldap.attributes;
+            "(&(memberOf=cn=mail_users,ou=groups,dc=example,dc=com)(${username}=%{user}))";
           description = ''
-            Filter for password lookups in Dovecot.
+            LDAP filter used to restrict which users are eligible to
+            authenticate against Dovecot.
 
-            See the pass_filter reference for
+            See the [pass_filter] reference in the Dovecot manual.
-            https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#pass-filter
+
-            in the Dovecot manual.
+            [pass_filter]: https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_settings_auth/#pass-filter
           '';
         };
       };
@@ -434,29 +483,14 @@ in
       postfix = {
         filter = mkOption {
           type = types.str;
-          default = "mail=%s";
+          default = with cfg.ldap.attributes; "${mail}=%s";
-          example = "(&(objectClass=inetOrgPerson)(mail=%s))";
+          defaultText = lib.literalExpression ''
-          description = ''
+            with config.mailserver.ldap.attributes; "''${mail}=%s";
-            LDAP filter used to search for an account by mail, where
-            `%s` is a substitute for the address in
-            question.
           '';
-        };
+          example = "(mail=%s)";
-
-        uidAttribute = mkOption {
-          type = types.str;
-          default = "mail";
-          example = "uid";
           description = ''
-            The LDAP attribute referencing the account name for a user.
+            LDAP filter used to search for an account by mail, where `%s` is a
-          '';
+            substitute for the address in question.
-        };
-
-        mailAttribute = mkOption {
-          type = types.str;
-          default = "mail";
-          description = ''
-            The LDAP attribute holding mail addresses for a user.
           '';
         };
       };
@@ -1630,5 +1664,22 @@ in
       [ "mailserver" "dkimKeyBits" ]
       [ "mailserver" "dkim" "defaults" "keyLength" ]
     )
+    (mkRemovedOptionModule [ "mailserver" "ldap" "dovecot" "userAttrs" ] ''
+      The user_attrs field is now used internally to map the home and mail directories.
+    '')
+    (mkRemovedOptionModule [ "mailserver" "ldap" "dovecot" "passAttrs" ] ''
+      The pass_attrs field is now used internally. You can customize the `mailserver.ldap.attributes.password` field instead.
+    '')
+    (mkRenamedOptionModule [ "mailserver" "ldap" "tlsCAFile" ] [ "mailserver" "ldap" "caFile" ])
+    (mkRenamedOptionModule [ "mailserver" "ldap" "searchBase" ] [ "mailserver" "ldap" "base" ])
+    (mkRenamedOptionModule [ "mailserver" "ldap" "searchScope" ] [ "mailserver" "ldap" "scope" ])
+    (mkRenamedOptionModule
+      [ "mailserver" "ldap" "postfix" "uidAttribute" ]
+      [ "mailserver" "ldap" "attributes" "username" ]
+    )
+    (mkRenamedOptionModule
+      [ "mailserver" "ldap" "postfix" "mailAttribute" ]
+      [ "mailserver" "ldap" "attributes" "mail" ]
+    )
   ];
 }

docs/index.rst

@@ -21,13 +21,18 @@ Welcome to NixOS Mailserver's documentation!
    options
    migrations
 
+.. toctree::
+   :maxdepth: 1
+   :caption: Account backends
+
+   ldap
+
 .. toctree::
    :maxdepth: 1
    :caption: Features
 
    dkim
    fts
-   ldap
    srs
 
 .. toctree::

docs/ldap-attrs.nix

@@ -0,0 +1,12 @@
+{
+  mailserver = {
+    ldap = {
+      attributes = {
+        uuid = "entryUUID";
+        username = "uid";
+        password = "userPassword";
+        mail = "mail";
+      };
+    };
+  };
+}

docs/ldap-basic.nix

@@ -0,0 +1,17 @@
+{
+  mailserver = {
+    ldap = {
+      enable = true;
+      uris = [
+        "ldaps://ldap1.example.com"
+        "ldaps://ldap2.example.com"
+      ];
+      bind = {
+        dn = "cn=mail,dc=example=dc=com";
+        passwordFile = "/run/keys/ldap-bind-pw";
+      };
+      base = "ou=users,dc=example,dc=com";
+      scope = "one";
+    };
+  };
+}

docs/ldap.rst

@@ -1,14 +1,82 @@
-LDAP Support
+LDAP
-============
+====
 
-It is possible to manage mail user accounts with LDAP rather than with
+LDAP (Lightweight Directory Access Protocol) is a protocol for accessing and
-the option `loginAccounts <options.html#mailserver-loginaccounts>`_.
+managing a centralized directory of user and group information. It can be used
+to authenticate users and provide a single source of truth for email accounts
+and aliases across mail services.
 
-All related LDAP options are described in the `LDAP options section
-<options.html#mailserver-ldap>`_ and the `LDAP test
-<https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/tests/ldap.nix>`_
-provides a getting started example.
 
-.. note::
+Requirements
-   The LDAP support can not be enabled if some accounts are also defined with ``mailserver.loginAccounts``.
+~~~~~~~~~~~~
 
+To enable the LDAP integration the following requirements must be fulfilled:
+
+- Existing LDAP service (we currently only test against OpenLDAP)
+- Bind credentials against LDAP with permissions to
+
+  - search for the acceptable set of users
+  - read the :option:`mailserver.ldap.attributes.password` attribute
+
+- Each user entry must provide attributes that can serve as
+
+  - :option:`mailserver.ldap.attributes.mail` (primary mail address)
+  - :option:`mailserver.ldap.attributes.username` (login name)
+  - :option:`mailserver.ldap.attributes.password` (login password)
+  - :option:`mailserver.ldap.attributes.uuid` (stable identifier)
+
+
+Features
+~~~~~~~~
+
+We currently have a basic feature set covering user accounts only and try to
+follow best practices to simplify maintenance.
+
+- Users authenticate with the username and password attribute
+- Maildir storage paths are constructed using the uuid attribute
+- Primary mail address read from mail attribute
+
+
+Limitations
+~~~~~~~~~~~
+
+We have various assertions in place, that prevent using LDAP together with
+other features. Most of them are not technical limitations per se, but instead
+lack configuration or validation.
+
+- Local users (:option:`mailserver.loginAccounts`) and aliases
+  (:option:`mailserver.extraVirtualAliases`) are not currently allowed with
+  :option:`mailserver.ldap.enable` enabled
+- Aliases based on LDAP attributes are currently not implemented
+- Quotas based on LDAP attributes are currently not implemented
+
+The following features will likely never be implemented, since they would
+complicate the setup significantly.
+
+- Domains based on LDAP entries (would require integration with everything we
+  already do for :option:`mailserver.domains`)
+- Use of ``homeDirectory``, ``uid``, ``gid`` LDAP attributes (we are
+  committed to a virtual setup with one vmail user/uid/gid and UUID based home
+  directories)
+
+
+Enabling LDAP support
+~~~~~~~~~~~~~~~~~~~~~
+
+Enable the LDAP integration by configuring an authenticated LDAP connection
+and how to locate all users. The bind DN must be allowed to read the configured
+password attribute, which may require additional configuration
+
+.. literalinclude:: ./ldap-basic.nix
+   :language: nix
+
+We provide sensible defaults for each attribute, that can be adapted to your
+local setup.
+
+.. literalinclude:: ./ldap-attrs.nix
+   :language: nix
+
+Refer to our `LDAP test`_ for an complete example, and see the `LDAP options`_ section for all possible settings.
+
+.. _LDAP options: options.html#mailserver-ldap
+.. _LDAP test: https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/tests/ldap.nix

docs/migrations.rst

@@ -5,10 +5,107 @@ With mail server configuration best practices changing over time we might need
 to make changes that require you to complete manual migration steps before you
 can deploy a new version of NixOS mailserver.
 
-The initial `mailserver.stateVersion` value should be copied from the setup
+The initial :option:`mailserver.stateVersion` value should be copied from the
-guide that you used to initially set up your mail server. If in doubt you can
+setup guide that you used to initially set up your mail server. If in doubt you
-always initialize it at `1` and walk through all assertions, that might apply
+can always initialize it at ``1`` and walk through all assertions, that might
-to your setup.
+apply to your setup.
+
+NixOS 26.05
+-----------
+
+#4 Dovecot LDAP UUID-based home directories
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+LDAP Support in NixOS mailserver was introduced during the 23.11 release cycle
+and came with a number of flaws that we are correcting now, three years later.
+
+This particular migration is needed because up until now we were
+relying on email addresses to construct the Dovecot home directory path
+(``var/vmail/ldap/user@example.com``) which is fragile: addresses can
+change, requiring manual homedir relocation. Switching to UUID-based homedirs
+(``/var/vmail/ldap/<uuid>``) ensures stable, unique paths and applies well-known
+best practices to mailserver management.
+
+1. Copy the migration script script to your mailserver and make it executable:
+
+   .. code-block:: console
+
+      cd /tmp
+      wcurl https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/migrations/nixos-mailserver-migration-04.py
+      chmod +x nixos-mailserver-migration-04.py
+
+2. Stop the ``dovecot.service``.
+
+   .. code-block:: bash
+
+      systemctl stop dovecot.service
+
+3. Create a backup or snapshot of your :option:`mailserver.mailDirectory`, so
+   you can restore should anything go wrong.
+
+4. Run the migration script and pass the required arguments to enable LDAP lookups:
+
+   The script should be run under the user who owns the :option:`mailserver.mailDirectory`.
+   If run as root it will automatically switch into the appropriate user by itself.
+
+   The script will not modify your data unless called with ``--execute``.
+
+   The migration script finds all Dovecot home directories in
+   ``/var/vmail/ldap/`` (or any other :option:`mailserver.mailDirectory`),
+   for example that of bob at ``/var/vmail/ldap/bob@example.com``.
+   It then takes ``bob@example.com`` and queries the LDAP server for
+   ``mail=bob@example.com`` to retrieve the UUID attribute. Finally
+   it starts suggesting the neceessary move operations to arrive at
+   ``/var/vmail/ldap/f3b4e8ea-087f-42cc-95f0-cbfd99386092`` for bob.
+
+   Example:
+
+   .. code-block:: bash
+
+      ./nixos-mailserver-migration-04.py \
+        --ldap-uri ldaps://ldap1.example.com
+        --ldap-bind-dn cn=mail,ou=accounts,dc=example,dc=com \
+        --ldap-bind-pw-file /run/keys/ldap-bind-pw \
+        --ldap-base ou=people,ou=accounts,dc=example,dc=com \
+        --ldap-scope sub \
+        --ldap-filter "(mail=%s)" \
+        --ldap-attr-uuid entryUUID \
+         /var/vmail
+
+   For the ``--ldap-attr-uuid`` parameter we expect a long-term stable
+   identifier, ideally a UUID field. The exact attribute name depends on your
+   LDAP implementation, for example:
+
+   - Authentik: ``uid`` `[1]`_
+   - Kanidm: ``uuid`` `[2]`_
+   - Keycloak ``entryUUID``
+   - OpenLDAP: ``entryUUID`` (`RFC4530`_)
+
+   If yours LDAP provider isn't listed you can determine the correct
+   attribute by quering a user entry with ``ldapsearch``. Finally, configure
+   :option:`mailserver.ldap.attributes.uuid` accordingly.
+
+   Add ``--ldap-starttls`` if you use the the `ldap://` URI scheme and require
+   explicit TLS.
+
+   .. _[1]: https://docs.goauthentik.io/add-secure-apps/providers/ldap#users
+   .. _[2]: https://kanidm.github.io/kanidm/stable/integrations/ldap.html#data-mapping
+   .. _RFC4530: https://www.rfc-editor.org/rfc/rfc4530.html
+
+5. Review the script output.
+
+   It's primary job is to determine the UUID for an LDAP account, so that it
+   can rename the Dovecot home directory from mail address to UUID within the
+   same directory.
+
+   The script can highlight various inconsistencies and problems, that should
+   be reviewed and acted upon.
+
+   If in doubt, join our community chat for help before applying any changes.
+
+6. Rerun the command with ``--execute`` or run the proposed commands manually.
+
+7. Update the ``mailserver.stateVersion`` to ``4``.
 
 NixOS 25.11
 -----------

docs/release-notes.rst

@@ -22,6 +22,14 @@ NixOS 26.05
   established by `agenix`_/`sops-nix`_ that instead rely on encryption. This
   option prevents files from leaking in to the Nix store.
   See :option:`mailserver.loginAccounts.<name>.passwordFile`.
+- LDAP setups require a migration of Dovecot home directories to
+  `UUID based home directories`_. The exact UUID attribute can be customized
+  through :option:`mailserver.ldap.attributes.uuid`.
+- The default login username for LDAP users has changed from the ``mail`` to
+  the ``uid`` attribute. This allows users to login with their account name
+  rather than their email address, which is more convenient and consistent
+  with typical LDAP practices. The exact attribute can be customized through
+  :option:`mailserver.ldap.attributes.username`.
 - The following integrations are deprecated and will be removed before the next
   release:
 
@@ -33,6 +41,7 @@ NixOS 26.05
 .. _DKIM key management: dkim.html
 .. _agenix: https://github.com/ryantm/agenix
 .. _sops-nix: https://github.com/Mic92/sops-nix
+.. _UUID based home directories: migrations.html#dovecot-ldap-uuid-based-home-directories
 
 NixOS 25.11
 -----------

mail-server/assertions.nix

@@ -134,5 +134,15 @@ in
         '';
       }
     ]
+    ++ lib.optionals (config.mailserver.ldap.enable) [
+      {
+        assertion = config.mailserver.stateVersion != null -> config.mailserver.stateVersion >= 4;
+        message = ''
+          NixOS Mailserver requires migrating LDAP home directories to UUID scheme
+
+          Check https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-ldap-uuid-based-home-directory for required migration steps.
+        '';
+      }
+    ]
   );
 }

mail-server/dovecot.nix

@@ -61,6 +61,7 @@ let
 
   postfixCfg = config.services.postfix;
 
+  ldapUuidAttribute = cfg.ldap.attributes.uuid;
   ldapConfig = pkgs.writeTextFile {
     name = "dovecot-ldap.conf.ext.template";
     text = ''
@@ -70,19 +71,20 @@ let
         tls = yes
       ''}
       tls_require_cert = hard
-      tls_ca_cert_file = ${cfg.ldap.tlsCAFile}
+      tls_ca_cert_file = ${cfg.ldap.caFile}
       dn = ${cfg.ldap.bind.dn}
       sasl_bind = no
       auth_bind = yes
-      base = ${cfg.ldap.searchBase}
+      base = ${cfg.ldap.base}
-      scope = ${mkLdapSearchScope cfg.ldap.searchScope}
+      scope = ${mkLdapSearchScope cfg.ldap.scope}
-      ${lib.optionalString (cfg.ldap.dovecot.userAttrs != null) ''
+      user_attrs = \
-        user_attrs = ${cfg.ldap.dovecot.userAttrs}
+        ${ldapUuidAttribute}=${ldapUuidAttribute}, \
-      ''}
+        =home=/var/vmail/ldap/%{ldap:${ldapUuidAttribute}}, \
+        =mail=maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}${
+          lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/ldap/%{ldap:${ldapUuidAttribute}}"
+        }
       user_filter = ${cfg.ldap.dovecot.userFilter}
-      ${lib.optionalString (cfg.ldap.dovecot.passAttrs != "") ''
+      pass_attrs = ${cfg.ldap.attributes.password}=password
-        pass_attrs = ${cfg.ldap.dovecot.passAttrs}
-      ''}
       pass_filter = ${cfg.ldap.dovecot.passFilter}
     '';
   };
@@ -443,14 +445,9 @@ in
           userdb {
             driver = ldap
             args = ${ldapConfFile}
-            default_fields = \
+            override_fields = \
-              home=${cfg.mailDirectory}/ldap/%{user} \
               uid=${toString cfg.vmailUID} \
-              gid=${toString cfg.vmailUID} \
+              gid=${toString cfg.vmailUID}
-              mail=maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}${
-                lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/ldap/%{user}"
-              }
-
           }
         ''}
 

mail-server/postfix.nix

@@ -209,20 +209,21 @@ let
     server_host = ${lib.concatStringsSep " " cfg.ldap.uris}
     start_tls = ${if cfg.ldap.startTls then "yes" else "no"}
     version = 3
-    tls_ca_cert_file = ${cfg.ldap.tlsCAFile}
+    tls_ca_cert_file = ${cfg.ldap.caFile}
     tls_require_cert = yes
 
-    search_base = ${cfg.ldap.searchBase}
+    search_base = ${cfg.ldap.base}
-    scope = ${cfg.ldap.searchScope}
+    scope = ${cfg.ldap.scope}
 
     bind = yes
     bind_dn = ${cfg.ldap.bind.dn}
   '';
 
+  # Enforce a mapping between SMTP user and envelope sender address
   ldapSenderLoginMap = pkgs.writeText "ldap-sender-login-map.cf" ''
     ${commonLdapConfig}
     query_filter = ${cfg.ldap.postfix.filter}
-    result_attribute = ${cfg.ldap.postfix.mailAttribute}
+    result_attribute = ${cfg.ldap.attributes.username}
   '';
   ldapSenderLoginMapFile = "/run/postfix/ldap-sender-login-map.cf";
   appendPwdInSenderLoginMap = appendLdapBindPwd {
@@ -233,10 +234,11 @@ let
     destination = ldapSenderLoginMapFile;
   };
 
+  # Check whether a recipient address exists, before accepting mail for it
   ldapVirtualMailboxMap = pkgs.writeText "ldap-virtual-mailbox-map.cf" ''
     ${commonLdapConfig}
     query_filter = ${cfg.ldap.postfix.filter}
-    result_attribute = ${cfg.ldap.postfix.uidAttribute}
+    result_attribute = ${cfg.ldap.attributes.username}
   '';
   ldapVirtualMailboxMapFile = "/run/postfix/ldap-virtual-mailbox-map.cf";
   appendPwdInVirtualMailboxMap = appendLdapBindPwd {

migrations/nixos-mailserver-migration-04.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ ldap3 ])"
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from pwd import getpwnam
+from typing import Literal, cast
+
+from ldap3 import BASE, LEVEL, SUBTREE, Connection, Server
+from ldap3.core.exceptions import LDAPException
+
+LDAPSearchScope = Literal["BASE", "LEVEL", "SUBTREE"]
+
+EXIT_OK = 0
+EXIT_ERROR = 1
+EXIT_LDAP_STARTTLS = 2
+EXIT_LDAP_BIND = 3
+
+GREEN = "32"
+YELLOW = "33"
+RED = "31"
+BOLD = "1"
+
+NO_COLOR = "NO_COLOR" in os.environ
+
+
+def color(text, code):
+    if NO_COLOR:
+        return text
+    return f"\033[{code}m{text}\033[0m"
+
+
+def check_user(vmail_root: Path):
+    owner = vmail_root.owner()
+    owner_uid = getpwnam(owner).pw_uid
+
+    if os.geteuid() == owner_uid:
+        return
+
+    try:
+        print(f"Trying to switch effective user id to {owner_uid} ({owner})")
+        os.seteuid(owner_uid)
+        return
+    except PermissionError:
+        print(
+            f"Failed switching to virtual mail user. Please run this script under it, for example by using `sudo -u {owner}`)"
+        )
+    sys.exit(1)
+
+
+def move(*, src: Path, dst: Path, dry_run: bool = True) -> bool:
+    print(f'mv "{src}" "{dst}"')
+    if not dry_run:
+        try:
+            src.rename(dst)
+        except OSError as exc:
+            print(f"Rename failed ({src=!s}, {dst=!s}): {exc}")
+            return False
+    return True
+
+
+def main(
+    *,
+    vmail_root: Path,
+    ldap_uri: str,
+    ldap_starttls: bool,
+    ldap_bind_dn: str,
+    ldap_bind_pw: str,
+    ldap_base: str,
+    ldap_scope: LDAPSearchScope,
+    ldap_filter: str,
+    ldap_attr_uuid: str,
+    dry_run: bool = True,
+    verbose: bool = False,
+):
+    # Begin with LDAP connection for fast feedback
+    server = Server(ldap_uri)
+    conn = Connection(server, ldap_bind_dn, ldap_bind_pw)
+
+    if ldap_starttls:
+        try:
+            if ldap_starttls:
+                conn.start_tls()
+        except LDAPException as exc:
+            print(color(f"LDAP connection setup failed: {exc!r}", RED))
+            sys.exit(EXIT_LDAP_STARTTLS)
+
+    if not conn.bind():
+        err = conn.result
+        print(
+            color(
+                f"""
+LDAP bind failed for {ldap_bind_dn}@{ldap_uri}
+Result: {err.get("result")} ({err.get("description")})
+Message: {err.get("message")!r}""",
+                RED,
+            )
+        )
+        sys.exit(EXIT_LDAP_BIND)
+
+    # Find existing dovecot home directories and collect account identifier
+    print(
+        color(
+            f"\nEnumerate accounts based on existing home directories in {(vmail_root / 'ldap')!s}",
+            BOLD,
+        )
+    )
+
+    skipped = 0
+    accounts = set()
+    homedirs = vmail_root.glob("ldap/*")
+    for path in homedirs:
+        if not path.is_dir():
+            print(f"- Not a directory ({path=!s}) (skipping)")
+            skipped += 1
+            continue
+        elif not (path / "mail").is_dir():
+            print(f"- No maildir in home ({path=!s}) (skipping)")
+            skipped += 1
+            continue
+
+        account = path.name
+        accounts.add(account)
+        if verbose:
+            print(f"- Home directory found ({path=!s}, {account=})")
+
+    print(
+        color(
+            f"\nFinding matching LDAP entries to retrieve `{ldap_attr_uuid}` attribute",
+            BOLD,
+        )
+    )
+
+    no_entry = 0
+    multiple_entries = 0
+    plan = {}
+    for account in sorted(accounts):
+        filter = ldap_filter % account
+        conn.search(
+            search_base=ldap_base,
+            search_filter=filter,
+            search_scope=ldap_scope,
+            attributes=[ldap_attr_uuid],
+        )
+
+        if conn.response is None:
+            print(f"- LDAP search produced no result for {filter}")
+
+        count = len(conn.entries)
+
+        if count < 1:
+            print(f"- No LDAP entry found ({account=}, {filter=}) (skipping)")
+            no_entry += 1
+            continue
+        elif count > 1:
+            print(f"- Multiple LDAP entries found ({account=}, {filter=}) (skipping)")
+            multiple_entries += 1
+            continue
+        else:
+            entry = conn.entries[0]
+            uuid = str(entry[ldap_attr_uuid].value)
+            if verbose:
+                print(f"- LDAP entry mapped ({account=}, {uuid=})")
+            plan.update({account: uuid})
+
+    print(color("\nThe following operations will be executed:", BOLD))
+    moved = 0
+    moves_failed = 0
+    for src, dst in plan.items():
+        _src = vmail_root / "ldap" / src
+        _dst = vmail_root / "ldap" / dst
+        if not move(src=_src, dst=_dst, dry_run=dry_run):
+            moves_failed += 1
+        else:
+            moved += 1
+
+    print(
+        color(
+            "\nMigration summary",
+            BOLD,
+        )
+    )
+
+    if any([skipped, no_entry, multiple_entries, not accounts, moves_failed]):
+        print("""
+We strongly recommend reviewing and remediating all potential issues before
+running with `--execute`. Specific details can be found further up.""")
+
+    if moved:
+        print(f"""
+- {color(f"{moved} home directories were migrated succesfully.", GREEN)} {"(dry run)" if dry_run else ""}
+  This is great news, they are now UUID-based and will be immune to username changes!""")
+
+    if skipped and accounts:
+        print(f"""
+- {color(f"{skipped} paths in {(vmail_root / 'ldap')!s} were skipped.", YELLOW)}
+  These were not a directory or did not contain a maildir. They should be
+  reviewed but can most likely be deleted.""")
+
+    if no_entry:
+        print(f"""
+- {color(f"{no_entry} LDAP queries found no entry.", YELLOW)}
+  This could be a problem, because we cannot migrate home directories without
+  finding the LDAP entry and retrieving its {ldap_attr_uuid} field. In practice
+  this can happen if an LDAP account was deleted but its mail home directory
+  remained.""")
+
+    if multiple_entries:
+        print(f"""
+- {color(f"{multiple_entries} LDAP queries returned multiple entries.", RED)}
+  This is a problem, because we cannot decide which LDAP entry owns the home
+  directory.""")
+
+    if not accounts:
+        print(f"""
+- {color("No home directories were found.", RED)}
+  Make sure you are passing the correct `vmail_root` argument. It must match
+  your `mailserver.mailDirectory` setting.""")
+
+    if moves_failed:
+        print(f"""
+- {color("{moves_failed} home directores could not be renamed", RED)}
+  No reason to panic, but the script tried to rename a home directory and that
+  triggered and error. Check further up what went wrong.""")
+
+    if dry_run:
+        print(f"\n{color('No changes were made.', YELLOW)}")
+        print("Run the script with `--execute` to apply the listed changes.")
+
+    sys.exit(EXIT_OK if moves_failed == 0 else EXIT_ERROR)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="""
+        NixOS Mailserver Migration #4: Dovecot LDAP UUID-based home directories
+        (https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-ldap-uuid-based-home-directory)
+        """
+    )
+    parser.add_argument(
+        "vmail_root", type=Path, help="Path to the `mailserver.mailDirectory`"
+    )
+    parser.add_argument(
+        "--ldap-uri",
+        type=str,
+        required=True,
+        help="URI for your LDAP server; ldaps://ldap1.example.com (TLS) or ldap://ldap1.exampe.com (Plain)",
+    )
+    parser.add_argument(
+        "--ldap-starttls",
+        action="store_true",
+        help="Enable StartTLS on plain LDAP connections",
+    )
+    parser.add_argument(
+        "--ldap-bind-dn",
+        type=str,
+        required=True,
+        help="The distinguished user allow to bind and search the LDAP server",
+    )
+    parser.add_argument(
+        "--ldap-bind-pw-file",
+        type=Path,
+        required=True,
+        help="Path to a file containing the bind password for the LDAP DN",
+    )
+    parser.add_argument(
+        "--ldap-base",
+        type=str,
+        required=True,
+        help="Base DN below which to search for LDAP accounts",
+    )
+    parser.add_argument(
+        "--ldap-scope",
+        choices=[
+            "sub",
+            "base",
+            "one",
+        ],
+        default="sub",
+        help="Scope relative to the base DN",
+    )
+    parser.add_argument(
+        "--ldap-filter",
+        default="(mail=%s)",
+        help="LDAP query that filters for an account by the name in /var/vmail/ldap/<name> field, e.g. mail=%%s or uid=%%s if the name is not an email adress.",
+    )
+    parser.add_argument(
+        "--ldap-attr-uuid",
+        default="entryUUID",
+        help="UUID attribute that uniquely identifies an LDAP account across login name changes",
+    )
+    parser.add_argument(
+        "--execute", action="store_true", help="Actually perform changes"
+    )
+    parser.add_argument("--verbose", action="store_true", help="Print more details")
+
+    args = parser.parse_args()
+
+    if args.ldap_filter.count("%s") != 1:
+        print(
+            "The --ldap-filter argument must contain exactly one '%s' as a placeholder for the primary email address.",
+        )
+        sys.exit(1)
+
+    def read_ldap_bind_pw():
+        try:
+            with open(args.ldap_bind_pw_file) as fd:
+                return fd.read().strip()
+        except OSError as exc:
+            print(f"Unable to read LDAP bind password file: {exc}")
+            sys.exit(1)
+
+    ldap_bind_pw = None
+    if os.geteuid() == 0:
+        # if we're root, read before priv drop
+        ldap_bind_pw = read_ldap_bind_pw()
+
+    check_user(args.vmail_root)
+
+    if ldap_bind_pw is None:
+        ldap_bind_pw = read_ldap_bind_pw()
+
+    ldap_scope: LDAPSearchScope = cast(
+        LDAPSearchScope,
+        {
+            "sub": SUBTREE,
+            "base": BASE,
+            "one": LEVEL,
+        }[args.ldap_scope],
+    )
+
+    main(
+        vmail_root=args.vmail_root,
+        ldap_uri=args.ldap_uri,
+        ldap_starttls=args.ldap_starttls,
+        ldap_bind_dn=args.ldap_bind_dn,
+        ldap_bind_pw=ldap_bind_pw,
+        ldap_base=args.ldap_base,
+        ldap_scope=ldap_scope,
+        ldap_filter=args.ldap_filter,
+        ldap_attr_uuid=args.ldap_attr_uuid,
+        dry_run=not args.execute,
+        verbose=args.verbose,
+    )

tests/ldap.nix

@@ -11,17 +11,10 @@ in
       { pkgs, ... }:
       {
         imports = [
-          ./../default.nix
+          ../default.nix
           ./lib/config.nix
         ];
 
-        virtualisation.memorySize = 1024;
-
-        services.openssh = {
-          enable = true;
-          settings.PermitRootLogin = "yes";
-        };
-
         environment.systemPackages = [
           (pkgs.writeScriptBin "mail-check" ''
             ${pkgs.python3}/bin/python ${../scripts/mail-check.py} $@
@@ -72,17 +65,23 @@ in
               ou: users
 
               dn: cn=alice,ou=users,dc=example
+              entryUUID: c52f777b-a6e8-4507-80f9-c4de47e8520d
               objectClass: inetOrgPerson
-              cn: alice
+              uid: alice
               sn: Foo
               mail: alice@example.com
               userPassword: ${alicePassword}
 
               dn: cn=bob,ou=users,dc=example
+              entryUUID: f3b4e8ea-087f-42cc-95f0-cbfd99386092
               objectClass: inetOrgPerson
-              cn: bob
+              objectClass: posixAccount
+              uid: bob
+              uidNumber: 9999
+              gidNumber: 9999
               sn: Bar
               mail: bob@example.com
+              homeDirectory: /home/bob
               userPassword: ${bobPassword}
             '';
         };
@@ -103,18 +102,13 @@ in
               dn = "cn=mail,dc=example";
               passwordFile = "/etc/bind-password";
             };
-            searchBase = "ou=users,dc=example";
+            base = "ou=users,dc=example";
-            searchScope = "sub";
+            scope = "sub";
           };
 
           forwards = {
             "bob_fw@example.com" = "bob@example.com";
           };
-
-          vmailGroupName = "vmail";
-          vmailUID = 5000;
-
-          enableImap = false;
         };
       };
   };
@@ -131,6 +125,9 @@ in
       machine.start()
       machine.wait_for_unit("multi-user.target")
 
+      # if the schema is broken, fail fast. helps during development.
+      machine.wait_for_unit("openldap.service")
+
       # TODO put this blocking into the systemd units?
       machine.wait_until_succeeds(
           "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
@@ -151,16 +148,25 @@ in
           raise
 
       with subtest("Test postmap lookups"):
-        test_lookup("postconf virtual_mailbox_maps", "alice@example.com", "alice@example.com")
+        test_lookup("postconf virtual_mailbox_maps", "alice@example.com", "alice")
-        test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "alice@example.com", "alice@example.com")
+        test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "alice@example.com", "alice")
 
-        test_lookup("postconf virtual_mailbox_maps", "bob@example.com", "bob@example.com")
+        test_lookup("postconf virtual_mailbox_maps", "bob@example.com", "bob")
-        test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "bob@example.com", "bob@example.com")
+        test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "bob@example.com", "bob")
 
       with subtest("Test doveadm lookups"):
         machine.succeed("doveadm user -u alice@example.com")
         machine.succeed("doveadm user -u bob@example.com")
 
+        machine.succeed("doveadm user -u alice")
+        machine.log(machine.succeed("doveadm user -u bob"))
+
+        machine.succeed("doveadm user -f uid bob@example.com | grep ${toString nodes.machine.mailserver.vmailUID}")
+        machine.succeed("doveadm user -f gid bob@example.com | grep ${toString nodes.machine.mailserver.vmailUID}")
+
+        machine.succeed("doveadm user -f home bob@example.com | grep ${nodes.machine.mailserver.mailDirectory}/ldap/f3b4e8ea-087f-42cc-95f0-cbfd99386092")
+        machine.succeed("doveadm user -f mail bob@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.mailserver.indexDir}/ldap/f3b4e8ea-087f-42cc-95f0-cbfd99386092'")
+
       with subtest("Files containing secrets are only readable by root"):
         machine.succeed("ls -l /run/postfix/*.cf | grep -e '-rw------- 1 root root'")
         machine.succeed("ls -l /run/dovecot2/dovecot-ldap.conf.ext | grep -e '-rw------- 1 root root'")
@@ -171,16 +177,16 @@ in
           "--smtp-port 587",
           "--smtp-starttls",
           "--smtp-host localhost",
-          "--smtp-username alice@example.com",
+          "--smtp-username alice",
           "--imap-host localhost",
-          "--imap-username bob@example.com",
+          "--imap-username bob",
           "--from-addr bob@example.com",
           "--to-addr aliceb@example.com",
           "--src-password-file <(echo '${alicePassword}')",
           "--dst-password-file <(echo '${bobPassword}')",
           "--ignore-dkim-spf"
         ]))
-        machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user alice@example.com'")
+        machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user alice'")
 
       with subtest("Test mail delivery via implicit TLS"):
         machine.succeed(" ".join([
@@ -188,9 +194,9 @@ in
           "--smtp-port 465",
           "--smtp-ssl",
           "--smtp-host localhost",
-          "--smtp-username alice@example.com",
+          "--smtp-username alice",
           "--imap-host localhost",
-          "--imap-username bob@example.com",
+          "--imap-username bob",
           "--from-addr alice@example.com",
           "--to-addr bob@example.com",
           "--src-password-file <(echo '${alicePassword}')",
@@ -204,9 +210,9 @@ in
           "--smtp-port 587",
           "--smtp-starttls",
           "--smtp-host localhost",
-          "--smtp-username alice@example.com",
+          "--smtp-username alice",
           "--imap-host localhost",
-          "--imap-username bob@example.com",
+          "--imap-username bob",
           "--from-addr alice@example.com",
           "--to-addr bob_fw@example.com",
           "--src-password-file <(echo '${alicePassword}')",
@@ -220,7 +226,7 @@ in
           "--smtp-port 465",
           "--smtp-ssl",
           "--smtp-host localhost",
-          "--smtp-username bob@example.com",
+          "--smtp-username bob",
           "--imap-host localhost",
           "--imap-username alice@example.com",
           "--from-addr bob_fw@example.com",
@@ -229,11 +235,7 @@ in
           "--dst-password-file <(echo '${alicePassword}')",
           "--ignore-dkim-spf"
         ]))
-        machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user bob@example.com'")
+        machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user bob'")
 
-      with subtest("Check dovecot mail and index locations"):
-        # If these paths change we need a migration
-        machine.succeed("doveadm user -f home bob@example.com | grep ${nodes.machine.mailserver.mailDirectory}/ldap/bob@example.com")
-        machine.succeed("doveadm user -f mail bob@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.mailserver.indexDir}/ldap/bob@example.com'")
     '';
 }