From b978799f716271a46b9d03a336e367e6ecadb7fb Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 24 Oct 2024 11:38:30 +0200
Subject: [PATCH] lib.types.defaultTypeMerge: refactor
 functor.{payload,wrapped} merging

---
 lib/tests/modules.sh                          |  4 ++
 lib/tests/modules/default-type-merge-both.nix | 28 +++++++++++++
 lib/types.nix                                 | 42 ++++++++++++-------
 3 files changed, 60 insertions(+), 14 deletions(-)
 create mode 100644 lib/tests/modules/default-type-merge-both.nix

diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh
index ad2ea44c361a..2884c00d0768 100755
--- a/lib/tests/modules.sh
+++ b/lib/tests/modules.sh
@@ -516,6 +516,10 @@ checkConfigError 'The option .theOption.nested. in .other.nix. is already declar
 # Test that types.optionType leaves types untouched as long as they don't need to be merged
 checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survives-type-merge.nix
 
+# Test that specifying both functor.wrapped and functor.payload isn't allowed
+checkConfigError 'Type foo defines both `functor.payload` and `functor.wrapped` at the same time, which is not supported.' config.result ./default-type-merge-both.nix
+
+
 # Anonymous submodules don't get nixed by import resolution/deduplication
 # because of an `extendModules` bug, issue 168767.
 checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix
diff --git a/lib/tests/modules/default-type-merge-both.nix b/lib/tests/modules/default-type-merge-both.nix
new file mode 100644
index 000000000000..0b02b940f8b1
--- /dev/null
+++ b/lib/tests/modules/default-type-merge-both.nix
@@ -0,0 +1,28 @@
+{ lib, options, ... }:
+let
+  foo = lib.mkOptionType {
+    name = "foo";
+    functor = lib.types.defaultFunctor "foo" // {
+      wrapped = lib.types.int;
+      payload = 10;
+    };
+  };
+in
+{
+  imports = [
+    {
+      options.foo = lib.mkOption {
+        type = foo;
+      };
+    }
+    {
+      options.foo = lib.mkOption {
+        type = foo;
+      };
+    }
+  ];
+
+  options.result = lib.mkOption {
+    default = builtins.seq options.foo null;
+  };
+}
diff --git a/lib/types.nix b/lib/types.nix
index 6c4a66c4e3c0..82d7425ca643 100644
--- a/lib/types.nix
+++ b/lib/types.nix
@@ -83,23 +83,37 @@ rec {
   # Default type merging function
   # takes two type functors and return the merged type
   defaultTypeMerge = f: f':
-    let wrapped = f.wrapped.typeMerge f'.wrapped.functor;
-        payload = f.binOp f.payload f'.payload;
+    let mergedWrapped = f.wrapped.typeMerge f'.wrapped.functor;
+        mergedPayload = f.binOp f.payload f'.payload;
+
+        hasPayload = assert (f'.payload != null) == (f.payload != null); f.payload != null;
+        hasWrapped = assert (f'.wrapped != null) == (f.wrapped != null); f.wrapped != null;
     in
-    # cannot merge different types
+    # Abort early: cannot merge different types
     if f.name != f'.name
        then null
-    # simple types
-    else if    (f.wrapped == null && f'.wrapped == null)
-            && (f.payload == null && f'.payload == null)
-       then f.type
-    # composed types
-    else if (f.wrapped != null && f'.wrapped != null) && (wrapped != null)
-       then f.type wrapped
-    # value types
-    else if (f.payload != null && f'.payload != null) && (payload != null)
-       then f.type payload
-    else null;
+    else
+
+    if hasPayload then
+      if hasWrapped then
+        # Has both wrapped and payload
+        throw ''
+          Type ${f.name} defines both `functor.payload` and `functor.wrapped` at the same time, which is not supported.
+
+          Use either `functor.payload` or `functor.wrapped` but not both.
+
+          If your code worked before remove `functor.payload` from the type definition.
+        ''
+      else
+        # Has payload
+        if mergedPayload == null then null else f.type mergedPayload
+    else
+      if hasWrapped then
+        # Has wrapped
+        # TODO(@hsjobeki): This could also be a warning and removed in the future
+        if mergedWrapped == null then null else f.type mergedWrapped
+      else
+        f.type;
 
   # Default type functor
   defaultFunctor = name: {