lib.types.defaultTypeMerge: refactor functor.{payload,wrapped} merging (#350906)

lib/tests/modules.sh

@@ -516,6 +516,10 @@ checkConfigError 'The option .theOption.nested. in .other.nix. is already declar
 # Test that types.optionType leaves types untouched as long as they don't need to be merged
 checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survives-type-merge.nix
 
+# Test that specifying both functor.wrapped and functor.payload isn't allowed
+checkConfigError 'Type foo defines both `functor.payload` and `functor.wrapped` at the same time, which is not supported.' config.result ./default-type-merge-both.nix
+
+
 # Anonymous submodules don't get nixed by import resolution/deduplication
 # because of an `extendModules` bug, issue 168767.
 checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix

lib/tests/modules/default-type-merge-both.nix

@@ -0,0 +1,28 @@
+{ lib, options, ... }:
+let
+  foo = lib.mkOptionType {
+    name = "foo";
+    functor = lib.types.defaultFunctor "foo" // {
+      wrapped = lib.types.int;
+      payload = 10;
+    };
+  };
+in
+{
+  imports = [
+    {
+      options.foo = lib.mkOption {
+        type = foo;
+      };
+    }
+    {
+      options.foo = lib.mkOption {
+        type = foo;
+      };
+    }
+  ];
+
+  options.result = lib.mkOption {
+    default = builtins.seq options.foo null;
+  };
+}

lib/types.nix

@@ -83,23 +83,37 @@ rec {
   # Default type merging function
   # takes two type functors and return the merged type
   defaultTypeMerge = f: f':
-    let wrapped = f.wrapped.typeMerge f'.wrapped.functor;
-        payload = f.binOp f.payload f'.payload;
+    let mergedWrapped = f.wrapped.typeMerge f'.wrapped.functor;
+        mergedPayload = f.binOp f.payload f'.payload;
+
+        hasPayload = assert (f'.payload != null) == (f.payload != null); f.payload != null;
+        hasWrapped = assert (f'.wrapped != null) == (f.wrapped != null); f.wrapped != null;
     in
-    # cannot merge different types
+    # Abort early: cannot merge different types
     if f.name != f'.name
        then null
-    # simple types
-    else if    (f.wrapped == null && f'.wrapped == null)
-            && (f.payload == null && f'.payload == null)
-       then f.type
-    # composed types
-    else if (f.wrapped != null && f'.wrapped != null) && (wrapped != null)
-       then f.type wrapped
-    # value types
-    else if (f.payload != null && f'.payload != null) && (payload != null)
-       then f.type payload
-    else null;
+    else
+
+    if hasPayload then
+      if hasWrapped then
+        # Has both wrapped and payload
+        throw ''
+          Type ${f.name} defines both `functor.payload` and `functor.wrapped` at the same time, which is not supported.
+
+          Use either `functor.payload` or `functor.wrapped` but not both.
+
+          If your code worked before remove `functor.payload` from the type definition.
+        ''
+      else
+        # Has payload
+        if mergedPayload == null then null else f.type mergedPayload
+    else
+      if hasWrapped then
+        # Has wrapped
+        # TODO(@hsjobeki): This could also be a warning and removed in the future
+        if mergedWrapped == null then null else f.type mergedWrapped
+      else
+        f.type;
 
   # Default type functor
   defaultFunctor = name: {