From de1fdc9fe0edb5f560f6a82a6fb57def80f265fc Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 3 Sep 2024 20:05:14 +1200 Subject: [PATCH] python3Packages.mkPythonEditablePackage: init --- doc/languages-frameworks/python.section.md | 44 +++++++++ .../interpreters/python/editable.nix | 99 +++++++++++++++++++ .../python/python-packages-base.nix | 4 +- .../development/interpreters/python/tests.nix | 39 +++++++- 4 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/interpreters/python/editable.nix diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index d94acf130a2c..77828110cc4f 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -374,6 +374,50 @@ mkPythonMetaPackage { } ``` +#### `mkPythonEditablePackage` function {#mkpythoneditablepackage-function} + +When developing Python packages it's common to install packages in [editable mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html). +Like `mkPythonMetaPackage` this function exists to create an otherwise empty package, but also containing a pointer to an impure location outside the Nix store that can be changed without rebuilding. + +The editable root is passed as a string. Normally `.pth` files contains absolute paths to the mutable location. This isn't always ergonomic with Nix, so environment variables are expanded at runtime. +This means that a shell hook setting up something like a `$REPO_ROOT` variable can be used as the relative package root. + +As an implementation detail, the [PEP-518](https://peps.python.org/pep-0518/) `build-system` specified won't be used, but instead the editable package will be built using [hatchling](https://pypi.org/project/hatchling/). +The `build-system`'s provided will instead become runtime dependencies of the editable package. + +Note that overriding packages deeper in the dependency graph _can_ work, but it's not the primary use case and overriding existing packages can make others break in unexpected ways. + +``` nix +{ pkgs ? import { } }: + +let + pyproject = pkgs.lib.importTOML ./pyproject.toml; + + myPython = pkgs.python.override { + self = myPython; + packageOverrides = pyfinal: pyprev: { + # An editable package with a script that loads our mutable location + my-editable = pyfinal.mkPythonEditablePackage { + # Inherit project metadata from pyproject.toml + pname = pyproject.project.name; + inherit (pyproject.project) version; + + # The editable root passed as a string + root = "$REPO_ROOT/src"; # Use environment variable expansion at runtime + + # Inject a script (other PEP-621 entrypoints are also accepted) + inherit (pyproject.project) scripts; + }; + }; + }; + + pythonEnv = testPython.withPackages (ps: [ ps.my-editable ]); + +in pkgs.mkShell { + packages = [ pythonEnv ]; +} +``` + #### `python.buildEnv` function {#python.buildenv-function} Python environments can be created using the low-level `pkgs.buildEnv` function. diff --git a/pkgs/development/interpreters/python/editable.nix b/pkgs/development/interpreters/python/editable.nix new file mode 100644 index 000000000000..df63bbd14fe3 --- /dev/null +++ b/pkgs/development/interpreters/python/editable.nix @@ -0,0 +1,99 @@ +{ + buildPythonPackage, + lib, + hatchling, + tomli-w, +}: +{ + pname, + version, + + # Editable root as string. + # Environment variables will be expanded at runtime using os.path.expandvars. + root, + + # Arguments passed on verbatim to buildPythonPackage + derivationArgs ? { }, + + # Python dependencies + dependencies ? [ ], + optional-dependencies ? { }, + + # PEP-518 build-system https://peps.python.org/pep-518 + build-system ? [ ], + + # PEP-621 entry points https://peps.python.org/pep-0621/#entry-points + scripts ? { }, + gui-scripts ? { }, + entry-points ? { }, + + passthru ? { }, + meta ? { }, +}: + +# Create a PEP-660 (https://peps.python.org/pep-0660/) editable package pointing to an impure location outside the Nix store. +# The primary use case of this function is to enable local development workflows where the local package is installed into a virtualenv-like environment using withPackages. + +assert lib.isString root; +let + # In editable mode build-system's are considered to be runtime dependencies. + dependencies' = dependencies ++ build-system; + + pyproject = { + # PEP-621 project table + project = { + name = pname; + inherit + version + scripts + gui-scripts + entry-points + ; + dependencies = map lib.getName dependencies'; + optional-dependencies = lib.mapAttrs (_: lib.getName) optional-dependencies; + }; + + # Allow empty package + tool.hatch.build.targets.wheel.bypass-selection = true; + + # Include our editable pointer file in build + tool.hatch.build.targets.wheel.force-include."_${pname}.pth" = "_${pname}.pth"; + + # Build editable package using hatchling + build-system = { + requires = [ "hatchling" ]; + build-backend = "hatchling.build"; + }; + }; + +in +buildPythonPackage ( + { + inherit + pname + version + optional-dependencies + passthru + meta + ; + dependencies = dependencies'; + + pyproject = true; + + unpackPhase = '' + python -c "import json, tomli_w; print(tomli_w.dumps(json.load(open('$pyprojectContentsPath'))))" > pyproject.toml + echo 'import os.path, sys; sys.path.insert(0, os.path.expandvars("${root}"))' > _${pname}.pth + ''; + + build-system = [ hatchling ]; + } + // derivationArgs + // { + # Note: Using formats.toml generates another intermediary derivation that needs to be built. + # We inline the same functionality for better UX. + nativeBuildInputs = (derivationArgs.nativeBuildInputs or [ ]) ++ [ tomli-w ]; + pyprojectContents = builtins.toJSON pyproject; + passAsFile = [ "pyprojectContents" ]; + preferLocalBuild = true; + } +) diff --git a/pkgs/development/interpreters/python/python-packages-base.nix b/pkgs/development/interpreters/python/python-packages-base.nix index edd0937098b6..9fad8e56d5d4 100644 --- a/pkgs/development/interpreters/python/python-packages-base.nix +++ b/pkgs/development/interpreters/python/python-packages-base.nix @@ -61,6 +61,8 @@ let removePythonPrefix = lib.removePrefix namePrefix; + mkPythonEditablePackage = callPackage ./editable.nix { }; + mkPythonMetaPackage = callPackage ./meta-package.nix { }; # Convert derivation to a Python module. @@ -99,7 +101,7 @@ in { inherit buildPythonPackage buildPythonApplication; inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf; inherit toPythonModule toPythonApplication; - inherit mkPythonMetaPackage; + inherit mkPythonMetaPackage mkPythonEditablePackage; python = toPythonModule python; diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index 2cd29ca99032..99bc38890c6a 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -122,6 +122,43 @@ let } ); + # Test editable package support + editableTests = let + testPython = python.override { + self = testPython; + packageOverrides = pyfinal: pyprev: { + # An editable package with a script that loads our mutable location + my-editable = pyfinal.mkPythonEditablePackage { + pname = "my-editable"; + version = "0.1.0"; + root = "$NIX_BUILD_TOP/src"; # Use environment variable expansion at runtime + # Inject a script + scripts = { + my-script = "my_editable.main:main"; + }; + }; + }; + }; + + + in { + editable-script = runCommand "editable-test" { + nativeBuildInputs = [ (testPython.withPackages (ps: [ ps.my-editable ])) ]; + } '' + mkdir -p src/my_editable + + cat > src/my_editable/main.py << EOF + def main(): + print("hello mutable") + EOF + + test "$(my-script)" == "hello mutable" + test "$(python -c 'import sys; print(sys.path[1])')" == "$NIX_BUILD_TOP/src" + + touch $out + ''; + }; + # Tests to ensure overriding works as expected. overrideTests = let extension = self: super: { @@ -192,4 +229,4 @@ let ''; }; -in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests // condaTests) +in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests // condaTests // editableTests)