pixi recipes

.github/workflows/pixi-packages.yml

@@ -21,7 +21,11 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-latest, macos-15]
-        package_variant: [tsan-free-threading, asan, default]
+        package_variant:
+          - default
+          - freethreading
+          - asan
+          - tsan-freethreading
     if: github.repository == 'numpy/numpy'
     steps:
     - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
@@ -32,7 +36,9 @@ jobs:
 
     - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
       with:
-        pixi-version: v0.60.0
+        # FIXME regression in 0.63.0:
+        # https://github.com/prefix-dev/rattler-build/issues/2094
+        pixi-version: v0.62.2
         run-install: false
 
     - name: Build

pixi-packages/README.md

@@ -1,37 +1,109 @@
 # NumPy Pixi packages
 
-This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section)
-which can be built from the NumPy source code.
+This directory contains definitions for [Pixi
+packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can
+be built from the NumPy source code.
 
-Downstream developers can make use of these packages by adding them as Git dependencies in a
-[Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
-
-```toml
-[dependencies]
-numpy = { git = "https://github.com/numpy/numpy", subdirectory = "pixi-packages/asan" }
-```
+Downstream developers can make use of these packages by adding them as Git dependencies
+in a [Pixi workspace](https://pixi.sh/latest/first_workspace/).
 
 This is particularly useful when developers need to build NumPy from source
 (for example, for an ASan-instrumented build), as it does not require any manual
 clone or build steps. Instead, Pixi will automatically handle both the build
 and installation of the package.
 
-See https://github.com/scipy/scipy/pull/24066 for a full example of downstream use.
+See [scipy#24066](https://github.com/scipy/scipy/pull/24066) for a full example of
+downstream use.
 
+## Variants
 Each package definition is contained in a subdirectory.
-Currently defined package variants:
+All package variants include debug symbols.
+
+Currently defined variants:
+
+### `default`
+GIL-enabled build.
+
+Usage:
+```toml
+[dependencies]
+python = "*"
+numpy.git = "https://github.com/numpy/numpy"
+numpy.subdirectory = "pixi-packages/default"
+```
+See `default/pixi.toml` if you wish to use python git tip instead.
+
+*Tip:* you may change fork and add `numpy.rev = "<branch or git hash>"` to test unmerged
+PRs.
+
+### `freethreading`
+noGIL build.
+
+Usage:
+```toml
+[dependencies]
+python-freethreading = "*"
+numpy.git = "https://github.com/numpy/numpy"
+numpy.subdirectory = "pixi-packages/freethreading"
+```
+See `freethreading/pixi.toml` if you wish to use python git tip instead.
 
-- `default`
-- `asan`: ASan-instrumented build with `-Db_sanitize=address`
+### `asan`
+ASan-instrumented build with `-Db_sanitize=address`.
+
+Usage:
+```toml
+[dependencies]
+python.git = "https://github.com/python/cpython"
+python.subdirectory = "Tools/pixi-packages/asan"
+numpy.git = "https://github.com/numpy/numpy"
+numpy.subdirectory = "pixi-packages/asan"
+```
+
+### `tsan-freethreading`
+Freethreading TSan-instrumented build with `-Db_sanitize=thread`.
+
+Usage:
+```toml
+[dependencies]
+python.git = "https://github.com/python/cpython"
+python.subdirectory = "Tools/pixi-packages/tsan-freethreading"
+numpy.git = "https://github.com/numpy/numpy"
+numpy.subdirectory = "pixi-packages/tsan-freethreading"
+```
 
 ## Maintenance
 
 - Keep host dependency requirements up to date
-- For dependencies on upstream CPython Pixi packages, keep the git revision at a compatible version
+
+## Troubleshooting
+
+TSan builds may crash on Linux with
+```
+FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000
+```
+To fix it, try reducing `mmap_rnd_bits`:
+
+```bash
+$ sudo sysctl vm.mmap_rnd_bits
+vm.mmap_rnd_bits = 32  # too high for TSan
+$ sudo sysctl vm.mmap_rnd_bits=28  # reduce it
+vm.mmap_rnd_bits = 28
+```
 
 ## Opportunities for future improvement
 
-- More package variants (such as TSan, UBSan)
+- More package variants (such as UBSan)
 - Support for Windows
-- Using a single `pixi.toml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/2813
+- Using a single `pixi.toml` for all package variants is blocked on
+  [pixi#2813](https://github.com/prefix-dev/pixi/issues/2813)
 - Consider pinning dependency versions to guard against upstream breakages over time
+
+## Known issues
+- [numpy#30561](https://github.com/numpy/numpy/issues/30561): `default` and
+  `freethreading` recipes must be manually tweaked to compile against cpython git tip;
+  see `default/pixi.toml` and `freethreading/pixi.toml` for details.
+- [pixi#5226](https://github.com/prefix-dev/pixi/issues/5226): lock file is invalidated
+  on all `pixi` invocations
+- [rattler-build#2094](https://github.com/prefix-dev/rattler-build/issues/2094): pixi
+  0.63.0 introduces a regression regarding the license file; please skip it

pixi-packages/asan/pixi.toml

@@ -17,9 +17,9 @@ env.ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_
 extra-args = ["-Csetup-args=-Db_sanitize=address", "-Csetup-args=-Dbuildtype=debug"] 
 
 [package.host-dependencies]
-python.git = "https://github.com/python/cpython"
+python.git = "https://github.com/crusaderky/cpython"
 python.subdirectory = "Tools/pixi-packages/asan"
-python.rev = "8bb5b6e8ce61da41b5affd5eb12d9cc46b5af448"
+python.rev = "tsan"
 
 meson-python = "*"
 cython = "*"

pixi-packages/default/pixi.toml

@@ -13,9 +13,22 @@ version = "*"
 [package.build.config]
 extra-input-globs = ["**/*.c.src"]
 compilers = ["c", "cxx"]
+extra-args = ["-Csetup-args=-Dbuildtype=debug"]
 
 [package.host-dependencies]
-python = "*"
+# FIXME https://github.com/numpy/numpy/issues/30561
+# python = "*" prevents downstream from building cpython from sources.
+# Workaround: fork numpy, then uncomment one of the following and
+# comment  out the other.
+
+# Use latest cpython release from conda-forge
+# python = "*"
+
+# Use cpython git tip
+python.git = "https://github.com/crusaderky/cpython"
+python.subdirectory = "Tools/pixi-packages/default"
+python.rev = "tsan"
+
 meson-python = "*"
 cython = "*"
 uv = "*" # used to invoke the wheel build

pixi-packages/freethreading/pixi.toml

@@ -0,0 +1,34 @@
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+preview = ["pixi-build"]
+
+[package.build]
+source.path = "../.."
+
+[package.build.backend]
+name = "pixi-build-python"
+version = "*"
+
+[package.build.config]
+extra-input-globs = ["**/*.c.src"]
+compilers = ["c", "cxx"]
+extra-args = ["-Csetup-args=-Dbuildtype=debug"]
+
+[package.host-dependencies]
+# FIXME https://github.com/numpy/numpy/issues/30561
+# python = "*" prevents downstream from building cpython from sources.
+# Workaround: fork numpy, then uncomment one of the following and
+# comment  out the other.
+
+# Use latest cpython release from conda-forge
+# python-freethreading = "*"
+
+# Use cpython git tip
+python.git = "https://github.com/crusaderky/cpython"
+python.subdirectory = "Tools/pixi-packages/freethreading"
+python.rev = "tsan"
+
+meson-python = "*"
+cython = "*"
+uv = "*" # used to invoke the wheel build

pixi-packages/tsan-free-threading/pixi.toml → pixi-packages/tsan-freethreading/pixi.toml

@@ -17,9 +17,10 @@ env.TSAN_OPTIONS = "halt_on_error=0:allocator_may_return_null=1"
 extra-args = ["-Csetup-args=-Db_sanitize=thread", "-Csetup-args=-Dbuildtype=debug"]
 
 [package.host-dependencies]
-python.git = "https://github.com/ngoldbaum/cpython"
-python.subdirectory = "Tools/pixi-packages/tsan-free-threading"
-python.rev = "39df6fe96a2da23b09a7f8930eff0cb1f9259862"
+python.git = "https://github.com/crusaderky/cpython"
+python.subdirectory = "Tools/pixi-packages/tsan-freethreading"
+python.rev = "tsan"
+
 meson-python = "*"
 cython = "*"
-uv = "*" # used to invoke the wheel build
+uv = "*" # used to invoke the wheel build