diff --git a/.github/workflows/pixi-packages.yml b/.github/workflows/pixi-packages.yml index 861bbf22b42f..9372b3501ca2 100644 --- a/.github/workflows/pixi-packages.yml +++ b/.github/workflows/pixi-packages.yml @@ -21,7 +21,11 @@ jobs: fail-fast: false matrix: runs-on: [ubuntu-latest, macos-15] - package_variant: [tsan-free-threading, asan, default] + package_variant: + - default + - freethreading + - asan + - tsan-freethreading if: github.repository == 'numpy/numpy' steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -32,7 +36,9 @@ jobs: - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3 with: - pixi-version: v0.60.0 + # FIXME regression in 0.63.0: + # https://github.com/prefix-dev/rattler-build/issues/2094 + pixi-version: v0.62.2 run-install: false - name: Build diff --git a/pixi-packages/README.md b/pixi-packages/README.md index 9f1fed5fdb2c..bd47c91bcc92 100644 --- a/pixi-packages/README.md +++ b/pixi-packages/README.md @@ -1,37 +1,109 @@ # NumPy Pixi packages -This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) -which can be built from the NumPy source code. +This directory contains definitions for [Pixi +packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can +be built from the NumPy source code. -Downstream developers can make use of these packages by adding them as Git dependencies in a -[Pixi workspace](https://pixi.sh/latest/first_workspace/), like: - -```toml -[dependencies] -numpy = { git = "https://github.com/numpy/numpy", subdirectory = "pixi-packages/asan" } -``` +Downstream developers can make use of these packages by adding them as Git dependencies +in a [Pixi workspace](https://pixi.sh/latest/first_workspace/). This is particularly useful when developers need to build NumPy from source (for example, for an ASan-instrumented build), as it does not require any manual clone or build steps. Instead, Pixi will automatically handle both the build and installation of the package. -See https://github.com/scipy/scipy/pull/24066 for a full example of downstream use. +See [scipy#24066](https://github.com/scipy/scipy/pull/24066) for a full example of +downstream use. +## Variants Each package definition is contained in a subdirectory. -Currently defined package variants: +All package variants include debug symbols. + +Currently defined variants: + +### `default` +GIL-enabled build. + +Usage: +```toml +[dependencies] +python = "*" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/default" +``` +See `default/pixi.toml` if you wish to use python git tip instead. + +*Tip:* you may change fork and add `numpy.rev = ""` to test unmerged +PRs. + +### `freethreading` +noGIL build. + +Usage: +```toml +[dependencies] +python-freethreading = "*" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/freethreading" +``` +See `freethreading/pixi.toml` if you wish to use python git tip instead. -- `default` -- `asan`: ASan-instrumented build with `-Db_sanitize=address` +### `asan` +ASan-instrumented build with `-Db_sanitize=address`. + +Usage: +```toml +[dependencies] +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +``` + +### `tsan-freethreading` +Freethreading TSan-instrumented build with `-Db_sanitize=thread`. + +Usage: +```toml +[dependencies] +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/tsan-freethreading" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/tsan-freethreading" +``` ## Maintenance - Keep host dependency requirements up to date -- For dependencies on upstream CPython Pixi packages, keep the git revision at a compatible version + +## Troubleshooting + +TSan builds may crash on Linux with +``` +FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000 +``` +To fix it, try reducing `mmap_rnd_bits`: + +```bash +$ sudo sysctl vm.mmap_rnd_bits +vm.mmap_rnd_bits = 32 # too high for TSan +$ sudo sysctl vm.mmap_rnd_bits=28 # reduce it +vm.mmap_rnd_bits = 28 +``` ## Opportunities for future improvement -- More package variants (such as TSan, UBSan) +- More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/2813 +- Using a single `pixi.toml` for all package variants is blocked on + [pixi#2813](https://github.com/prefix-dev/pixi/issues/2813) - Consider pinning dependency versions to guard against upstream breakages over time + +## Known issues +- [numpy#30561](https://github.com/numpy/numpy/issues/30561): `default` and + `freethreading` recipes must be manually tweaked to compile against cpython git tip; + see `default/pixi.toml` and `freethreading/pixi.toml` for details. +- [pixi#5226](https://github.com/prefix-dev/pixi/issues/5226): lock file is invalidated + on all `pixi` invocations +- [rattler-build#2094](https://github.com/prefix-dev/rattler-build/issues/2094): pixi + 0.63.0 introduces a regression regarding the license file; please skip it diff --git a/pixi-packages/asan/pixi.toml b/pixi-packages/asan/pixi.toml index 19a6a6553806..c16937d07d3c 100644 --- a/pixi-packages/asan/pixi.toml +++ b/pixi-packages/asan/pixi.toml @@ -17,9 +17,9 @@ env.ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_ extra-args = ["-Csetup-args=-Db_sanitize=address", "-Csetup-args=-Dbuildtype=debug"] [package.host-dependencies] -python.git = "https://github.com/python/cpython" +python.git = "https://github.com/crusaderky/cpython" python.subdirectory = "Tools/pixi-packages/asan" -python.rev = "8bb5b6e8ce61da41b5affd5eb12d9cc46b5af448" +python.rev = "tsan" meson-python = "*" cython = "*" diff --git a/pixi-packages/default/pixi.toml b/pixi-packages/default/pixi.toml index 0b5d53e41ef2..296a3d956422 100644 --- a/pixi-packages/default/pixi.toml +++ b/pixi-packages/default/pixi.toml @@ -13,9 +13,22 @@ version = "*" [package.build.config] extra-input-globs = ["**/*.c.src"] compilers = ["c", "cxx"] +extra-args = ["-Csetup-args=-Dbuildtype=debug"] [package.host-dependencies] -python = "*" +# FIXME https://github.com/numpy/numpy/issues/30561 +# python = "*" prevents downstream from building cpython from sources. +# Workaround: fork numpy, then uncomment one of the following and +# comment out the other. + +# Use latest cpython release from conda-forge +# python = "*" + +# Use cpython git tip +python.git = "https://github.com/crusaderky/cpython" +python.subdirectory = "Tools/pixi-packages/default" +python.rev = "tsan" + meson-python = "*" cython = "*" uv = "*" # used to invoke the wheel build diff --git a/pixi-packages/freethreading/pixi.toml b/pixi-packages/freethreading/pixi.toml new file mode 100644 index 000000000000..1cd2b7ee7f3e --- /dev/null +++ b/pixi-packages/freethreading/pixi.toml @@ -0,0 +1,34 @@ +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["osx-arm64", "linux-64"] +preview = ["pixi-build"] + +[package.build] +source.path = "../.." + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +extra-input-globs = ["**/*.c.src"] +compilers = ["c", "cxx"] +extra-args = ["-Csetup-args=-Dbuildtype=debug"] + +[package.host-dependencies] +# FIXME https://github.com/numpy/numpy/issues/30561 +# python = "*" prevents downstream from building cpython from sources. +# Workaround: fork numpy, then uncomment one of the following and +# comment out the other. + +# Use latest cpython release from conda-forge +# python-freethreading = "*" + +# Use cpython git tip +python.git = "https://github.com/crusaderky/cpython" +python.subdirectory = "Tools/pixi-packages/freethreading" +python.rev = "tsan" + +meson-python = "*" +cython = "*" +uv = "*" # used to invoke the wheel build diff --git a/pixi-packages/tsan-free-threading/pixi.toml b/pixi-packages/tsan-freethreading/pixi.toml similarity index 70% rename from pixi-packages/tsan-free-threading/pixi.toml rename to pixi-packages/tsan-freethreading/pixi.toml index 8ab1117026e4..491aa0f2763c 100644 --- a/pixi-packages/tsan-free-threading/pixi.toml +++ b/pixi-packages/tsan-freethreading/pixi.toml @@ -17,9 +17,10 @@ env.TSAN_OPTIONS = "halt_on_error=0:allocator_may_return_null=1" extra-args = ["-Csetup-args=-Db_sanitize=thread", "-Csetup-args=-Dbuildtype=debug"] [package.host-dependencies] -python.git = "https://github.com/ngoldbaum/cpython" -python.subdirectory = "Tools/pixi-packages/tsan-free-threading" -python.rev = "39df6fe96a2da23b09a7f8930eff0cb1f9259862" +python.git = "https://github.com/crusaderky/cpython" +python.subdirectory = "Tools/pixi-packages/tsan-freethreading" +python.rev = "tsan" + meson-python = "*" cython = "*" -uv = "*" # used to invoke the wheel build \ No newline at end of file +uv = "*" # used to invoke the wheel build