diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 2fddfdeb..86b84c4a 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -29,6 +29,8 @@ jobs: ( cd kernels-data/bindings/python && cargo fmt --all -- --check ) - name: Cargo fmt (kernel-builder) run: ( cd kernel-builder && cargo fmt --all -- --check ) + - name: Cargo fmt (kernel-port) + run: ( cd kernel-port && cargo fmt --all -- --check ) clippy: name: Clippy @@ -56,6 +58,8 @@ jobs: ( cd kernels-data/bindings/python && cargo clippy -- -D warnings ) - name: Clippy (kernel-builder) run: ( cd kernel-builder && cargo clippy -- -D warnings ) + - name: Clippy (kernel-port) + run: ( cd kernel-port && cargo clippy -- -D warnings ) test: name: Test @@ -78,3 +82,5 @@ jobs: run: cargo test -p kernels-data - name: Test (kernel-builder) run: cargo test -p hf-kernel-builder + - name: Test (kernel-port) + run: cargo test -p kernel-port diff --git a/.gitignore b/.gitignore index 26741b98..3bf8a13e 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,6 @@ poetry.lock # Nix examples/kernels/**/flake.lock + + +kernel-port/examples \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index aa218881..c11e6692 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,16 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" +dependencies = [ + "anstyle", + "unicode-width 0.2.2", +] + [[package]] name = "anstream" version = "1.0.0" @@ -90,9 +100,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -123,7 +133,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -223,7 +233,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.117", ] [[package]] @@ -320,9 +330,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -339,9 +349,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -360,14 +370,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -397,7 +407,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -419,7 +429,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width", + "unicode-width 0.2.2", "windows-sys 0.59.0", ] @@ -572,7 +582,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -583,7 +593,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -612,7 +622,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "unicode-xid", ] @@ -645,7 +655,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -656,7 +666,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -699,7 +709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -817,7 +827,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -864,7 +874,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -927,7 +937,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -959,9 +969,9 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" dependencies = [ "aho-corasick", "bstr", @@ -1394,7 +1404,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width", + "unicode-width 0.2.2", "web-time", ] @@ -1422,7 +1432,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1491,7 +1501,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -1510,7 +1520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1535,6 +1545,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kdl" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a29e7b50079ff44549f68c0becb1c73d7f6de2a4ea952da77966daf3d4761e" +dependencies = [ + "miette", + "num", + "winnow 0.6.24", +] + [[package]] name = "kernel-abi-check" version = "0.17.0-dev0" @@ -1549,6 +1570,22 @@ dependencies = [ "toml", ] +[[package]] +name = "kernel-port" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "globset", + "kdl", + "libcst", + "self_cell", + "serde", + "serde_json", + "sha2", + "similar", +] + [[package]] name = "kernels-data" version = "0.17.0-dev0" @@ -1620,6 +1657,31 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libcst" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aea7143e4a0ed59b87a1ee71e198500889f8b005311136be15e84c97a6fcd8d" +dependencies = [ + "annotate-snippets", + "libcst_derive", + "memchr", + "paste", + "peg", + "regex", + "thiserror 2.0.18", +] + +[[package]] +name = "libcst_derive" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0903173ea316c34a44d0497161e04d9210af44f5f5e89bf2f55d9a254c9a0e8d" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "libgit2-sys" version = "0.18.5+1.9.4" @@ -1808,6 +1870,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if 1.0.4", + "unicode-width 0.1.14", +] + [[package]] name = "mime" version = "0.3.17" @@ -1880,7 +1952,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1904,7 +1976,40 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", ] [[package]] @@ -1922,6 +2027,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2054,6 +2180,33 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "peg" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aad070be5b63aa72103f2fcdd70a83adbd5e90112ce5b574171ff1c65501773" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8ef6825cae95355031ae26a99b616a2a21f22ba2de0197c43dfb05acbe7ee" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7011d97b484a5ebdc4b1fdb3b12d5e4bbbea56e9d22b688f2e79e04b65a7d8a6" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2115,7 +2268,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2167,7 +2320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -2224,7 +2377,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2237,7 +2390,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2427,9 +2580,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2547,7 +2700,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2604,7 +2757,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2750,6 +2903,12 @@ dependencies = [ "libc", ] +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -2793,7 +2952,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2817,7 +2976,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2900,6 +3059,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "similar" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f66ca1f7aca2474dc10c942eb22feffc897735f54cd1db90138c2fddb490987" +dependencies = [ + "bstr", +] + [[package]] name = "siphasher" version = "1.0.2" @@ -2979,6 +3147,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2996,7 +3175,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3050,7 +3229,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3079,7 +3258,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3090,7 +3269,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3190,7 +3369,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3248,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3283,7 +3462,7 @@ dependencies = [ "serde_spanned", "toml_datetime", "toml_write", - "winnow", + "winnow 0.7.15", ] [[package]] @@ -3369,7 +3548,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3522,6 +3701,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -3731,7 +3916,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -3855,7 +4040,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3917,7 +4102,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3928,7 +4113,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4076,6 +4261,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.15" @@ -4121,7 +4315,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4137,7 +4331,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -4345,7 +4539,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4366,7 +4560,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4386,7 +4580,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4426,7 +4620,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3a1f7926..99ec9a02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,6 @@ members = [ "kernel-builder", "kernels-data", "kernels-data/bindings/python", + "kernel-port", ] resolver = "2" diff --git a/flake.nix b/flake.nix index b42ce72e..428dd604 100644 --- a/flake.nix +++ b/flake.nix @@ -223,7 +223,7 @@ formatter = pkgs.nixfmt-tree; packages = rec { - inherit (buildSet.pkgs) kernel-builder; + inherit (buildSet.pkgs) kernel-builder kernel-port; inherit (buildSet.pkgs.python3.pkgs) kernels; update-build = pkgs.writeShellScriptBin "update-build" '' diff --git a/kernel-port/Cargo.toml b/kernel-port/Cargo.toml new file mode 100644 index 00000000..5daa8f30 --- /dev/null +++ b/kernel-port/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "kernel-port" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.104" +clap = { version = "4.6.6", features = ["derive"] } +globset = "0.4.20" +kdl = "6.5" +libcst = { version = "1.8.6", default-features = false } +self_cell = "1.3.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sha2 = "0.11.0" +similar = "3.2.0" + +# Clippy runs at pedantic, plus two nursery lints worth the churn. +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +use_self = "warn" +needless_pass_by_ref_mut = "warn" + +# The manifest is a String built by appending. write! plus an unwrap on a write +# that cannot fail would read worse than push_str(&format!(..)). +format_push_string = "allow" + +# This is a workspace-internal tool, not a published library: its callers do not +# match on error variants, and every accessor returns the value that is the +# point of calling it. +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" + +# The CLI flags are bools because they are flags. +struct_excessive_bools = "allow" diff --git a/kernel-port/README.md b/kernel-port/README.md new file mode 100644 index 00000000..048ba9b4 --- /dev/null +++ b/kernel-port/README.md @@ -0,0 +1,618 @@ +# kernel-port + +> [!WARNING] +> This is an experiment. The recipe language, the op set, and the CLI are all subject to change without notice, and nothing here is covered by any stability guarantee. Do not depend on it (yet). + +Ports a kernel repo into the [kernel-builder](https://github.com/huggingface/kernel-builder) layout by running a recipe. + +- A port is a `port.kdl` recipe plus an `overlay/` directory of checked-in files. +- Same pins and same recipe gives byte-identical output, every run. +- Every op fails loudly on drift, so upstream changes cannot silently mis-port. +- Judgment is spent once, while writing the recipe. Running it is mechanical. + +## The recipe + +A recipe is not a programming language. It is an ordered list of operations, written as a [KDL 2.0](https://kdl.dev) document. + +- One node per op, with `key="value"` properties. +- `//` comments, and raw multi-line strings (`#"""..."""#`) for exact-text payloads. +- No variables, no expressions, no control flow. Every pin is a literal. +- KDL features with no meaning here are rejected rather than ignored: positional arguments, children blocks, type annotations. +- A recipe kept on disk opens with a `recipe version=N` header, declaring the format it was written against. See [versioning](#versioning). + +## See it work, without a checkout + +Two flags remove the setup: + +- `-e` takes the recipe inline, instead of a path to one. +- `--file path=content` supplies the input tree, instead of `--dir`. + +Nothing is read from disk and nothing is written to it, so one command shows what an op does: + +```sh +kernel-port -e 'relativize_imports in="pkg/**" package_root="pkg" changes=1' \ + --file 'pkg/__init__.py=from pkg.ops import hello' +``` + +``` +[line 1] relativize_imports rewrote 1 import(s) in 1 file(s) +M pkg/__init__.py + +FILE: pkg/__init__.py +from .ops import hello +``` + +Change `changes=1` to `changes=2` and the same command shows the other half of the design. The run refuses to proceed rather than porting something you did not sign off on: + +``` +error: recipe line 1: relativize_imports: expected exactly 2 change(s) but made 1 - upstream drifted; review the new rewrites and update changes= +``` + +The [cookbook](#cookbook) is one such command per op. + +## Running a real port + +```sh +cargo run -p kernel-port -- .kdl --dir --out \ + [--vendor name=dir] [--dry-run] [--diff] [--print] [--partial] +``` + +- `--dir` is the upstream checkout. It is read, never written. +- `--out` writes the ported tree, wiped and regenerated on every run. Without it, `--dir` is modified in place. +- `--dry-run` computes the changes and writes nothing. +- `--diff` prints unified diffs for changed files, moves included. +- `--print` dumps every file in the resulting tree. +- `--vendor name=dir` supplies a second pinned checkout for [`vendor`](#vendor). +- `--partial` writes the state as of the last successful op to `--out`, to inspect what a failing op saw. The run still exits non-zero. + +## Ops + +Each op links to its entry in the [cookbook](#cookbook) below, which gives the full argument list, a runnable example, and the failure modes. + +| Op | What it does | +| --- | --- | +| [`source`](#source) | Pin the upstream: verify the checkout's HEAD commit, origin URL, and cleanliness | +| [`vendor`](#vendor) | Verify a second pinned upstream (given via `--vendor`) and copy a subtree from it | +| [`prune`](#prune) | Delete everything except the given globs | +| [`delete`](#delete) | Delete files matching a glob (must match something) | +| [`move`](#move) | Rename a file or directory | +| [`overlay`](#overlay) | Copy checked-in files (bindings, flake, docs) over the workspace | +| [`replace`](#replace) | Exact-text find/replace with an optional occurrence `count` | +| [`strip_suffix`](#strip_suffix) | Remove one literal suffix from every matched file, with a pinned `files=N` | +| [`expect`](#expect) | Guard: assert an exact text occurs `count` times (0 asserts absence), or that a glob matches `files=N` | +| [`convert_import`](#convert_import) | Rewrite `import a.b.c as x` into `from a.b import c as x` | +| [`remap_module`](#remap_module) | Rewrite `from a.b import x` module prefixes onto a new namespace | +| [`relativize_imports`](#relativize_imports) | Rewrite absolute intra-package imports to minimal-dot relative form | +| [`kernelize_imports`](#kernelize_imports) | Resolve imports of a package through a Hub kernel | +| [`ensure_import`](#ensure_import) | Ensure a module has an explicit top-level `from` import | +| [`ensure_init`](#ensure_init) | Add an empty `__init__.py` to any package dir missing one | +| [`kernel`](#kernel) | Record one `[kernel.]` section for the manifest | +| [`manifest`](#manifest) | Generate `build.toml` from the recorded kernel sections (or noarch mode) | + +## What the runner enforces + +On top of whatever the recipe says: + +- `source` and `vendor` require a full 40-character SHA and a clean checkout, untracked files included. +- Every op is built before the first one runs, so an argument typo or a bad glob anywhere fails the recipe before anything is mutated. +- After the last op, every added or modified Python file must still parse. +- No absolute in-package import may remain under `torch-ext`. The Hub loads a kernel under a build-variant directory name, so only relative intra-package imports resolve at run time. +- `--out` runs write a `port-provenance.json`: recipe hash, runner version, pinned sources, output tree hash. All deterministic, so the file reproduces byte-for-byte. + +## Versioning + +A recipe is re-run later, by someone else, against a newer build of this tool. The header says which format it was written against. + +```kdl +recipe version=1 +``` + +- The header must be the first node in the file. Comments above it are fine. +- A recipe file that does not declare a version is rejected. An inline `-e` recipe may omit it, since it does not outlive the command. +- A version this build does not implement is rejected, rather than run under a meaning the author never saw. +- The version is recorded as `format` in `port-provenance.json`. + +`version` is bumped when a change would give an existing recipe a *different meaning*, not when it gains a new capability. Adding an op or an optional argument leaves every existing recipe alone, so it is not a bump. Changing what an existing argument does is. + +## Cookbook + +One entry per op: what it takes, what it guarantees, a command you can run as written, and what makes it fail. Ops run top to bottom against an in-memory copy of the tree; nothing reaches disk until every op has succeeded. + +Conventions used throughout: + +- **Globs** match repo-relative paths with `/` separators. `*` does not cross a `/`, `**` does. `{a,b}` alternates. Args that take several globs (`src`, `keep`, `torch_src`) split on commas outside braces. +- **Pins** (`count=`, `files=`, `changes=`) are literal integers that must match exactly. They are the point: they turn "upstream changed" from a silent mis-port into a failed run. +- **Payload strings** use `"..."` with `\n`/`\t`/`\"` escapes, or KDL's raw multi-line form for exact text that contains quotes and newlines: + ```kdl + replace in="a.cpp" count=1 with="" find=#""" + static auto registry = torch::RegisterOperators() + .op("pkg::thing", &thing); + """# + ``` + The first and last newlines are the delimiters, so the payload above starts at `static` and ends at `;`. +- The examples below build their input with `--file path=content` so they run with no repository at all. `$'...'` is shell syntax for a string containing real newlines. With no `--out`, the run prints the resulting tree; a trailing `...` in an output block means the rest of that dump is elided here. + +### The header + +#### `recipe` + +```kdl +recipe version=1 +``` + +Declare the recipe format, as the first node in the file. Not an op: it runs nothing and is not part of the pipeline. Required in a recipe file, optional inline. See [versioning](#versioning). + +Fails when: the version is one this build does not implement, or the header is not first. + +### Pinning the upstream + +#### `source` + +```kdl +source repo="" commit="<40-char sha>" +``` + +Assert what `--dir` is. Verifies the checkout's `HEAD` is exactly `commit`, that its `origin` URL is `repo`, and that the working tree is clean (untracked files included), then records the pin in `port-provenance.json`. Every recipe that ports a real repository starts with this line. + +```kdl +source repo="https://github.com/rusty1s/pytorch_scatter" commit="f514c10f920b5aeed2eb162092f0ad20d3edee52" +``` + +Fails when: `--dir` is not a git checkout, sits at a different commit, has a different origin, or has uncommitted or untracked changes. + +#### `vendor` + +```kdl +vendor name="" repo="" commit="<40-char sha>" path="" to="" +``` + +The same verification for a second repository, supplied on the command line as `--vendor =`, then copies its `path` subtree into the workspace at `to`. For kernels that vendor a dependency's sources instead of depending on it. + +```kdl +vendor name="quack" repo="https://github.com/Dao-AILab/quack" commit="" \ + path="quack/cute" to="torch-ext/kernel/cute" +``` + +Fails when: no `--vendor` was passed for that name, the checkout drifts from the pin, or `path` is not a directory in it. + +### Shaping the tree + +#### `prune` + +```kdl +prune keep="[,...]" +``` + +Delete everything the globs do not match. This is the port's statement of what it carries over, so it belongs near the top: whatever upstream adds later lands outside `keep` and is dropped, rather than silently shipping. + +```sh +kernel-port -e 'prune keep="csrc/**,pkg/**"' \ + --file 'csrc/k.cu=// kernel' --file 'pkg/__init__.py=x' --file 'setup.py=setup()' +``` + +``` +[line 1] prune removed 1 file(s), kept 2 +D setup.py +... +``` + +Fails when: `keep` is empty, or any glob in it matches nothing (a stale keep entry is a bug, not a no-op). + +#### `delete` + +```kdl +delete in="" +``` + +Delete the matching files. Use it for a handful of paths; use `prune` when the list of what to keep is shorter than the list of what to drop. + +```sh +kernel-port -e 'delete in="**/*.pyc"' --file 'pkg/a.py=a' --file 'pkg/a.pyc=binary' +``` + +``` +[line 1] delete removed 1 file(s) +D pkg/a.pyc +... +``` + +Fails when: the glob matches nothing. + +#### `move` + +```kdl +move from="" to="" +``` + +Rename a file, or a whole directory if `from` names one. Moves are recorded, so `--diff` can show a moved file's content change as `old path -> new path` instead of a delete plus an add. + +```sh +kernel-port -e 'move from="csrc" to="hello-kernel"' \ + --file 'csrc/k.cu=// kernel' --file 'csrc/k.h=// header' +``` + +``` +[line 1] move moved 2 file(s) to "hello-kernel" +A hello-kernel/k.cu +A hello-kernel/k.h +D csrc/k.cu +D csrc/k.h +... +``` + +Fails when: `from` matches no file or directory, or a destination path already exists. + +#### `overlay` + +```kdl +overlay from="" +``` + +Copy a directory of checked-in files over the workspace, overwriting what is there. `from` is relative to the recipe file. This is where the files with no upstream equivalent live: `torch_binding.cpp`, `flake.nix`, a CARD.md. Keep it small - anything derivable from upstream should be an op, not an overlay file, so that upstream drift is detected rather than papered over. + +How big the overlay gets depends on the port. An external repository that already follows kernel-builder conventions needs almost nothing beyond a `flake.nix` - a PR against such a repo is essentially `port.kdl` plus that one file. A vendored port that combines upstream code with custom additions (the common case in kernels-community) checks all of those additions in as overlay files, so a larger overlay is normal there. The rule is the same either way: overlay files are for content with no upstream equivalent, and anything derived from upstream belongs in an op. + +```kdl +overlay from="overlay" +``` + +Fails when: the directory does not exist or contains no files. + +### Editing text + +#### `replace` + +```kdl +replace in="" find="" with="" [count=N] +``` + +Exact-text find and replace across every matching file. Without `count`, the +text must occur exactly once in every matched file. When provided, `count` is +the required total number of occurrences across all matched files. `with=""` +deletes the text. No regexes and no capture groups: what you name is what gets +rewritten. + +```sh +kernel-port -e 'replace in="*.cpp" find="TORCH_EXTENSION_NAME" with="ops" count=2' \ + --file $'b.cpp=TORCH_LIBRARY(TORCH_EXTENSION_NAME, m) {}\nREGISTER(TORCH_EXTENSION_NAME)' +``` + +``` +[line 1] replace 2 replacement(s) in 1 file(s) +M b.cpp + +FILE: b.cpp +TORCH_LIBRARY(ops, m) {} +REGISTER(ops) +``` + +Fails when: the glob matches no files, or the occurrence total is not exactly `count`. The error names the per-file counts it did find. + +#### `strip_suffix` + +```kdl +strip_suffix in="" suffix="" files=N +``` + +Remove one literal suffix from the end of every matching file, with both the number of files and the suffix on each of them pinned. For trailing content that upstream appends uniformly - a generated footer, a license tail. + +```sh +kernel-port -e 'strip_suffix in="*.h" suffix="\n// EOF\n" files=1' \ + --file $'k.h=#pragma once\n// EOF\n' +``` + +``` +[line 1] strip_suffix stripped suffix from 1 file(s) +M k.h +... +``` + +Fails when: the file count is not `files`, or any matched file does not end with `suffix`. + +#### `expect` + +```kdl +expect in="" find="" count=N +expect in="" files=N +``` + +A guard that changes nothing. The first form asserts an exact text occurs exactly `count` times across the matched files - `count=0` asserts absence. The second asserts the glob matches exactly `N` files. + +Reach for it to state an invariant the rest of the recipe depends on but does not itself enforce: that no absolute import survived, that the upstream source list is still the size you reviewed. Guards hold regardless of which op was supposed to do the work, so they keep holding when the recipe is edited. + +```sh +kernel-port -e 'expect in="**/*.cu" files=3' --file 'a.cu=x' --file 'sub/b.cu=y' +``` + +``` +error: recipe line 1: expect: expected "**/*.cu" to match exactly 3 file(s), found 2 - the upstream file set drifted; update the port definition (a.cu, sub/b.cu) +``` + +Fails when: the count or file count does not match. `find` and `files` are mutually exclusive. + +### Rewriting Python imports + +These operations go through libcst, so comments, quoting, and formatting survive byte-for-byte. Each takes an optional `changes=N` pinning exactly how many import statements it rewrites or adds, which is what stops a newly added upstream file from being changed silently. + +#### `convert_import` + +```kdl +convert_import in="" prefix="" [changes=N] +``` + +Rewrite `import a.b.c as x` into `from a.b import c as x`, for modules under `prefix`. The `from` form is what the later ops can relativize; the `import` form cannot be made relative at all. + +```sh +kernel-port -e 'convert_import in="**/*.py" prefix="pkg" changes=1' \ + --file $'m.py=import pkg.ops as ops\nimport os' +``` + +``` +[line 1] convert_import converted 1 import(s) in 1 file(s) +M m.py + +FILE: m.py +from pkg import ops as ops +import os +``` + +`import os` is untouched: only `prefix` is in scope. + +#### `remap_module` + +```kdl +remap_module in="" from="" to="" [changes=N] +``` + +Move a module prefix onto a new namespace. Prefix matching is boundary-aware, so `from="pkg.utils"` does not touch `pkg.utils_extra`. + +```sh +kernel-port -e 'remap_module in="**/*.py" from="pkg" to="torch_ext.pkg" changes=2' \ + --file $'m.py=from pkg.ops import a\nfrom pkg.util import b' +``` + +``` +[line 1] remap_module rewrote 2 import(s) in 1 file(s) +M m.py + +FILE: m.py +from torch_ext.pkg.ops import a +from torch_ext.pkg.util import b +``` + +#### `relativize_imports` + +```kdl +relativize_imports in="" package_root="" [root_relative=#true] [changes=N] +``` + +Rewrite absolute intra-package imports to their minimal-dot relative form. This is required, not cosmetic: the Hub loads a kernel under a build-variant directory name, so an absolute self-import resolves to nothing at run time. The runner re-checks this after the last op regardless of whether you ran this op. + +`package_root` is the package directory itself (`torch-ext/`); each file's own package is derived from where it sits under it. `root_relative=#true` rewrites relative to the package root instead of the file, keeping the full module path visible (`from ...ops import base` rather than `from . import base`). + +```sh +kernel-port -e 'relativize_imports in="pkg/**" package_root="pkg" changes=1' \ + --file 'pkg/__init__.py=from pkg.ops import hello' +``` + +``` +[line 1] relativize_imports rewrote 1 import(s) in 1 file(s) +M pkg/__init__.py + +FILE: pkg/__init__.py +from .ops import hello +``` + +#### `kernelize_imports` + +```kdl +kernelize_imports in="" package="" \ + kernel="" version=N [changes=N] +``` + +Resolve static imports rooted at `package` through `kernels.get_kernel`. The +rewrite preserves the name each import binds and stays at the original lexical +location, including inside functions and conditionals. Submodules are loaded +with `importlib` using the kernel module's generated runtime name, rather than +assuming the submodule is already exposed as an attribute. + +```sh +kernel-port -e 'kernelize_imports in="tests/**" package="pkg" kernel="org/pkg" version=1 changes=2' \ + --file $'tests/test_x.py=import pkg\nfrom pkg.layers import Layer as L\n' +``` + +The result includes one lazy helper per changed file and concise bindings at +each original import location: + +```python +__kernel_port_pkg_root = None +def __kernel_port_pkg(module=""): + global __kernel_port_pkg_root + if __kernel_port_pkg_root is None: + __kernel_port_pkg_root = __import__("kernels").get_kernel("org/pkg", version=1) + root = __kernel_port_pkg_root + if not module: + return root + return __import__("importlib").import_module(root.__name__ + "." + module) + +pkg = __kernel_port_pkg() +L = getattr(__kernel_port_pkg("layers"), "Layer") +``` + +The helper name is made collision-free against the input file, and it does not +load the kernel until an original import location calls it. It then caches that +root once per changed file. After rewriting, the op parses the result again and +verifies that no matching static import remains. + +Wildcard imports, parenthesized imports, and mixed multi-name `import` +statements are rejected rather than approximated. Split those statements with +a reviewed `replace` first. Dynamic imports through `__import__` or +`importlib.import_module` are outside this static operation's scope. + +#### `ensure_import` + +```kdl +ensure_import in="" from="" name="" [changes=N] +``` + +Ensure every matched Python module has an explicit top-level `from` import. +If a compatible import already contains `name`, the file is left untouched; +otherwise a new import is appended to the module body. Appending is deliberate: +package initializers often define names before importing modules that use them, +so moving the import into a guessed import block could change initialization or +circular-import behavior. + +```sh +kernel-port -e 'ensure_import in="pkg/__init__.py" from="." name="array_api" changes=1' \ + --file $'pkg/__init__.py=VALUE = 1\n' +``` + +``` +[line 1] ensure_import added 1 import(s) in 1 file(s) +M pkg/__init__.py + +FILE: pkg/__init__.py +VALUE = 1 +from . import array_api +``` + +Only explicit imports satisfy the operation: wildcard imports and imports +nested inside a function or conditional do not. Duplicate satisfying imports +are rejected. With `changes=1`, the recipe also fails when upstream starts +providing the import itself, prompting removal of the now-redundant operation. + +#### `ensure_init` + +```kdl +ensure_init under="" [changes=N] +``` + +Add an empty `__init__.py` to every directory under `under` that holds Python files but has no `__init__.py`. Upstream layouts that relied on namespace packages or on setuptools discovery need this to import as a package. + +```sh +kernel-port -e 'ensure_init under="torch-ext/pkg" changes=1' \ + --file 'torch-ext/pkg/__init__.py=x' --file 'torch-ext/pkg/sub/a.py=y' +``` + +``` +[line 1] ensure_init added 1 missing __init__.py file(s) +A torch-ext/pkg/sub/__init__.py +... +``` + +Fails when: nothing exists under `under`. + +### Generating the manifest + +#### `kernel` + +```kdl +kernel name="" backend="" src="[,...]" + [include=",..."] [depends="torch,..."] [capabilities="8.0,9.0"] + [cxx_flags="..."] [cuda_flags="..."] [cuda_minver="12.8"] + [rocm_archs="gfx942,..."] [repeat_src=",..."] +``` + +Record one `[kernel.]` section for the manifest. It writes no files; the `manifest` op emits everything recorded before it. One `kernel` per backend, and the `src` globs are resolved when the op runs, so a source file added upstream inside an already-matched directory is picked up (and one added outside it is not - that is what `expect ... files=N` is for). + +`depends` defaults to `torch`. `repeat_src` lists paths that must be compiled twice, and each must already be selected by `src`. + +#### `manifest` + +```kdl +manifest name="" backends="[,...]" torch_src="[,...]" + [version=N] [edition=N] [license="..."] [upstream="..."] + [repo_id="org/name"] [hub_branch="..."] [python_depends="..."] + [cuda_minver="..."] [cuda_maxver="..."] [cuda_python_depends="..."] + [torch_pyext="py,pyi,..."] [torch_include=",..."] + [stable_abi="cuda=2.11,..."] [stable_abi_version="2.11"] + +manifest name="" backends="..." noarch=#true [noarch_pyext="..."] +``` + +Generate `build.toml` from the `kernel` sections recorded before it, plus the `[general]` and `[torch]` settings given here. The manifest is always generated, never overlaid: if it needs a field this op cannot emit, extend the op. + +`torch_src` selects the binding sources; `torch_include` adds include directories for them (which is how a binding can `#include` a header from the kernel directory instead of restating its declarations). `noarch=#true` switches to a `[torch-noarch]` manifest and rejects the torch-only arguments. + +```sh +kernel-port --file 'hello-kernel/k.cu=// kernel' --file 'torch-ext/binding.cpp=// binding' \ + -e $'kernel name="hello" backend="cuda" src="hello-kernel/**" capabilities="8.0,9.0"\nmanifest name="hello" backends="cuda" torch_src="torch-ext/*.cpp" version=1 license="Apache-2.0"' +``` + +``` +[line 1] kernel declared [kernel.hello] with 1 src file(s) +[line 2] manifest wrote build.toml (1 torch src, 1 kernel section(s)) +A build.toml + +FILE: build.toml +[general] +name = "hello" +version = 1 +license = "Apache-2.0" +backends = ["cuda"] + +[torch] +src = [ + "torch-ext/binding.cpp", +] + +[kernel.hello] +backend = "cuda" +cuda-capabilities = ["8.0", "9.0"] +depends = ["torch"] +src = [ + "hello-kernel/k.cu", +] +... +``` + +Fails when: no `kernel` section was declared (and `noarch` is not set), a glob matches nothing, or an `include` directory holds no files. + +### A whole port in one command + +The ops composed: an upstream layout goes in, a kernel-builder layout comes out, and still nothing touches the disk. + +```sh +kernel-port \ + --file 'hello/__init__.py=from hello.ops import hello' \ + --file $'hello/ops.py=from hello import _C\n\ndef hello(x):\n return _C.hello(x)' \ + --file 'csrc/k.cu=// kernel' \ + --file 'torch-ext/binding.cpp=// op registration' \ + --file 'setup.py=setup()' \ + -e ' +prune keep="csrc/**,hello/**,torch-ext/**" +move from="csrc" to="hello-kernel" +move from="hello" to="torch-ext/hello" +relativize_imports in="torch-ext/hello/**" package_root="torch-ext/hello" changes=2 +replace in="torch-ext/hello/ops.py" find="from . import _C" with="from ._ops import ops as _C" count=1 +kernel name="hello" backend="cuda" src="hello-kernel/**" +manifest name="hello" backends="cuda" torch_src="torch-ext/*.cpp"' +``` + +``` +[line 2] prune removed 1 file(s), kept 4 +[line 3] move moved 1 file(s) to "hello-kernel" +[line 4] move moved 2 file(s) to "torch-ext/hello" +[line 5] relativize_imports rewrote 2 import(s) in 2 file(s) +[line 6] replace 1 replacement(s) in 1 file(s) +[line 7] kernel declared [kernel.hello] with 1 src file(s) +[line 8] manifest wrote build.toml (1 torch src, 1 kernel section(s)) +... + +FILE: torch-ext/hello/__init__.py +from .ops import hello + +FILE: torch-ext/hello/ops.py +from ._ops import ops as _C + +def hello(x): + return _C.hello(x) +``` + +Drop the `relativize_imports` and `replace` lines and every op still succeeds - but the run fails anyway, because the verify stage runs after the last op: + +``` +error: verify: absolute in-package imports remain under torch-ext (they must be relative): + torch-ext/hello/__init__.py: from hello.ops import ... + torch-ext/hello/ops.py: from hello import ... +``` diff --git a/kernel-port/src/lib.rs b/kernel-port/src/lib.rs new file mode 100644 index 00000000..9386ed87 --- /dev/null +++ b/kernel-port/src/lib.rs @@ -0,0 +1,156 @@ +pub mod ops; +pub(crate) mod python; +pub mod recipe; +#[cfg(test)] +mod tests; +pub mod workspace; + +use anyhow::{Context, Result, bail}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use workspace::Workspace; + +pub(crate) fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut out, b| { + let _ = write!(out, "{b:02x}"); + out + }) +} + +pub(crate) fn is_python(path: &str) -> bool { + std::path::Path::new(path) + .extension() + .is_some_and(|e| e == "py") +} + +pub fn run_pipeline( + ws: &mut Workspace, + recipe: &[recipe::Invocation], + recipe_dir: &Path, + root: &Path, + vendors: BTreeMap, +) -> Result { + run_pipeline_with( + ws, + recipe, + recipe_dir, + root, + vendors, + &mut |inv, summary| { + println!("[line {:>3}] {:<19} {}", inv.line, inv.op, summary); + }, + ) +} + +// Same pipeline, with each op's summary handed to a callback instead of +// stdout: a caller that is not a terminal (a wasm host, a test) has somewhere +// to put the log. +pub fn run_pipeline_with( + ws: &mut Workspace, + recipe: &[recipe::Invocation], + recipe_dir: &Path, + root: &Path, + vendors: BTreeMap, + log: &mut dyn FnMut(&recipe::Invocation, &str), +) -> Result { + let inputs = ops::Inputs { + root: root.to_path_buf(), + vendors, + }; + let mut facts = ops::Facts::default(); + // Build every op before applying any, so an argument typo or a bad glob + // anywhere in the recipe fails before the first mutation. + let mut built = Vec::with_capacity(recipe.len()); + for inv in recipe { + let op = ops::build(inv, recipe_dir) + .with_context(|| format!("recipe line {}: {}", inv.line, inv.op))?; + built.push((inv, op)); + } + for (inv, op) in built { + let summary = op + .apply(ws, &inputs, &mut facts) + .with_context(|| format!("recipe line {}: {}", inv.line, inv.op))?; + log(inv, &summary); + } + let changes = ws.changes(); + for path in changes.added.iter().chain(&changes.modified) { + if is_python(path) { + python::check_parses(path, ws.get_text(path)?)?; + } + } + // The Hub imports a kernel under a build-variant directory name, so an + // absolute self-import resolves to nothing at run time. + let mut offenders = Vec::new(); + for path in ws.glob_str("torch-ext/**")? { + if !is_python(&path) { + continue; + } + let rest = &path["torch-ext/".len()..]; + let Some((pkg, _)) = rest.split_once('/') else { + continue; + }; + for found in python::absolute_self_imports(&path, ws.get_text(&path)?, pkg)? { + offenders.push(format!("{path}: {found}")); + } + } + if !offenders.is_empty() { + bail!( + "verify: absolute in-package imports remain under torch-ext (they must \ + be relative):\n {}", + offenders.join("\n ") + ); + } + Ok(facts) +} + +// Every field is derived from the inputs, so re-running the port reproduces +// this file byte-for-byte. +pub fn provenance_json( + recipe_text: &str, + recipe_version: u64, + sources: &[ops::SourceRecord], + tree: &str, +) -> String { + use sha2::{Digest, Sha256}; + + #[derive(serde::Serialize)] + struct Pin<'a> { + repo: &'a str, + commit: &'a str, + } + + #[derive(serde::Serialize)] + struct Provenance<'a> { + format: u64, + recipe: String, + runner: String, + sources: BTreeMap<&'a str, Pin<'a>>, + tree: String, + } + + let recipe = hex(&Sha256::digest(recipe_text.as_bytes())); + let record = Provenance { + format: recipe_version, + recipe: format!("sha256:{recipe}"), + runner: format!("{} {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), + sources: sources + .iter() + .map(|s| { + ( + s.name.as_str(), + Pin { + repo: &s.repo, + commit: &s.commit, + }, + ) + }) + .collect(), + tree: format!("sha256:{tree}"), + }; + let mut json = serde_json::to_string_pretty(&record).expect("provenance serializes"); + json.push('\n'); + json +} diff --git a/kernel-port/src/main.rs b/kernel-port/src/main.rs new file mode 100644 index 00000000..61ce7249 --- /dev/null +++ b/kernel-port/src/main.rs @@ -0,0 +1,274 @@ +use anyhow::{Context, Result, bail}; +use clap::Parser; +use kernel_port::{provenance_json, recipe, run_pipeline, workspace::Workspace}; +use std::path::{Path, PathBuf}; + +#[derive(Parser)] +#[command(name = "kernel-port")] +struct Cli { + #[arg(required_unless_present = "recipe")] + recipe_path: Option, + #[arg( + short = 'e', + long = "recipe", + value_name = "TEXT", + conflicts_with = "recipe_path" + )] + recipe: Option, + #[arg(long, required_unless_present = "files")] + dir: Option, + #[arg(long = "file", value_name = "PATH=CONTENT", conflicts_with = "dir")] + files: Vec, + #[arg(long)] + print: bool, + #[arg(long, conflicts_with = "dry_run")] + out: Option, + #[arg(long)] + dry_run: bool, + #[arg(long)] + diff: bool, + #[arg(long = "vendor", value_name = "NAME=DIR")] + vendor: Vec, + #[arg(long, requires = "out")] + partial: bool, +} + +fn main() { + if let Err(err) = run() { + eprintln!("error: {err:#}"); + std::process::exit(2); + } +} + +fn run() -> Result<()> { + let Cli { + recipe_path, + recipe: recipe_inline, + dir, + files, + print, + out, + dry_run, + diff, + vendor, + partial, + } = Cli::parse(); + + let mut vendors = std::collections::BTreeMap::new(); + for spec in &vendor { + let (name, path) = spec + .split_once('=') + .with_context(|| format!("--vendor {spec:?} must be name=dir"))?; + vendors.insert(name.to_string(), PathBuf::from(path)); + } + + let (text, recipe_dir) = match (&recipe_path, recipe_inline) { + (Some(path), None) => { + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + let dir = path + .canonicalize() + .with_context(|| format!("resolving {}", path.display()))? + .parent() + .map(PathBuf::from) + .context("recipe has no parent directory")?; + (text, dir) + } + // An inline recipe has no file to be relative to, so paths it names + // (`overlay from=`) resolve against the working directory. + (None, Some(text)) => (text, PathBuf::from(".")), + _ => unreachable!("clap requires exactly one of and --recipe"), + }; + let recipe = recipe::parse(&text)?; + check_declares_version(&recipe, recipe_path.as_deref())?; + + if let Some(dir) = &dir + && !dir.is_dir() + { + bail!("--dir {} is not a directory", dir.display()); + } + if let (Some(out), Some(dir)) = (&out, &dir) { + check_out_disjoint(out, dir)?; + } + // Both modes converge on one in-memory workspace: --dir reads it off disk, + // --file builds it from the command line. + let (mut ws, root) = match &dir { + Some(dir) => (Workspace::load(dir)?, dir.clone()), + None => (inline_workspace(&files)?, PathBuf::from(".")), + }; + let facts = match run_pipeline(&mut ws, &recipe.ops, &recipe_dir, &root, vendors) { + Ok(facts) => facts, + Err(err) => { + if partial { + let out = out.as_ref().unwrap(); + ws.materialize_into(out)?; + eprintln!( + "--partial: workspace state before the failing op written to {}", + out.display() + ); + } + return Err(err); + } + }; + + let changes = ws.changes(); + for path in &changes.added { + println!("A {path}"); + } + for path in &changes.modified { + println!("M {path}"); + } + for path in &changes.deleted { + println!("D {path}"); + } + if changes.added.is_empty() && changes.modified.is_empty() && changes.deleted.is_empty() { + println!("no changes"); + } + if diff { + print_diffs(&ws, &changes.modified, &facts.moved); + } + + // An inline workspace has nowhere to be written back to, so printing the + // result is the whole point of the run. + if print || (dir.is_none() && out.is_none()) { + print_tree(&ws); + } + match (&out, &dir, dry_run) { + (Some(out), _, _) => { + ws.materialize_into(out)?; + let provenance = provenance_json( + &text, + recipe.effective_version(), + &facts.sources, + &ws.tree_hash(), + ); + std::fs::write(out.join("port-provenance.json"), provenance)?; + } + (None, Some(dir), false) => ws.materialize(dir)?, + (None, _, _) => {} + } + Ok(()) +} + +// A recipe kept on disk is re-run later, by someone else, against a newer +// build, so it has to say which format it was written against. An inline +// recipe is gone the moment the command finishes. +fn check_declares_version(parsed: &recipe::Recipe, path: Option<&Path>) -> Result<()> { + if let Some(path) = path + && parsed.version.is_none() + { + bail!( + "{}: recipe does not declare its format; add `recipe version={}` as \ + the first line", + path.display(), + recipe::VERSION + ); + } + Ok(()) +} + +// --out is wiped and regenerated on every run, so it must not overlap the +// source. The directory need not exist yet; its parent must. +fn check_out_disjoint(out: &Path, dir: &Path) -> Result<()> { + let dir_canon = dir.canonicalize()?; + let out_canon = match out.canonicalize() { + Ok(p) => p, + Err(_) => out + .parent() + .filter(|p| p.as_os_str().is_empty() || p.exists()) + .map(|p| { + let base = if p.as_os_str().is_empty() { + PathBuf::from(".") + } else { + p.to_path_buf() + }; + Ok::<_, anyhow::Error>(base.canonicalize()?.join(out.file_name().unwrap())) + }) + .transpose()? + .with_context(|| format!("--out {}: parent directory does not exist", out.display()))?, + }; + if out_canon.starts_with(&dir_canon) || dir_canon.starts_with(&out_canon) { + bail!( + "--out {} overlaps --dir {}; the output directory is wiped on every \ + run and must be disjoint from the source", + out.display(), + dir.display() + ); + } + Ok(()) +} + +fn inline_workspace(files: &[String]) -> Result { + let mut map = std::collections::BTreeMap::new(); + for spec in files { + let (path, content) = spec + .split_once('=') + .with_context(|| format!("--file {spec:?} must be path=content"))?; + if map + .insert(path.to_string(), content.as_bytes().to_vec()) + .is_some() + { + bail!("--file {path:?} given more than once"); + } + } + Ok(Workspace::from_files(map)) +} + +fn print_tree(ws: &Workspace) { + for path in ws.paths() { + println!("\nFILE: {path}"); + match std::str::from_utf8(ws.current_bytes(&path).unwrap_or_default()) { + Ok(text) => print!("{text}{}", if text.ends_with('\n') { "" } else { "\n" }), + Err(_) => println!(""), + } + } +} + +fn print_diffs(ws: &Workspace, modified: &[String], moved: &[(String, String)]) { + let mut pairs: Vec<(&str, &str)> = modified.iter().map(|p| (p.as_str(), p.as_str())).collect(); + // A moved file is diffed old path against new, rather than showing up as a + // delete plus an add. + for (old, new) in moved { + let changed = match (ws.initial_bytes(old), ws.current_bytes(new)) { + (Some(a), Some(b)) => a != b, + _ => false, + }; + if changed { + pairs.push((old.as_str(), new.as_str())); + } + } + pairs.sort_by_key(|&(_, new)| new.to_string()); + for (old_path, new_path) in pairs { + let old = ws.initial_bytes(old_path).unwrap_or_default(); + let new = ws.current_bytes(new_path).unwrap_or_default(); + match (std::str::from_utf8(old), std::str::from_utf8(new)) { + (Ok(old), Ok(new)) => { + let diff = similar::TextDiff::from_lines(old, new); + print!( + "--- a/{old_path}\n+++ b/{new_path}\n{}", + diff.unified_diff().context_radius(2) + ); + } + _ => println!("--- {new_path}: binary file changed"), + } + } +} + +#[cfg(test)] +mod tests { + use super::inline_workspace; + + #[test] + fn inline_files_split_on_the_first_equals() { + let ws = inline_workspace(&["a.py=x = 1 == 1\n".to_string()]).unwrap(); + assert_eq!(ws.get_text("a.py").unwrap(), "x = 1 == 1\n"); + } + + #[test] + fn inline_files_reject_bad_specs() { + let err = |specs: &[String]| inline_workspace(specs).err().unwrap().to_string(); + assert!(err(&["a.py".to_string()]).contains("must be path=content")); + let dup = ["a.py=1".to_string(), "a.py=2".to_string()]; + assert!(err(&dup).contains("more than once")); + } +} diff --git a/kernel-port/src/ops/convert_import.rs b/kernel-port/src/ops/convert_import.rs new file mode 100644 index 00000000..e4f51fe3 --- /dev/null +++ b/kernel-port/src/ops/convert_import.rs @@ -0,0 +1,28 @@ +use super::apply_rewrite; +use crate::python::{self, DottedPath}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::Result; + +#[derive(Debug)] +pub struct ConvertImport { + pattern: Pattern, + prefix: DottedPath, + changes: Option, +} + +impl ConvertImport { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + pattern: args.take("in")?.parse()?, + prefix: args.take("prefix")?.parse()?, + changes: args.take_usize_opt("changes")?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + apply_rewrite(ws, &self.pattern, self.changes, "converted", |path, src| { + python::convert_imports_source(path, src, &self.prefix) + }) + } +} diff --git a/kernel-port/src/ops/delete.rs b/kernel-port/src/ops/delete.rs new file mode 100644 index 00000000..4a40cfc0 --- /dev/null +++ b/kernel-port/src/ops/delete.rs @@ -0,0 +1,28 @@ +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct Delete { + pattern: Pattern, +} + +impl Delete { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + pattern: args.take("in")?.parse()?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + let matches = ws.glob(&self.pattern); + if matches.is_empty() { + bail!("{:?} matches nothing", self.pattern); + } + let n = matches.len(); + for path in matches { + ws.delete(&path)?; + } + Ok(format!("removed {n} file(s)")) + } +} diff --git a/kernel-port/src/ops/ensure_import.rs b/kernel-port/src/ops/ensure_import.rs new file mode 100644 index 00000000..b6bbb051 --- /dev/null +++ b/kernel-port/src/ops/ensure_import.rs @@ -0,0 +1,32 @@ +use super::apply_rewrite; +use crate::python; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::Result; + +#[derive(Debug)] +pub struct EnsureImport { + pattern: Pattern, + from: String, + name: String, + changes: Option, +} + +impl EnsureImport { + pub(super) fn build(args: &mut Args) -> Result { + let op = Self { + pattern: args.take("in")?.parse()?, + from: args.take("from")?, + name: args.take("name")?, + changes: args.take_usize_opt("changes")?, + }; + python::validate_ensure_import(&op.from, &op.name)?; + Ok(op) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + apply_rewrite(ws, &self.pattern, self.changes, "added", |path, src| { + python::ensure_import_source(path, src, &self.from, &self.name) + }) + } +} diff --git a/kernel-port/src/ops/ensure_init.rs b/kernel-port/src/ops/ensure_init.rs new file mode 100644 index 00000000..4b71232f --- /dev/null +++ b/kernel-port/src/ops/ensure_init.rs @@ -0,0 +1,51 @@ +use super::check_changes_pin; +use crate::recipe::Args; +use crate::workspace::Workspace; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct EnsureInit { + under: String, + changes: Option, +} + +impl EnsureInit { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + under: args.take("under")?.trim_matches('/').to_string(), + changes: args.take_usize_opt("changes")?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + let files = ws.glob_str(&format!("{}/**", self.under))?; + if files.is_empty() { + bail!("no files under {:?}", self.under); + } + let mut dirs = std::collections::BTreeSet::new(); + dirs.insert(self.under.clone()); + for path in &files { + if !crate::is_python(path) { + continue; + } + let mut dir = path.rsplit_once('/').map(|(d, _)| d.to_string()); + while let Some(d) = dir { + if d.len() <= self.under.len() { + break; + } + dirs.insert(d.clone()); + dir = d.rsplit_once('/').map(|(p, _)| p.to_string()); + } + } + let mut added = 0; + for dir in dirs { + let init = format!("{dir}/__init__.py"); + if ws.current_bytes(&init).is_none() { + ws.insert(&init, Vec::new()); + added += 1; + } + } + check_changes_pin(self.changes, added)?; + Ok(format!("added {added} missing __init__.py file(s)")) + } +} diff --git a/kernel-port/src/ops/expect.rs b/kernel-port/src/ops/expect.rs new file mode 100644 index 00000000..67b315ab --- /dev/null +++ b/kernel-port/src/ops/expect.rs @@ -0,0 +1,72 @@ +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct Expect { + pattern: Pattern, + kind: ExpectKind, +} + +#[derive(Debug)] +enum ExpectKind { + Text { find: String, count: usize }, + FileCount(usize), +} + +impl Expect { + pub(super) fn build(args: &mut Args) -> Result { + let pattern = args.take("in")?.parse()?; + let kind = match (args.take_opt("find"), args.take_usize_opt("files")?) { + (Some(find), None) => { + if find.is_empty() { + bail!("find must not be empty"); + } + ExpectKind::Text { + find, + count: args.take_usize("count")?, + } + } + (None, Some(n)) => ExpectKind::FileCount(n), + (Some(_), Some(_)) => bail!("find and files are mutually exclusive"), + (None, None) => bail!("expect requires either find=.../count=N or files=N"), + }; + Ok(Self { pattern, kind }) + } + + pub(super) fn apply(&self, ws: &Workspace) -> Result { + let files = ws.glob(&self.pattern); + match &self.kind { + ExpectKind::FileCount(expected) => { + if files.len() != *expected { + bail!( + "expected {:?} to match exactly {expected} file(s), found {} - \ + the upstream file set drifted; update the port definition ({})", + self.pattern, + files.len(), + files.join(", ") + ); + } + Ok(format!("{expected} file(s), as expected")) + } + ExpectKind::Text { find, count } => { + if files.is_empty() { + bail!("{:?} matches no files", self.pattern); + } + let mut found = 0; + for path in &files { + found += ws.get_text(path)?.matches(find).count(); + } + if found != *count { + bail!( + "expected exactly {count} occurrence(s) of {find:?} in {:?}, found \ + {found} - upstream changed text this port depends on; update the \ + port definition", + self.pattern + ); + } + Ok(format!("{found} occurrence(s), as expected")) + } + } + } +} diff --git a/kernel-port/src/ops/git.rs b/kernel-port/src/ops/git.rs new file mode 100644 index 00000000..bd854af9 --- /dev/null +++ b/kernel-port/src/ops/git.rs @@ -0,0 +1,65 @@ +use anyhow::{Context, Result, bail}; +use std::path::Path; + +#[derive(Debug)] +pub struct CommitSha(String); + +impl CommitSha { + pub(super) fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::str::FromStr for CommitSha { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if s.len() != 40 || !s.chars().all(|c| c.is_ascii_hexdigit()) { + bail!("commit must be a full 40-character hex SHA, got {s:?}"); + } + Ok(Self(s.to_string())) + } +} + +impl std::fmt::Display for CommitSha { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +pub(super) fn git(root: &Path, args: &[&str]) -> Result { + let out = std::process::Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .output() + .context("running git")?; + if !out.status.success() { + bail!( + "git {} failed in {}: {}", + args.join(" "), + root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +// So a pin written as .../repo.git matches a checkout cloned from .../repo. +pub(super) fn normalize_git_url(url: &str) -> String { + url.trim_end_matches('/') + .trim_end_matches(".git") + .to_string() +} + +pub(super) fn check_clean_checkout(root: &Path, what: &str) -> Result<()> { + let status = git(root, &["status", "--porcelain"])?; + if !status.is_empty() { + bail!( + "{what} checkout {} is not clean (modified or untracked files present); \ + the output would not be reproducible from the pin:\n{status}", + root.display() + ); + } + Ok(()) +} diff --git a/kernel-port/src/ops/kernel.rs b/kernel-port/src/ops/kernel.rs new file mode 100644 index 00000000..679b28b7 --- /dev/null +++ b/kernel-port/src/ops/kernel.rs @@ -0,0 +1,71 @@ +use super::{Facts, KernelSection, check_include_dir, comma_list, glob_list, glob_union}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct Kernel { + name: String, + backend: String, + src: Vec, + include: Vec, + depends: Vec, + capabilities: Vec, + cxx_flags: Vec, + cuda_flags: Vec, + cuda_minver: Option, + rocm_archs: Vec, + repeat_src: Vec, +} + +impl Kernel { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + name: args.take("name")?, + backend: args.take("backend")?, + src: glob_list(&args.take("src")?, "src")?, + include: comma_list(&args.take_opt("include").unwrap_or_default()), + depends: comma_list(&args.take_opt("depends").unwrap_or_else(|| "torch".into())), + capabilities: comma_list(&args.take_opt("capabilities").unwrap_or_default()), + cxx_flags: comma_list(&args.take_opt("cxx_flags").unwrap_or_default()), + cuda_flags: comma_list(&args.take_opt("cuda_flags").unwrap_or_default()), + cuda_minver: args.take_opt("cuda_minver"), + rocm_archs: comma_list(&args.take_opt("rocm_archs").unwrap_or_default()), + repeat_src: comma_list(&args.take_opt("repeat_src").unwrap_or_default()), + }) + } + + pub(super) fn apply(&self, ws: &Workspace, facts: &mut Facts) -> Result { + if facts.kernels.iter().any(|k| k.name == self.name) { + bail!("kernel section {:?} already declared", self.name); + } + let mut src = glob_union(ws, &self.src, "src")?; + for path in &self.repeat_src { + if !src.contains(path) { + bail!("repeat_src path {path:?} is not selected by src globs"); + } + src.push(path.clone()); + } + src.sort(); + for dir in &self.include { + check_include_dir(ws, dir)?; + } + let n = src.len(); + facts.kernels.push(KernelSection { + name: self.name.clone(), + backend: self.backend.clone(), + cxx_flags: self.cxx_flags.clone(), + cuda_flags: self.cuda_flags.clone(), + cuda_minver: self.cuda_minver.clone(), + rocm_archs: self.rocm_archs.clone(), + depends: self.depends.clone(), + include: self.include.clone(), + capabilities: self.capabilities.clone(), + src, + }); + Ok(format!( + "declared [kernel.{}] with {n} src file(s)", + self.name + )) + } +} diff --git a/kernel-port/src/ops/kernelize_imports.rs b/kernel-port/src/ops/kernelize_imports.rs new file mode 100644 index 00000000..f21ce241 --- /dev/null +++ b/kernel-port/src/ops/kernelize_imports.rs @@ -0,0 +1,53 @@ +use super::apply_rewrite; +use crate::python::{self, DottedPath}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct KernelizeImports { + pattern: Pattern, + package: String, + kernel: String, + version: usize, + changes: Option, +} + +impl KernelizeImports { + pub(super) fn build(args: &mut Args) -> Result { + let package = args.take("package")?; + let parsed: DottedPath = package.parse()?; + if parsed.parts().len() != 1 { + bail!("package must be one top-level Python name, got {package:?}"); + } + let kernel = args.take("kernel")?; + if kernel.is_empty() { + bail!("kernel must not be empty"); + } + Ok(Self { + pattern: args.take("in")?.parse()?, + package, + kernel, + version: args.take_usize("version")?, + changes: args.take_usize_opt("changes")?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + apply_rewrite( + ws, + &self.pattern, + self.changes, + "kernelized", + |path, src| { + python::kernelize_imports_source( + path, + src, + &self.package, + &self.kernel, + self.version, + ) + }, + ) + } +} diff --git a/kernel-port/src/ops/manifest.rs b/kernel-port/src/ops/manifest.rs new file mode 100644 index 00000000..a7bb3667 --- /dev/null +++ b/kernel-port/src/ops/manifest.rs @@ -0,0 +1,287 @@ +use super::{Facts, check_include_dir, comma_list, glob_list, glob_union, toml_str_list}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Context, Result, bail}; + +// build.toml is generated, never overlaid. A field this op cannot emit is a +// reason to extend the op. +#[derive(Debug)] +pub struct Manifest { + name: String, + version: Option, + license: Option, + edition: Option, + upstream: Option, + backends: Vec, + repo_id: Option, + hub_branch: Option, + python_depends: Vec, + cuda_minver: Option, + cuda_maxver: Option, + cuda_python_depends: Vec, + kind: ManifestKind, +} + +#[derive(Debug)] +enum ManifestKind { + Noarch { + pyext: Vec, + }, + Torch { + torch_src: Vec, + pyext: Vec, + include: Vec, + stable_abi_version: Option, + stable_abi: Vec<(String, String)>, + }, +} + +impl Manifest { + pub(super) fn build(args: &mut Args) -> Result { + let backends = comma_list(&args.take("backends")?); + if backends.is_empty() { + bail!("backends must list at least one backend"); + } + let version = args.take_opt("version"); + if let Some(v) = &version { + v.parse::() + .with_context(|| format!("version must be an integer, got {v:?}"))?; + } + let edition = args.take_opt("edition"); + if let Some(e) = &edition { + e.parse::() + .with_context(|| format!("edition must be an integer, got {e:?}"))?; + } + let noarch = match args.take_opt("noarch").as_deref() { + None => false, + Some("true") => true, + Some(other) => bail!("noarch must be true if given, got {other:?}"), + }; + let kind = if noarch { + if args.take_opt("torch_src").is_some() { + bail!("torch_src does not apply when noarch=true"); + } + if args.take_opt("stable_abi").is_some() { + bail!("stable_abi does not apply when noarch=true"); + } + if args.take_opt("stable_abi_version").is_some() { + bail!("stable_abi_version does not apply when noarch=true"); + } + if args.take_opt("torch_include").is_some() { + bail!("torch_include does not apply when noarch=true"); + } + ManifestKind::Noarch { + pyext: comma_list(&args.take_opt("noarch_pyext").unwrap_or_default()), + } + } else { + if args.take_opt("noarch_pyext").is_some() { + bail!("noarch_pyext requires noarch=true"); + } + let stable_abi_version = args.take_opt("stable_abi_version"); + let pyext = comma_list(&args.take_opt("torch_pyext").unwrap_or_default()); + let stable_abi = comma_list(&args.take_opt("stable_abi").unwrap_or_default()) + .iter() + .map(|pair| { + pair.split_once('=') + .map(|(b, v)| (b.to_string(), v.to_string())) + .with_context(|| { + format!("stable_abi entry {pair:?} must be backend=version") + }) + }) + .collect::>>()?; + if stable_abi_version.is_some() && !stable_abi.is_empty() { + bail!("stable_abi_version and stable_abi are mutually exclusive"); + } + ManifestKind::Torch { + torch_src: glob_list(&args.take("torch_src")?, "torch_src")?, + pyext, + include: comma_list(&args.take_opt("torch_include").unwrap_or_default()), + stable_abi_version, + stable_abi, + } + }; + Ok(Self { + name: args.take("name")?, + version, + license: args.take_opt("license"), + edition, + upstream: args.take_opt("upstream"), + backends, + repo_id: args.take_opt("repo_id"), + hub_branch: args.take_opt("hub_branch"), + python_depends: comma_list(&args.take_opt("python_depends").unwrap_or_default()), + cuda_minver: args.take_opt("cuda_minver"), + cuda_maxver: args.take_opt("cuda_maxver"), + cuda_python_depends: comma_list( + &args.take_opt("cuda_python_depends").unwrap_or_default(), + ), + kind, + }) + } + + fn toml_list(key: &str, items: &[String]) -> String { + if items.len() == 1 { + format!("{key} = [{}]\n", toml_str_list(items)) + } else { + let mut block = format!("{key} = [\n"); + for item in items { + block.push_str(&format!(" {item:?},\n")); + } + block.push_str("]\n"); + block + } + } + + fn src_block(files: &[String]) -> String { + let mut block = String::from("src = [\n"); + for f in files { + block.push_str(&format!(" {f:?},\n")); + } + block.push(']'); + block + } + + fn general_section(&self) -> String { + let mut toml = String::from("[general]\n"); + toml.push_str(&format!("name = {:?}\n", self.name)); + if let Some(version) = &self.version { + toml.push_str(&format!("version = {version}\n")); + } + if let Some(license) = &self.license { + toml.push_str(&format!("license = {license:?}\n")); + } + if let Some(edition) = &self.edition { + toml.push_str(&format!("edition = {edition}\n")); + } + if let Some(upstream) = &self.upstream { + toml.push_str(&format!("upstream = {upstream:?}\n")); + } + toml.push_str(&Self::toml_list("backends", &self.backends)); + if !self.python_depends.is_empty() { + toml.push_str(&Self::toml_list("python-depends", &self.python_depends)); + } + if self.cuda_minver.is_some() + || self.cuda_maxver.is_some() + || !self.cuda_python_depends.is_empty() + { + toml.push_str("\n[general.cuda]\n"); + if let Some(minver) = &self.cuda_minver { + toml.push_str(&format!("minver = {minver:?}\n")); + } + if let Some(maxver) = &self.cuda_maxver { + toml.push_str(&format!("maxver = {maxver:?}\n")); + } + if !self.cuda_python_depends.is_empty() { + toml.push_str(&Self::toml_list( + "python-depends", + &self.cuda_python_depends, + )); + } + } + if self.repo_id.is_some() || self.hub_branch.is_some() { + toml.push_str("\n[general.hub]\n"); + if let Some(repo_id) = &self.repo_id { + toml.push_str(&format!("repo-id = {repo_id:?}\n")); + } + if let Some(branch) = &self.hub_branch { + toml.push_str(&format!("branch = {branch:?}\n")); + } + } + toml + } + + pub(super) fn apply(&self, ws: &mut Workspace, facts: &Facts) -> Result { + let mut toml = self.general_section(); + + let summary = match &self.kind { + ManifestKind::Noarch { pyext } => { + if !facts.kernels.is_empty() { + bail!("noarch=true but kernel sections were declared"); + } + toml.push_str("\n[torch-noarch]\n"); + if !pyext.is_empty() { + toml.push_str(&format!("pyext = [{}]\n", toml_str_list(pyext))); + } + toml.push_str("\n[kernel]\n"); + "wrote build.toml (noarch)".to_string() + } + ManifestKind::Torch { + torch_src, + pyext, + include, + stable_abi_version, + stable_abi, + } => { + let torch_files = glob_union(ws, torch_src, "torch_src")?; + if facts.kernels.is_empty() { + bail!("no kernel sections declared; add `kernel` statements before manifest"); + } + toml.push_str("\n[torch]\n"); + if let Some(version) = stable_abi_version { + toml.push_str(&format!("stable-abi = {version:?}\n")); + } + if !pyext.is_empty() { + toml.push_str(&Self::toml_list("pyext", pyext)); + } + for dir in include { + check_include_dir(ws, dir)?; + } + if !include.is_empty() { + toml.push_str(&format!("include = [{}]\n", toml_str_list(include))); + } + toml.push_str(&Self::src_block(&torch_files)); + toml.push('\n'); + if !stable_abi.is_empty() { + toml.push_str("\n[torch.stable-abi]\n"); + for (backend, version) in stable_abi { + toml.push_str(&format!("{backend} = {version:?}\n")); + } + } + for k in &facts.kernels { + toml.push_str(&format!("\n[kernel.{}]\n", k.name)); + toml.push_str(&format!("backend = {:?}\n", k.backend)); + if !k.cxx_flags.is_empty() { + toml.push_str("cxx-flags = [\n"); + for f in &k.cxx_flags { + toml.push_str(&format!(" {f:?},\n")); + } + toml.push_str("]\n"); + } + if !k.capabilities.is_empty() { + toml.push_str(&format!( + "cuda-capabilities = [{}]\n", + toml_str_list(&k.capabilities) + )); + } + if !k.cuda_flags.is_empty() { + toml.push_str("cuda-flags = [\n"); + for f in &k.cuda_flags { + toml.push_str(&format!(" {f:?},\n")); + } + toml.push_str("]\n"); + } + if let Some(minver) = &k.cuda_minver { + toml.push_str(&format!("cuda-minver = {minver:?}\n")); + } + toml.push_str(&format!("depends = [{}]\n", toml_str_list(&k.depends))); + if !k.rocm_archs.is_empty() { + toml.push_str(&Self::toml_list("rocm-archs", &k.rocm_archs)); + } + if !k.include.is_empty() { + toml.push_str(&format!("include = [{}]\n", toml_str_list(&k.include))); + } + toml.push_str(&Self::src_block(&k.src)); + toml.push('\n'); + } + format!( + "wrote build.toml ({} torch src, {} kernel section(s))", + torch_files.len(), + facts.kernels.len() + ) + } + }; + + ws.set_text("build.toml", toml); + Ok(summary) + } +} diff --git a/kernel-port/src/ops/mod.rs b/kernel-port/src/ops/mod.rs new file mode 100644 index 00000000..8ca52241 --- /dev/null +++ b/kernel-port/src/ops/mod.rs @@ -0,0 +1,338 @@ +mod convert_import; +mod delete; +mod ensure_import; +mod ensure_init; +mod expect; +mod git; +mod kernel; +mod kernelize_imports; +mod manifest; +mod r#move; +mod overlay; +mod prune; +mod relativize_imports; +mod remap_module; +mod replace; +mod source; +mod strip_suffix; +mod vendor; + +pub use convert_import::ConvertImport; +pub use delete::Delete; +pub use ensure_import::EnsureImport; +pub use ensure_init::EnsureInit; +pub use expect::Expect; +pub use kernel::Kernel; +pub use kernelize_imports::KernelizeImports; +pub use manifest::Manifest; +pub use r#move::Move; +pub use overlay::Overlay; +pub use prune::Prune; +pub use relativize_imports::RelativizeImports; +pub use remap_module::RemapModule; +pub use replace::Replace; +pub use source::Source; +pub use strip_suffix::StripSuffix; +pub use vendor::Vendor; + +use crate::recipe::Invocation; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Context, Result, bail}; +use std::path::{Path, PathBuf}; + +#[derive(Default)] +pub struct Inputs { + pub root: PathBuf, + pub vendors: std::collections::BTreeMap, +} + +#[derive(Default)] +pub struct Facts { + pub moved: Vec<(String, String)>, + pub kernels: Vec, + pub sources: Vec, +} + +pub struct SourceRecord { + pub name: String, + pub repo: String, + pub commit: String, +} + +impl Facts { + fn record_source(&mut self, name: &str, repo: &str, commit: &str) { + if !self.sources.iter().any(|s| s.name == name) { + self.sources.push(SourceRecord { + name: name.to_string(), + repo: repo.to_string(), + commit: commit.to_string(), + }); + } + } +} + +pub struct KernelSection { + pub name: String, + pub backend: String, + pub cxx_flags: Vec, + pub cuda_flags: Vec, + pub cuda_minver: Option, + pub rocm_archs: Vec, + pub depends: Vec, + pub include: Vec, + pub capabilities: Vec, + pub src: Vec, +} + +fn check_changes_pin(expected: Option, actual: usize) -> Result<()> { + if let Some(expected) = expected + && actual != expected + { + bail!( + "expected exactly {expected} change(s) but made {actual} - upstream \ + drifted; review the new rewrites and update changes=" + ); + } + Ok(()) +} + +// Comma-separated, where `,,` is an escaped comma: compiler flags are values +// that contain commas. +fn comma_list(value: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut chars = value.chars().peekable(); + while let Some(c) = chars.next() { + if c == ',' { + if chars.peek() == Some(&',') { + chars.next(); + cur.push(','); + } else { + let item = cur.trim(); + if !item.is_empty() { + out.push(item.to_string()); + } + cur.clear(); + } + } else { + cur.push(c); + } + } + let item = cur.trim(); + if !item.is_empty() { + out.push(item.to_string()); + } + out +} + +fn toml_str_list(items: &[String]) -> String { + items + .iter() + .map(|s| format!("{s:?}")) + .collect::>() + .join(", ") +} + +fn apply_rewrite( + ws: &mut Workspace, + pattern: &Pattern, + changes: Option, + verb: &str, + mut rewrite: impl FnMut(&str, &str) -> Result>, +) -> Result { + let files = ws.glob(pattern); + if files.is_empty() { + bail!("{pattern:?} matches no files"); + } + let mut n_files = 0; + let mut n_imports = 0; + for path in files { + if !crate::is_python(&path) { + continue; + } + if let Some((updated, count)) = rewrite(&path, ws.get_text(&path)?)? { + ws.set_text(&path, updated); + n_files += 1; + n_imports += count; + } + } + // The pin counts rewritten imports, not files, so a new upstream file + // cannot be rewritten without moving it. + check_changes_pin(changes, n_imports)?; + Ok(format!("{verb} {n_imports} import(s) in {n_files} file(s)")) +} + +fn copy_tree( + ws: &mut Workspace, + source: &Path, + dest: impl Fn(&str) -> String, + refuse_existing: bool, +) -> Result { + let mut stack = vec![source.to_path_buf()]; + let mut copied = 0usize; + while let Some(dir) = stack.pop() { + let mut entries: Vec = std::fs::read_dir(&dir)? + .map(|e| Ok(e?.path())) + .collect::>()?; + entries.sort(); + for path in entries { + if path.file_name().is_some_and(|n| n == ".git") { + continue; + } + if path.is_dir() { + stack.push(path); + } else { + let rel = path + .strip_prefix(source) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + let dest = dest(&rel); + if refuse_existing && ws.current_bytes(&dest).is_some() { + bail!("vendor destination {dest:?} already exists in the workspace"); + } + let content = + std::fs::read(&path).with_context(|| format!("reading {}", path.display()))?; + ws.insert(&dest, content); + copied += 1; + } + } + } + Ok(copied) +} + +// Commas inside a `{a,b}` alternation belong to the glob, not to the list. +fn glob_comma_list(value: &str) -> Vec { + let mut out = Vec::new(); + let mut depth = 0usize; + let mut cur = String::new(); + for c in value.chars() { + match c { + '{' => { + depth += 1; + cur.push(c); + } + '}' => { + depth = depth.saturating_sub(1); + cur.push(c); + } + ',' if depth == 0 => { + out.push(std::mem::take(&mut cur)); + } + _ => cur.push(c), + } + } + out.push(cur); + out.iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + +fn glob_list(value: &str, what: &str) -> Result> { + let globs = glob_comma_list(value) + .iter() + .map(|s| s.parse()) + .collect::>>()?; + if globs.is_empty() { + bail!("{what} must list at least one glob"); + } + Ok(globs) +} + +fn glob_union(ws: &Workspace, globs: &[Pattern], what: &str) -> Result> { + let mut union = std::collections::BTreeSet::new(); + for glob in globs { + let matches = ws.glob(glob); + if matches.is_empty() { + bail!("{what} glob {glob:?} matches no files"); + } + union.extend(matches); + } + Ok(union.into_iter().collect()) +} + +// An include directory is named, not globbed, so a typo would otherwise +// surface at build time rather than here. +fn check_include_dir(ws: &Workspace, dir: &str) -> Result<()> { + let glob = if dir.trim_matches('/') == "." { + "**".to_string() + } else { + format!("{}/**", dir.trim_matches('/')) + }; + if ws.glob_str(&glob)?.is_empty() { + bail!("include directory {dir:?} contains no files"); + } + Ok(()) +} + +#[derive(Debug)] +pub enum Op { + Source(Source), + Vendor(Vendor), + Delete(Delete), + Move(Move), + Replace(Replace), + StripSuffix(StripSuffix), + Expect(Expect), + Overlay(Overlay), + Prune(Prune), + Kernel(Kernel), + KernelizeImports(KernelizeImports), + Manifest(Manifest), + RelativizeImports(RelativizeImports), + RemapModule(RemapModule), + ConvertImport(ConvertImport), + EnsureImport(EnsureImport), + EnsureInit(EnsureInit), +} + +pub fn build(inv: &Invocation, recipe_dir: &Path) -> Result { + let mut args = inv.take_args(); + let op = match inv.op.as_str() { + "source" => Op::Source(Source::build(&mut args)?), + "vendor" => Op::Vendor(Vendor::build(&mut args)?), + "delete" => Op::Delete(Delete::build(&mut args)?), + "move" => Op::Move(Move::build(&mut args)?), + "replace" => Op::Replace(Replace::build(&mut args)?), + "strip_suffix" => Op::StripSuffix(StripSuffix::build(&mut args)?), + "expect" => Op::Expect(Expect::build(&mut args)?), + "overlay" => Op::Overlay(Overlay::build(&mut args, recipe_dir)?), + "prune" => Op::Prune(Prune::build(&mut args)?), + "kernel" => Op::Kernel(Kernel::build(&mut args)?), + "kernelize_imports" => Op::KernelizeImports(KernelizeImports::build(&mut args)?), + "manifest" => Op::Manifest(Manifest::build(&mut args)?), + "relativize_imports" => Op::RelativizeImports(RelativizeImports::build(&mut args)?), + "remap_module" => Op::RemapModule(RemapModule::build(&mut args)?), + "convert_import" => Op::ConvertImport(ConvertImport::build(&mut args)?), + "ensure_import" => Op::EnsureImport(EnsureImport::build(&mut args)?), + "ensure_init" => Op::EnsureInit(EnsureInit::build(&mut args)?), + other => bail!("unknown op {other:?}"), + }; + args.finish()?; + Ok(op) +} + +impl Op { + pub fn apply(&self, ws: &mut Workspace, inputs: &Inputs, facts: &mut Facts) -> Result { + match self { + Self::Source(op) => op.apply(inputs, facts), + Self::Vendor(op) => op.apply(ws, inputs, facts), + Self::Delete(op) => op.apply(ws), + Self::Move(op) => op.apply(ws, facts), + Self::Replace(op) => op.apply(ws), + Self::StripSuffix(op) => op.apply(ws), + Self::Expect(op) => op.apply(ws), + Self::Overlay(op) => op.apply(ws), + Self::Prune(op) => op.apply(ws), + Self::Kernel(op) => op.apply(ws, facts), + Self::KernelizeImports(op) => op.apply(ws), + Self::Manifest(op) => op.apply(ws, facts), + Self::RelativizeImports(op) => op.apply(ws), + Self::RemapModule(op) => op.apply(ws), + Self::ConvertImport(op) => op.apply(ws), + Self::EnsureImport(op) => op.apply(ws), + Self::EnsureInit(op) => op.apply(ws), + } + } +} diff --git a/kernel-port/src/ops/move.rs b/kernel-port/src/ops/move.rs new file mode 100644 index 00000000..cfd65718 --- /dev/null +++ b/kernel-port/src/ops/move.rs @@ -0,0 +1,26 @@ +use super::Facts; +use crate::recipe::Args; +use crate::workspace::Workspace; +use anyhow::Result; + +#[derive(Debug)] +pub struct Move { + from: String, + to: String, +} + +impl Move { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + from: args.take("from")?, + to: args.take("to")?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace, facts: &mut Facts) -> Result { + let pairs = ws.rename(&self.from, &self.to)?; + let n = pairs.len(); + facts.moved.extend(pairs); + Ok(format!("moved {n} file(s) to {:?}", self.to)) + } +} diff --git a/kernel-port/src/ops/overlay.rs b/kernel-port/src/ops/overlay.rs new file mode 100644 index 00000000..e9780213 --- /dev/null +++ b/kernel-port/src/ops/overlay.rs @@ -0,0 +1,29 @@ +use super::copy_tree; +use crate::recipe::Args; +use crate::workspace::Workspace; +use anyhow::{Result, bail}; +use std::path::{Path, PathBuf}; + +#[derive(Debug)] +pub struct Overlay { + source: PathBuf, +} + +impl Overlay { + pub(super) fn build(args: &mut Args, recipe_dir: &Path) -> Result { + Ok(Self { + source: recipe_dir.join(args.take("from")?), + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + if !self.source.is_dir() { + bail!("{} is not a directory", self.source.display()); + } + let copied = copy_tree(ws, &self.source, std::string::ToString::to_string, false)?; + if copied == 0 { + bail!("{} contains no files", self.source.display()); + } + Ok(format!("copied {copied} file(s)")) + } +} diff --git a/kernel-port/src/ops/prune.rs b/kernel-port/src/ops/prune.rs new file mode 100644 index 00000000..5186df0f --- /dev/null +++ b/kernel-port/src/ops/prune.rs @@ -0,0 +1,43 @@ +use super::glob_comma_list; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct Prune { + keep: Vec, +} + +impl Prune { + pub(super) fn build(args: &mut Args) -> Result { + let keep = glob_comma_list(&args.take("keep")?) + .iter() + .map(|s| s.parse()) + .collect::>>()?; + if keep.is_empty() { + bail!("keep must list at least one glob"); + } + Ok(Self { keep }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + let mut kept = std::collections::BTreeSet::new(); + for glob in &self.keep { + let matches = ws.glob(glob); + if matches.is_empty() { + bail!("keep glob {glob:?} matches nothing"); + } + kept.extend(matches); + } + let doomed: Vec = ws + .glob_str("**")? + .into_iter() + .filter(|p| !kept.contains(p)) + .collect(); + let n = doomed.len(); + for path in doomed { + ws.delete(&path)?; + } + Ok(format!("removed {n} file(s), kept {}", kept.len())) + } +} diff --git a/kernel-port/src/ops/relativize_imports.rs b/kernel-port/src/ops/relativize_imports.rs new file mode 100644 index 00000000..dcee8621 --- /dev/null +++ b/kernel-port/src/ops/relativize_imports.rs @@ -0,0 +1,55 @@ +use super::apply_rewrite; +use crate::python::{self}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct RelativizeImports { + pattern: Pattern, + package_root: String, + changes: Option, + root_relative: bool, +} + +impl RelativizeImports { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + pattern: args.take("in")?.parse()?, + package_root: args.take("package_root")?.trim_end_matches('/').to_string(), + changes: args.take_usize_opt("changes")?, + root_relative: match args.take_opt("root_relative").as_deref() { + None | Some("false") => false, + Some("true") => true, + Some(other) => bail!("root_relative must be true or false, got {other:?}"), + }, + }) + } + + fn package_of(&self, path: &str) -> Result> { + let parent = self + .package_root + .rfind('/') + .map_or("", |i| &self.package_root[..=i]); + let Some(rel) = path.strip_prefix(parent) else { + bail!("{path:?} is not under package_root {:?}", self.package_root); + }; + let mut components: Vec = rel.split('/').map(str::to_string).collect(); + components.pop(); + if components.is_empty() { + bail!("{path:?} sits at the package root's parent; it has no package"); + } + Ok(components) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + apply_rewrite(ws, &self.pattern, self.changes, "rewrote", |path, src| { + let package = self.package_of(path)?; + if self.root_relative { + python::relativize_source_from_root(path, src, &package) + } else { + python::relativize_source(path, src, &package) + } + }) + } +} diff --git a/kernel-port/src/ops/remap_module.rs b/kernel-port/src/ops/remap_module.rs new file mode 100644 index 00000000..890edb42 --- /dev/null +++ b/kernel-port/src/ops/remap_module.rs @@ -0,0 +1,30 @@ +use super::apply_rewrite; +use crate::python::{self, DottedPath}; +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::Result; + +#[derive(Debug)] +pub struct RemapModule { + pattern: Pattern, + from: DottedPath, + to: DottedPath, + changes: Option, +} + +impl RemapModule { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + pattern: args.take("in")?.parse()?, + from: args.take("from")?.parse()?, + to: args.take("to")?.parse()?, + changes: args.take_usize_opt("changes")?, + }) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + apply_rewrite(ws, &self.pattern, self.changes, "rewrote", |path, src| { + python::remap_source(path, src, &self.from, &self.to) + }) + } +} diff --git a/kernel-port/src/ops/replace.rs b/kernel-port/src/ops/replace.rs new file mode 100644 index 00000000..84c006ac --- /dev/null +++ b/kernel-port/src/ops/replace.rs @@ -0,0 +1,85 @@ +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct Replace { + pattern: Pattern, + find: String, + with: String, + count: Option, +} + +impl Replace { + pub(super) fn build(args: &mut Args) -> Result { + let op = Self { + pattern: args.take("in")?.parse()?, + find: args.take("find")?, + with: args.take("with")?, + count: args.take_usize_opt("count")?, + }; + if op.find.is_empty() { + bail!("find must not be empty"); + } + Ok(op) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + let files = ws.glob(&self.pattern); + if files.is_empty() { + bail!("{:?} matches no files", self.pattern); + } + let mut found = 0; + let mut per_file = Vec::new(); + for path in &files { + let n = ws.get_text(path)?.matches(&self.find).count(); + per_file.push((path.clone(), n)); + found += n; + } + if let Some(count) = self.count + && found != count + { + bail!( + "expected exactly {} match(es) of {:?} in {:?}, found {} ({})", + count, + self.find, + self.pattern, + found, + if found == 0 { + "no files matched the text".to_string() + } else { + per_file + .iter() + .filter(|(_, n)| *n > 0) + .map(|(p, n)| format!("{p}: {n}")) + .collect::>() + .join(", ") + } + ); + } + if self.count.is_none() { + let mismatches = per_file + .iter() + .filter(|(_, n)| *n != 1) + .map(|(path, n)| format!("{path}: {n}")) + .collect::>(); + if !mismatches.is_empty() { + bail!( + "expected exactly one match of {:?} in every file matched by {:?} ({})", + self.find, + self.pattern, + mismatches.join(", ") + ); + } + } + let n_files = per_file.iter().filter(|(_, n)| *n > 0).count(); + for (path, n) in per_file { + if n == 0 { + continue; + } + let updated = ws.get_text(&path)?.replace(&self.find, &self.with); + ws.set_text(&path, updated); + } + Ok(format!("{found} replacement(s) in {n_files} file(s)")) + } +} diff --git a/kernel-port/src/ops/source.rs b/kernel-port/src/ops/source.rs new file mode 100644 index 00000000..73e53c24 --- /dev/null +++ b/kernel-port/src/ops/source.rs @@ -0,0 +1,47 @@ +use super::git::{CommitSha, check_clean_checkout, git, normalize_git_url}; +use super::{Facts, Inputs}; +use crate::recipe::Args; +use anyhow::{Context, Result, bail}; + +#[derive(Debug)] +pub struct Source { + repo: String, + commit: CommitSha, +} + +impl Source { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + repo: args.take("repo")?, + commit: args.take("commit")?.parse()?, + }) + } + + pub(super) fn apply(&self, inputs: &Inputs, facts: &mut Facts) -> Result { + let head = git(&inputs.root, &["rev-parse", "HEAD"]).with_context(|| { + format!( + "source pinning requires --dir to be a git checkout of {}", + self.repo + ) + })?; + if head != self.commit.as_str() { + bail!( + "checkout is at {head} but this port is written against {} - \ + re-clone at the pinned commit, or re-verify the port and update the pin", + self.commit + ); + } + // A checkout with no origin still passes; only a mismatch fails. + if let Ok(origin) = git(&inputs.root, &["remote", "get-url", "origin"]) + && normalize_git_url(&origin) != normalize_git_url(&self.repo) + { + bail!( + "checkout origin is {origin} but this port is for {} - wrong repository", + self.repo + ); + } + check_clean_checkout(&inputs.root, "source")?; + facts.record_source("upstream", &self.repo, self.commit.as_str()); + Ok(format!("verified {} @ {}", self.repo, &head[..12])) + } +} diff --git a/kernel-port/src/ops/strip_suffix.rs b/kernel-port/src/ops/strip_suffix.rs new file mode 100644 index 00000000..4d322102 --- /dev/null +++ b/kernel-port/src/ops/strip_suffix.rs @@ -0,0 +1,53 @@ +use crate::recipe::Args; +use crate::workspace::{Pattern, Workspace}; +use anyhow::{Result, bail}; + +#[derive(Debug)] +pub struct StripSuffix { + pattern: Pattern, + suffix: String, + files: usize, +} + +impl StripSuffix { + pub(super) fn build(args: &mut Args) -> Result { + let op = Self { + pattern: args.take("in")?.parse()?, + suffix: args.take("suffix")?, + files: args.take_usize("files")?, + }; + if op.suffix.is_empty() { + bail!("suffix must not be empty"); + } + Ok(op) + } + + pub(super) fn apply(&self, ws: &mut Workspace) -> Result { + let paths = ws.glob(&self.pattern); + if paths.is_empty() { + bail!("{:?} matches no files", self.pattern); + } + if paths.len() != self.files { + bail!( + "expected exactly {} file(s) for {:?}, found {}", + self.files, + self.pattern, + paths.len() + ); + } + // Every file is checked before any is written, so a partial strip + // cannot land. + let mut updates = Vec::with_capacity(paths.len()); + for path in paths { + let text = ws.get_text(&path)?; + let Some(stripped) = text.strip_suffix(&self.suffix) else { + bail!("{path:?} does not end with pinned suffix {:?}", self.suffix); + }; + updates.push((path, stripped.to_string())); + } + for (path, updated) in updates { + ws.set_text(&path, updated); + } + Ok(format!("stripped suffix from {} file(s)", self.files)) + } +} diff --git a/kernel-port/src/ops/vendor.rs b/kernel-port/src/ops/vendor.rs new file mode 100644 index 00000000..9a07b3dd --- /dev/null +++ b/kernel-port/src/ops/vendor.rs @@ -0,0 +1,83 @@ +use super::git::{CommitSha, check_clean_checkout, git, normalize_git_url}; +use super::{Facts, Inputs, copy_tree}; +use crate::recipe::Args; +use crate::workspace::Workspace; +use anyhow::{Context, Result, bail}; + +#[derive(Debug)] +pub struct Vendor { + name: String, + repo: String, + commit: CommitSha, + path: String, + to: String, +} + +impl Vendor { + pub(super) fn build(args: &mut Args) -> Result { + Ok(Self { + name: args.take("name")?, + repo: args.take("repo")?, + commit: args.take("commit")?.parse()?, + path: args.take("path")?.trim_matches('/').to_string(), + to: args.take("to")?.trim_matches('/').to_string(), + }) + } + + pub(super) fn apply( + &self, + ws: &mut Workspace, + inputs: &Inputs, + facts: &mut Facts, + ) -> Result { + let Some(checkout) = inputs.vendors.get(&self.name) else { + bail!( + "no checkout supplied for vendor {:?}; pass --vendor {}= on the CLI", + self.name, + self.name + ); + }; + let head = git(checkout, &["rev-parse", "HEAD"]).with_context(|| { + format!( + "vendor {:?} must be a git checkout of {}", + self.name, self.repo + ) + })?; + if head != self.commit.as_str() { + bail!( + "vendor {:?} checkout is at {head} but this port is written against {}", + self.name, + self.commit + ); + } + if let Ok(origin) = git(checkout, &["remote", "get-url", "origin"]) + && normalize_git_url(&origin) != normalize_git_url(&self.repo) + { + bail!( + "vendor {:?} checkout origin is {origin} but this port expects {}", + self.name, + self.repo + ); + } + check_clean_checkout(checkout, "vendor")?; + facts.record_source(&self.name, &self.repo, self.commit.as_str()); + + let src_root = checkout.join(&self.path); + if !src_root.is_dir() { + bail!( + "{} is not a directory in the vendor checkout", + src_root.display() + ); + } + let copied = copy_tree(ws, &src_root, |rel| format!("{}/{rel}", self.to), true)?; + if copied == 0 { + bail!("{} contains no files", src_root.display()); + } + Ok(format!( + "vendored {copied} file(s) from {}:{} @ {}", + self.name, + self.path, + &head[..12] + )) + } +} diff --git a/kernel-port/src/python.rs b/kernel-port/src/python.rs new file mode 100644 index 00000000..f4b13f9c --- /dev/null +++ b/kernel-port/src/python.rs @@ -0,0 +1,992 @@ +// Python edits go through libcst, so comments, quoting and layout outside the +// rewritten import survive byte-for-byte. +use anyhow::{Context, Result, bail}; +use libcst_native::{ + AssignTargetExpression, Codegen, CodegenState, CompoundStatement, Expression, Import, + ImportAlias, ImportFrom, ImportNames, Module, NameOrAttribute, OrElse, SmallStatement, + Statement, Suite, +}; + +#[derive(Debug)] +pub struct DottedPath(Vec); + +impl DottedPath { + pub fn parts(&self) -> &[String] { + &self.0 + } +} + +impl std::str::FromStr for DottedPath { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let parts: Vec = s.split('.').map(str::to_string).collect(); + if parts.iter().any(String::is_empty) { + bail!("{s:?} must be a non-empty dotted module path"); + } + Ok(Self(parts)) + } +} + +// A parsed module borrows the text it came from, so caching one means keeping +// the two together. `self_cell` owns the source alongside the tree that points +// into it, which is what lets a `Module` outlive the caller's `&str`. +self_cell::self_cell!( + struct OwnedModule { + owner: String, + #[covariant] + dependent: Module, + } +); + +// Parsing is what a port spends its time on, and roughly half of it re-reads +// bytes that were already parsed: an op matches a file an earlier op left +// untouched, and the end-of-pipeline verify re-reads the whole tree once more. +// Both checks below are pure functions of the text, so their answers are +// memoized on a content digest instead of being recomputed. +// +// The digest is SHA-256, not a fast hash: a collision here would silently skip +// a validation, and hashing is a rounding error next to a libcst parse. +mod memo { + use super::OwnedModule; + use std::cell::RefCell; + use std::collections::{HashMap, HashSet}; + use std::rc::Rc; + + pub type Digest = [u8; 32]; + + // The wasm playground re-runs the pipeline on every recipe edit against one + // long-lived module, so the tables are capped rather than left to grow. + const CAP: usize = 8192; + + // Parsed modules are held by content, so an op that matches a file an + // earlier op left alone reuses the tree instead of rebuilding it. Sharing + // is by `Rc` and the map borrow is released before the tree is handed out, + // so a caller that parses another file while holding one cannot deadlock. + thread_local! { + static PARSES: RefCell> = RefCell::new(HashSet::new()); + static SELF_IMPORTS: RefCell>> = + RefCell::new(HashMap::new()); + static MODULES: RefCell<(HashMap>, usize)> = + RefCell::new((HashMap::new(), 0)); + } + + // Syntax trees are far larger than the text they came from, so the module + // cache is bounded by the source bytes it is holding rather than by entry + // count. Ports run against trees far below this; the bound is there for the + // playground, which keeps one module alive across many edits. + const MODULE_BUDGET: usize = 8 << 20; + + pub fn module(key: &Digest) -> Option> { + MODULES.with_borrow(|(map, _)| map.get(key).cloned()) + } + + pub fn note_module(key: Digest, owned: OwnedModule) -> Rc { + let owned = Rc::new(owned); + MODULES.with_borrow_mut(|(map, bytes)| { + if *bytes >= MODULE_BUDGET { + map.clear(); + *bytes = 0; + } + *bytes += owned.borrow_owner().len(); + map.insert(key, Rc::clone(&owned)); + }); + owned + } + + pub fn digest(src: &str) -> Digest { + use sha2::{Digest as _, Sha256}; + Sha256::digest(src.as_bytes()).into() + } + + pub fn parses(key: Digest) -> bool { + PARSES.with_borrow(|set| set.contains(&key)) + } + + pub fn note_parses(key: Digest) { + PARSES.with_borrow_mut(|set| { + if set.len() >= CAP { + set.clear(); + } + set.insert(key); + }); + } + + pub fn self_imports(key: &(Digest, String)) -> Option> { + SELF_IMPORTS.with_borrow(|map| map.get(key).cloned()) + } + + pub fn note_self_imports(key: (Digest, String), found: Vec) { + SELF_IMPORTS.with_borrow_mut(|map| { + if map.len() >= CAP { + map.clear(); + } + map.insert(key, found); + }); + } +} + +fn render<'a>(node: &impl Codegen<'a>) -> String { + render_indented(node, &[]) +} + +fn render_indented<'a>(node: &impl Codegen<'a>, indents: &[&'a str]) -> String { + let mut state = CodegenState { + default_newline: "\n", + default_indent: " ", + indent_tokens: indents.to_vec(), + ..Default::default() + }; + node.codegen(&mut state); + state.tokens +} + +fn check_roundtrip(module: &Module, src: &str) -> Result<()> { + let mut state = CodegenState { + default_newline: module.default_newline, + default_indent: module.default_indent, + ..Default::default() + }; + module.codegen(&mut state); + if state.tokens != src { + bail!("libcst does not round-trip this file byte-for-byte; refusing to rewrite it"); + } + Ok(()) +} + +fn flatten<'a>(node: &NameOrAttribute<'a>) -> Option> { + match node { + NameOrAttribute::N(name) => Some(vec![name.value]), + NameOrAttribute::A(attr) => { + let mut parts = flatten_expr(&attr.value)?; + parts.push(attr.attr.value); + Some(parts) + } + } +} + +fn flatten_expr<'a>(expr: &Expression<'a>) -> Option> { + match expr { + Expression::Name(name) => Some(vec![name.value]), + Expression::Attribute(attr) => { + let mut parts = flatten_expr(&attr.value)?; + parts.push(attr.attr.value); + Some(parts) + } + _ => None, + } +} + +#[derive(Default)] +struct Imports<'m, 'a> { + from_imports: Vec<(&'m ImportFrom<'a>, Vec<&'a str>)>, + plain_imports: Vec<(&'m Import<'a>, Vec<&'a str>)>, +} + +fn collect_small<'m, 'a>( + small: &'m SmallStatement<'a>, + stack: &[&'a str], + out: &mut Imports<'m, 'a>, +) { + match small { + SmallStatement::ImportFrom(import) => out.from_imports.push((import, stack.to_vec())), + SmallStatement::Import(import) => out.plain_imports.push((import, stack.to_vec())), + _ => {} + } +} + +fn collect_imports<'m, 'a>( + stmts: &'m [Statement<'a>], + stack: &mut Vec<&'a str>, + out: &mut Imports<'m, 'a>, +) { + for stmt in stmts { + match stmt { + Statement::Simple(line) => { + for small in &line.body { + collect_small(small, stack, out); + } + } + Statement::Compound(compound) => collect_compound(compound, stack, out), + } + } +} + +fn collect_compound<'m, 'a>( + stmt: &'m CompoundStatement<'a>, + stack: &mut Vec<&'a str>, + out: &mut Imports<'m, 'a>, +) { + match stmt { + CompoundStatement::FunctionDef(f) => collect_suite(&f.body, stack, out), + CompoundStatement::ClassDef(c) => collect_suite(&c.body, stack, out), + CompoundStatement::If(i) => collect_if(i, stack, out), + CompoundStatement::For(f) => { + collect_suite(&f.body, stack, out); + if let Some(e) = &f.orelse { + collect_suite(&e.body, stack, out); + } + } + CompoundStatement::While(w) => { + collect_suite(&w.body, stack, out); + if let Some(e) = &w.orelse { + collect_suite(&e.body, stack, out); + } + } + CompoundStatement::Try(t) => { + collect_suite(&t.body, stack, out); + for handler in &t.handlers { + collect_suite(&handler.body, stack, out); + } + if let Some(e) = &t.orelse { + collect_suite(&e.body, stack, out); + } + if let Some(f) = &t.finalbody { + collect_suite(&f.body, stack, out); + } + } + CompoundStatement::TryStar(t) => { + collect_suite(&t.body, stack, out); + for handler in &t.handlers { + collect_suite(&handler.body, stack, out); + } + if let Some(e) = &t.orelse { + collect_suite(&e.body, stack, out); + } + if let Some(f) = &t.finalbody { + collect_suite(&f.body, stack, out); + } + } + CompoundStatement::With(w) => collect_suite(&w.body, stack, out), + CompoundStatement::Match(m) => { + for case in &m.cases { + collect_suite(&case.body, stack, out); + } + } + } +} + +fn collect_if<'m, 'a>( + node: &'m libcst_native::If<'a>, + stack: &mut Vec<&'a str>, + out: &mut Imports<'m, 'a>, +) { + collect_suite(&node.body, stack, out); + if let Some(orelse) = &node.orelse { + match orelse.as_ref() { + OrElse::Elif(elif) => collect_if(elif, stack, out), + OrElse::Else(e) => collect_suite(&e.body, stack, out), + } + } +} + +fn collect_suite<'m, 'a>( + suite: &'m Suite<'a>, + stack: &mut Vec<&'a str>, + out: &mut Imports<'m, 'a>, +) { + match suite { + Suite::IndentedBlock(block) => { + stack.push(block.indent.unwrap_or(" ")); + collect_imports(&block.body, stack, out); + stack.pop(); + } + Suite::SimpleStatementSuite(line) => { + for small in &line.body { + collect_small(small, stack, out); + } + } + } +} + +fn module_imports<'m, 'a>(module: &'m Module<'a>) -> Imports<'m, 'a> { + let mut imports = Imports::default(); + collect_imports(&module.body, &mut Vec::new(), &mut imports); + imports +} + +fn render_with_module<'a>( + import: &ImportFrom<'a>, + dots: usize, + module: &str, + indents: &[&'a str], +) -> String { + let mut text = String::from("from"); + text.push_str(&render_indented(&import.whitespace_after_from, indents)); + text.push_str(&".".repeat(dots)); + text.push_str(module); + text.push_str(&render_indented(&import.whitespace_before_import, indents)); + text.push_str("import"); + text.push_str(&render_indented(&import.whitespace_after_import, indents)); + if let Some(lpar) = &import.lpar { + text.push_str(&render_indented(lpar, indents)); + } + text.push_str(&render_indented(&import.names, indents)); + if let Some(rpar) = &import.rpar { + text.push_str(&render_indented(rpar, indents)); + } + if let Some(semi) = &import.semicolon { + text.push_str(&render_indented(semi, indents)); + } + text +} + +struct Rewrite { + old: String, + new: String, + nodes: usize, +} + +fn add_rewrite(rewrites: &mut Vec, old: String, new: String) { + match rewrites.iter_mut().find(|r| r.old == old) { + Some(existing) => { + debug_assert_eq!(existing.new, new); + existing.nodes += 1; + } + None => rewrites.push(Rewrite { old, new, nodes: 1 }), + } +} + +// Every parse in this module goes through here, including the ones that only +// want to know whether the text is still valid Python: a validated tree is +// worth keeping, because the op that reads that file next, and the verify pass +// at the end of the run, would otherwise rebuild it from scratch. +// +// The error is returned as libcst wrote it so that each caller can keep the +// wording of its own failure. +fn cached_module(src: &str) -> std::result::Result, String> { + let key = memo::digest(src); + if let Some(hit) = memo::module(&key) { + return Ok(hit); + } + let owned = OwnedModule::try_new(src.to_string(), |owner| { + libcst_native::parse_module(owner.as_str(), None).map_err(|e| e.to_string()) + })?; + memo::note_parses(key); + Ok(memo::note_module(key, owned)) +} + +fn module_of(path: &str, src: &str) -> Result> { + cached_module(src).map_err(|e| anyhow::anyhow!("parsing {path}: {e}")) +} + +// The round-trip check gates rewriting, not reading, so it stays outside the +// cache: `absolute_self_imports` only inspects imports and must not reject a +// file merely because libcst would reformat it. +fn parsed_module(path: &str, src: &str) -> Result> { + let owned = module_of(path, src)?; + check_roundtrip(owned.borrow_dependent(), src).with_context(|| path.to_string())?; + Ok(owned) +} + +pub fn validate_ensure_import(from: &str, name: &str) -> Result<()> { + let statement = format!("from {from} import {name}\n"); + let module = libcst_native::parse_module(&statement, None) + .map_err(|e| anyhow::anyhow!("invalid import `from {from} import {name}`: {e}"))?; + let [Statement::Simple(line)] = module.body.as_slice() else { + bail!("invalid import `from {from} import {name}`"); + }; + let [SmallStatement::ImportFrom(import)] = line.body.as_slice() else { + bail!("invalid import `from {from} import {name}`"); + }; + let ImportNames::Aliases(names) = &import.names else { + bail!("import name must be a Python identifier, got {name:?}"); + }; + if names.len() != 1 + || names[0].asname.is_some() + || flatten(&names[0].name).is_none_or(|parts| parts.as_slice() != [name]) + { + bail!("import name must be a Python identifier, got {name:?}"); + } + Ok(()) +} + +fn same_import_source(import: &ImportFrom<'_>, wanted: &ImportFrom<'_>) -> bool { + import.relative.len() == wanted.relative.len() + && import.module.as_ref().and_then(flatten) == wanted.module.as_ref().and_then(flatten) +} + +fn imported_name_count(import: &ImportFrom<'_>, name: &str) -> usize { + let ImportNames::Aliases(names) = &import.names else { + return 0; + }; + names + .iter() + .filter(|alias| { + alias.asname.is_none() + && flatten(&alias.name).is_some_and(|parts| parts.as_slice() == [name]) + }) + .count() +} + +// Ensure an explicit top-level from-import exists. New imports are appended to +// the module body: package initializers commonly define names before their +// imports, so moving the import into a guessed "import block" can change +// initialization and circular-import behavior. +pub fn ensure_import_source( + path: &str, + src: &str, + from: &str, + name: &str, +) -> Result> { + validate_ensure_import(from, name)?; + let wanted_text = format!("from {from} import {name}\n"); + let wanted_module = libcst_native::parse_module(&wanted_text, None).unwrap(); + let Statement::Simple(wanted_line) = &wanted_module.body[0] else { + unreachable!(); + }; + let SmallStatement::ImportFrom(wanted) = &wanted_line.body[0] else { + unreachable!(); + }; + + let owned = parsed_module(path, src)?; + let module = owned.borrow_dependent(); + let mut matches = 0; + for statement in &module.body { + let Statement::Simple(line) = statement else { + continue; + }; + for small in &line.body { + if let SmallStatement::ImportFrom(import) = small + && same_import_source(import, wanted) + { + matches += imported_name_count(import, name); + } + } + } + if matches > 1 { + bail!( + "{path}: `from {from} import {name}` is already satisfied by {matches} top-level imports; remove the duplicate" + ); + } + if matches == 1 { + return Ok(None); + } + + let mut state = CodegenState { + default_newline: module.default_newline, + default_indent: module.default_indent, + ..Default::default() + }; + for header in &module.header { + header.codegen(&mut state); + } + for statement in &module.body { + statement.codegen(&mut state); + } + let insert_at = state.tokens.len(); + if !src.starts_with(&state.tokens) { + bail!("{path}: could not locate the end of the module body"); + } + + let newline = module.default_newline; + let mut addition = String::new(); + if insert_at > 0 && !state.tokens.ends_with(['\n', '\r']) { + addition.push_str(newline); + } + addition.push_str(&format!("from {from} import {name}")); + if module.has_trailing_newline || !module.footer.is_empty() { + addition.push_str(newline); + } + + let mut result = src.to_string(); + result.insert_str(insert_at, &addition); + if let Err(e) = cached_module(&result) { + bail!("{path}: ensured import output no longer parses: {e}"); + } + Ok(Some((result, 1))) +} + +fn finish_rewrites( + path: &str, + src: &str, + rewrites: Vec, + what: &str, +) -> Result> { + if rewrites.is_empty() { + return Ok(None); + } + let count = rewrites.iter().map(|r| r.nodes).sum(); + let result = splice(path, src, rewrites)?; + if let Err(e) = cached_module(&result) { + bail!("{path}: {what} output no longer parses: {e}"); + } + Ok(Some((result, count))) +} + +// Boundary-aware, so `from pkg.utils import x` does not match inside +// `from pkg.utils_extra import x`. +fn statement_occurrences(text: &str, needle: &str) -> Vec { + let is_ident = |c: u8| c.is_ascii_alphanumeric() || c == b'_'; + let bytes = text.as_bytes(); + let mut positions = Vec::new(); + let mut start = 0; + while let Some(rel) = text[start..].find(needle) { + let pos = start + rel; + let before_ok = pos == 0 || !is_ident(bytes[pos - 1]); + let end = pos + needle.len(); + let suffix = &text[end..]; + let after_ok = end == bytes.len() + || (!is_ident(bytes[end]) + && bytes[end] != b'.' + && bytes[end] != b',' + && !suffix.starts_with(" as ")); + if before_ok && after_ok { + positions.push(pos); + } + start = pos + 1; + } + positions +} + +// Rewrites are spliced into the text. Regenerating the module through libcst +// would reformat everything the rewrite did not touch. +fn splice(path: &str, src: &str, rewrites: Vec) -> Result { + let mut out = src.to_string(); + for rw in rewrites { + let positions = statement_occurrences(&out, &rw.old); + if positions.len() != rw.nodes { + bail!( + "{path}: statement {:?} occurs {} time(s) in the text but \ + {} time(s) as an import node; refusing to splice", + rw.old, + positions.len(), + rw.nodes + ); + } + for pos in positions.into_iter().rev() { + out.replace_range(pos..pos + rw.old.len(), &rw.new); + } + } + Ok(out) +} + +fn rewrite_from_imports( + path: &str, + src: &str, + plan: impl Fn(&[&str]) -> Option<(usize, String)>, +) -> Result> { + let owned = parsed_module(path, src)?; + let module = owned.borrow_dependent(); + let mut rewrites: Vec = Vec::new(); + for (import, indents) in module_imports(module).from_imports { + if !import.relative.is_empty() { + continue; + } + let Some(target) = import.module.as_ref().and_then(flatten) else { + continue; + }; + let Some((dots, new_module)) = plan(&target) else { + continue; + }; + add_rewrite( + &mut rewrites, + render_indented(import, &indents), + render_with_module(import, dots, &new_module, &indents), + ); + } + finish_rewrites(path, src, rewrites, "rewritten") +} + +pub fn relativize_source( + path: &str, + src: &str, + package: &[impl AsRef], +) -> Result> { + rewrite_from_imports(path, src, |target| { + if target[0] != package[0].as_ref() { + return None; + } + let k = package + .iter() + .zip(target.iter()) + .take_while(|(x, y)| x.as_ref() == **y) + .count(); + // One dot for the file's own package, plus one per level climbed out + // of it to reach the common ancestor. + let dots = package.len() - k + 1; + Some((dots, target[k..].join("."))) + }) +} + +pub fn relativize_source_from_root( + path: &str, + src: &str, + package: &[impl AsRef], +) -> Result> { + rewrite_from_imports(path, src, |target| { + if target[0] != package[0].as_ref() { + return None; + } + Some((package.len(), target[1..].join("."))) + }) +} + +pub fn remap_source( + path: &str, + src: &str, + from_prefix: &DottedPath, + to_prefix: &DottedPath, +) -> Result> { + let from = from_prefix.parts(); + rewrite_from_imports(path, src, |target| { + if target.len() < from.len() || target[..from.len()] != *from { + return None; + } + let mut new = to_prefix.parts().to_vec(); + new.extend( + target[from.len()..] + .iter() + .map(std::string::ToString::to_string), + ); + Some((0, new.join("."))) + }) +} + +pub fn convert_imports_source( + path: &str, + src: &str, + prefix: &DottedPath, +) -> Result> { + let prefix = prefix.parts(); + let owned = parsed_module(path, src)?; + let module = owned.borrow_dependent(); + let mut rewrites: Vec = Vec::new(); + for (import, indents) in module_imports(module).plain_imports { + let matching: Vec<_> = import + .names + .iter() + .filter(|alias| { + flatten(&alias.name) + .is_some_and(|t| t.len() >= prefix.len() && t[..prefix.len()] == *prefix) + }) + .collect(); + if matching.is_empty() { + continue; + } + if import.names.len() > 1 { + bail!( + "{path}: multi-name import statement mentions the prefix; split it \ + with a replace before convert_import" + ); + } + let alias = matching[0]; + let target = flatten(&alias.name).unwrap(); + if target.len() < 2 { + bail!( + "{path}: `import {}` cannot be converted to a from-import; handle it \ + with a replace", + target.join(".") + ); + } + let Some(asname) = &alias.asname else { + bail!( + "{path}: `import {}` without an alias binds the top-level package; \ + handle it with a replace", + target.join(".") + ); + }; + let libcst_native::AssignTargetExpression::Name(alias_name) = &asname.name else { + bail!("{path}: unsupported import alias form"); + }; + let mut new = format!( + "from {} import {} as {}", + target[..target.len() - 1].join("."), + target[target.len() - 1], + alias_name.value + ); + if let Some(semi) = &import.semicolon { + new.push_str(&render(semi)); + } + add_rewrite(&mut rewrites, render_indented(import, &indents), new); + } + finish_rewrites(path, src, rewrites, "converted") +} + +fn import_alias_binding<'a>(path: &str, alias: &'a ImportAlias<'_>) -> Result<&'a str> { + if let Some(asname) = &alias.asname { + let AssignTargetExpression::Name(name) = &asname.name else { + bail!("{path}: unsupported import alias form"); + }; + return Ok(name.value); + } + let Some(parts) = flatten(&alias.name) else { + bail!("{path}: unsupported import name"); + }; + Ok(parts[0]) +} + +fn kernel_module_expr(helper: &str, suffix: &[&str]) -> String { + if suffix.is_empty() { + format!("{helper}()") + } else { + let suffix = serde_json::to_string(&suffix.join(".")).unwrap(); + format!("{helper}({suffix})") + } +} + +fn append_original_semicolon(text: &mut String, semicolon: Option<&libcst_native::Semicolon<'_>>) { + if let Some(semicolon) = semicolon { + text.push_str(&render(semicolon)); + } +} + +fn kernelize_from_import( + path: &str, + import: &ImportFrom<'_>, + package: &str, + helper: &str, +) -> Result> { + if !import.relative.is_empty() { + return Ok(None); + } + let Some(target) = import.module.as_ref().and_then(flatten) else { + return Ok(None); + }; + if target.first() != Some(&package) { + return Ok(None); + } + if import.lpar.is_some() || import.rpar.is_some() { + bail!( + "{path}: parenthesized import from `{}` is not supported by kernelize_imports; split it into a one-line import", + target.join(".") + ); + } + let ImportNames::Aliases(aliases) = &import.names else { + bail!( + "{path}: wildcard import from `{}` cannot be kernelized safely", + target.join(".") + ); + }; + let module = kernel_module_expr(helper, &target[1..]); + let mut bindings = Vec::with_capacity(aliases.len()); + let mut values = Vec::with_capacity(aliases.len()); + for alias in aliases { + let Some(imported) = flatten(&alias.name) else { + bail!("{path}: unsupported imported name"); + }; + if imported.len() != 1 { + bail!("{path}: unsupported imported name {}", imported.join(".")); + } + let binding = import_alias_binding(path, alias)?; + let imported = serde_json::to_string(imported[0]).unwrap(); + bindings.push(binding); + values.push(format!("getattr({module}, {imported})")); + } + let mut replacement = format!("{} = {}", bindings.join(", "), values.join(", ")); + append_original_semicolon(&mut replacement, import.semicolon.as_ref()); + Ok(Some(replacement)) +} + +fn kernelize_plain_import( + path: &str, + import: &Import<'_>, + package: &str, + helper: &str, +) -> Result> { + let matching: Vec<_> = import + .names + .iter() + .filter(|alias| flatten(&alias.name).is_some_and(|parts| parts.first() == Some(&package))) + .collect(); + if matching.is_empty() { + return Ok(None); + } + if import.names.len() != 1 { + bail!( + "{path}: a multi-name import statement mentions {package:?}; split it before kernelize_imports" + ); + } + let alias = matching[0]; + let target = flatten(&alias.name).unwrap(); + let binding = import_alias_binding(path, alias)?; + let root = kernel_module_expr(helper, &[]); + let mut replacement = if target.len() == 1 { + format!("{binding} = {root}") + } else { + let module = kernel_module_expr(helper, &target[1..]); + if alias.asname.is_some() { + format!("{binding} = {module}") + } else { + // `import pkg.sub` binds pkg and also loads pkg.sub. + format!("{binding} = {root}; {module}") + } + }; + append_original_semicolon(&mut replacement, import.semicolon.as_ref()); + Ok(Some(replacement)) +} + +fn is_module_docstring(statement: &Statement<'_>) -> bool { + let Statement::Simple(line) = statement else { + return false; + }; + let [SmallStatement::Expr(expr)] = line.body.as_slice() else { + return false; + }; + matches!( + expr.value, + Expression::SimpleString(_) | Expression::ConcatenatedString(_) + ) +} + +fn is_future_import(statement: &Statement<'_>) -> bool { + let Statement::Simple(line) = statement else { + return false; + }; + let [SmallStatement::ImportFrom(import)] = line.body.as_slice() else { + return false; + }; + import.relative.is_empty() + && import.module.as_ref().and_then(flatten).as_deref() == Some(["__future__"].as_slice()) +} + +fn insert_kernel_helper( + path: &str, + src: &str, + helper: &str, + kernel: &str, + version: usize, +) -> Result { + let owned = parsed_module(path, src)?; + let module = owned.borrow_dependent(); + let mut body_index = usize::from(module.body.first().is_some_and(is_module_docstring)); + while module.body.get(body_index).is_some_and(is_future_import) { + body_index += 1; + } + + let mut state = CodegenState { + default_newline: module.default_newline, + default_indent: module.default_indent, + ..Default::default() + }; + for header in &module.header { + header.codegen(&mut state); + } + for statement in &module.body[..body_index] { + statement.codegen(&mut state); + } + let insert_at = state.tokens.len(); + if !src.starts_with(&state.tokens) { + bail!("{path}: could not locate the kernel import helper insertion point"); + } + + let newline = module.default_newline; + let indent = module.default_indent; + let kernel = serde_json::to_string(kernel).unwrap(); + let mut helper_source = String::new(); + if insert_at > 0 && !state.tokens.ends_with(['\n', '\r']) { + helper_source.push_str(newline); + } + let cached_root = format!("{helper}_root"); + helper_source.push_str(&format!("{cached_root} = None{newline}")); + helper_source.push_str(&format!("def {helper}(module=\"\"):{newline}")); + helper_source.push_str(&format!("{indent}global {cached_root}{newline}")); + helper_source.push_str(&format!("{indent}if {cached_root} is None:{newline}")); + helper_source.push_str(&format!( + "{indent}{indent}{cached_root} = __import__(\"kernels\").get_kernel({kernel}, version={version}){newline}" + )); + helper_source.push_str(&format!("{indent}root = {cached_root}{newline}")); + helper_source.push_str(&format!("{indent}if not module:{newline}")); + helper_source.push_str(&format!("{indent}{indent}return root{newline}")); + helper_source.push_str(&format!( + "{indent}return __import__(\"importlib\").import_module(root.__name__ + \".\" + module){newline}{newline}" + )); + + let mut result = src.to_string(); + result.insert_str(insert_at, &helper_source); + check_parses(path, &result)?; + Ok(result) +} + +pub fn kernelize_imports_source( + path: &str, + src: &str, + package: &str, + kernel: &str, + version: usize, +) -> Result> { + let parsed_package: DottedPath = package.parse()?; + if parsed_package.parts().len() != 1 { + bail!("kernelized package must be one top-level Python name, got {package:?}"); + } + if kernel.is_empty() { + bail!("kernel must not be empty"); + } + + let helper_base = format!("__kernel_port_{package}"); + let mut helper = helper_base.clone(); + let mut suffix = 2; + while src.contains(&helper) { + helper = format!("{helper_base}_{suffix}"); + suffix += 1; + } + + let owned = parsed_module(path, src)?; + let module = owned.borrow_dependent(); + let imports = module_imports(module); + let mut rewrites = Vec::new(); + for (import, indents) in imports.from_imports { + if let Some(new) = kernelize_from_import(path, import, package, &helper)? { + add_rewrite(&mut rewrites, render_indented(import, &indents), new); + } + } + for (import, indents) in imports.plain_imports { + if let Some(new) = kernelize_plain_import(path, import, package, &helper)? { + add_rewrite(&mut rewrites, render_indented(import, &indents), new); + } + } + let Some((result, count)) = finish_rewrites(path, src, rewrites, "kernelized")? else { + return Ok(None); + }; + let result = insert_kernel_helper(path, &result, &helper, kernel, version)?; + let remaining = absolute_self_imports(path, &result, package)?; + if !remaining.is_empty() { + bail!( + "{path}: kernelize_imports left matching static import(s): {}", + remaining.join(", ") + ); + } + Ok(Some((result, count))) +} + +pub fn absolute_self_imports(path: &str, src: &str, package: &str) -> Result> { + let key = (memo::digest(src), package.to_string()); + if let Some(found) = memo::self_imports(&key) { + return Ok(found); + } + let owned = module_of(path, src)?; + let imports = module_imports(owned.borrow_dependent()); + + let mut offenders = Vec::new(); + for (import, _) in imports.from_imports { + if !import.relative.is_empty() { + continue; + } + if let Some(target) = import.module.as_ref().and_then(flatten) + && target[0] == package + { + offenders.push(format!("from {} import ...", target.join("."))); + } + } + for (import, _) in imports.plain_imports { + for alias in &import.names { + if let Some(target) = flatten(&alias.name) + && target[0] == package + { + offenders.push(format!("import {}", target.join("."))); + } + } + } + memo::note_self_imports(key, offenders.clone()); + Ok(offenders) +} + +pub fn check_parses(path: &str, src: &str) -> Result<()> { + if memo::parses(memo::digest(src)) { + return Ok(()); + } + if let Err(e) = cached_module(src) { + bail!("verify: {path} does not parse as Python: {e}"); + } + Ok(()) +} diff --git a/kernel-port/src/recipe.rs b/kernel-port/src/recipe.rs new file mode 100644 index 00000000..ba02ad8d --- /dev/null +++ b/kernel-port/src/recipe.rs @@ -0,0 +1,170 @@ +use anyhow::{Context, Result, bail}; +use std::collections::BTreeMap; + +#[derive(Debug)] +pub struct Invocation { + pub line: usize, + pub op: String, + pub args: BTreeMap, +} + +impl Invocation { + pub fn take_args(&self) -> Args<'_> { + Args { + op: &self.op, + map: self.args.clone(), + } + } +} + +pub struct Args<'a> { + op: &'a str, + map: BTreeMap, +} + +impl Args<'_> { + pub fn take(&mut self, key: &str) -> Result { + self.map + .remove(key) + .with_context(|| format!("op {:?} requires argument {key}=\"...\"", self.op)) + } + + pub fn take_opt(&mut self, key: &str) -> Option { + self.map.remove(key) + } + + pub fn take_usize(&mut self, key: &str) -> Result { + self.take_usize_opt(key)? + .with_context(|| format!("op {:?} requires argument {key}=\"...\"", self.op)) + } + + pub fn take_usize_opt(&mut self, key: &str) -> Result> { + match self.map.remove(key) { + Some(v) => Ok(Some(v.parse().with_context(|| { + format!("argument {key} of op {:?} must be an integer", self.op) + })?)), + None => Ok(None), + } + } + + pub fn finish(self) -> Result<()> { + if let Some(key) = self.map.keys().next() { + bail!("op {:?} does not take an argument named {key:?}", self.op); + } + Ok(()) + } +} + +// Bumped when a change gives an existing recipe a different meaning, not when +// it adds a capability. +pub const VERSION: u64 = 1; + +pub struct Recipe { + pub version: Option, + pub ops: Vec, +} + +impl Recipe { + pub const fn effective_version(&self) -> u64 { + match self.version { + Some(v) => v, + None => VERSION, + } + } +} + +pub fn parse(text: &str) -> Result { + let doc = kdl::KdlDocument::parse_v2(text).map_err(|e| format_parse_error(&e))?; + let mut out = Vec::new(); + for node in doc.nodes() { + let line = line_of(text, node.span().offset()); + let op = node.name().value().to_string(); + if node.ty().is_some() { + bail!("recipe line {line}: {op}: type annotations are not part of the recipe language"); + } + if node.children().is_some() { + bail!("recipe line {line}: {op}: children blocks are not part of the recipe language"); + } + let mut args = BTreeMap::new(); + for entry in node.entries() { + let Some(name) = entry.name() else { + bail!( + "recipe line {line}: {op}: positional arguments are not allowed; \ + every argument is key=value" + ); + }; + let key = name.value().to_string(); + if entry.ty().is_some() { + bail!( + "recipe line {line}: {op}: type annotations are not part of the \ + recipe language" + ); + } + let value = match entry.value() { + kdl::KdlValue::String(s) => s.clone(), + kdl::KdlValue::Integer(n) => n.to_string(), + kdl::KdlValue::Bool(b) => b.to_string(), + other => bail!( + "recipe line {line}: {op}: argument {key} must be a string, \ + integer, or #true/#false, got {other}" + ), + }; + // KDL takes the rightmost of a repeated property; here that would + // swallow a typo. + if args.insert(key.clone(), value).is_some() { + bail!("recipe line {line}: duplicate argument {key:?}"); + } + } + out.push(Invocation { line, op, args }); + } + let version = take_version_header(&mut out)?; + Ok(Recipe { version, ops: out }) +} + +// The header is not an op: it declares the format the rest of the file is +// written in, so it means nothing anywhere but the first node. +fn take_version_header(ops: &mut Vec) -> Result> { + if let Some(stray) = ops.iter().skip(1).find(|inv| inv.op == "recipe") { + bail!( + "recipe line {}: the `recipe version=N` header must be the first \ + node in the file", + stray.line + ); + } + if ops.first().is_none_or(|inv| inv.op != "recipe") { + return Ok(None); + } + let header = ops.remove(0); + let mut args = header.take_args(); + let raw = args.take("version")?; + args.finish()?; + let version: u64 = raw + .parse() + .with_context(|| format!("recipe version must be a positive integer, got {raw:?}"))?; + if version == 0 || version > VERSION { + bail!( + "recipe declares version {version}, but this kernel-port implements \ + version {VERSION} - upgrade kernel-port, or re-verify the recipe \ + against this version and update its header" + ); + } + Ok(Some(version)) +} + +// KDL reports char offsets. Recipes are ASCII, where chars and bytes coincide. +fn line_of(src: &str, offset: usize) -> usize { + src.chars().take(offset).filter(|&c| c == '\n').count() + 1 +} + +fn format_parse_error(err: &kdl::KdlError) -> anyhow::Error { + let mut lines = vec!["recipe parse error:".to_string()]; + for d in &err.diagnostics { + let line = line_of(&err.input, d.span.offset()); + let msg = d.message.as_deref().unwrap_or("invalid syntax"); + match &d.help { + Some(help) => lines.push(format!(" line {line}: {msg} ({help})")), + None => lines.push(format!(" line {line}: {msg}")), + } + } + anyhow::anyhow!(lines.join("\n")) +} diff --git a/kernel-port/src/tests.rs b/kernel-port/src/tests.rs new file mode 100644 index 00000000..cc08de20 --- /dev/null +++ b/kernel-port/src/tests.rs @@ -0,0 +1,740 @@ +use crate::{ops, python, recipe, workspace::Workspace}; +use std::collections::BTreeMap; +use std::path::Path; + +#[test] +fn recipe_basic_statement() { + let parsed = recipe::parse("delete in=\".github/**\" // trailing comment\n").unwrap(); + assert_eq!(parsed.ops.len(), 1); + assert_eq!(parsed.ops[0].op, "delete"); + assert_eq!(parsed.ops[0].take_args().take("in").unwrap(), ".github/**"); + assert_eq!(parsed.ops[0].line, 1); +} + +#[test] +fn replace_defaults_to_one_match() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("a.py".into(), b"before\n".to_vec()), + ("b.py".into(), b"before\n".to_vec()), + ])); + run_recipe( + &mut ws, + "replace in=\"*.py\" find=\"before\" with=\"after\"\n", + ); + assert_eq!(ws.get_text("a.py").unwrap(), "after\n"); + assert_eq!(ws.get_text("b.py").unwrap(), "after\n"); +} + +#[test] +fn recipe_escapes_and_typed_values() { + let parsed = + recipe::parse(r#"replace in="f" find="a\nb\t\"q\"\\" with="" count=3 flag=#true"#).unwrap(); + let mut args = parsed.ops[0].take_args(); + assert_eq!(args.take("find").unwrap(), "a\nb\t\"q\"\\"); + assert_eq!(args.take("with").unwrap(), ""); + assert_eq!(args.take_usize("count").unwrap(), 3); + assert_eq!(args.take("flag").unwrap(), "true"); +} + +#[test] +fn recipe_multiline_raw_string_boundary_newlines() { + let parsed = recipe::parse( + "replace in=\"f\" count=1 with=#\"\"\"\n\nno \\n escape \"quoted\"\n\n\"\"\"# find=\"x\"\n", + ) + .unwrap(); + let mut args = parsed.ops[0].take_args(); + assert_eq!(args.take("with").unwrap(), "\nno \\n escape \"quoted\"\n"); + assert_eq!(args.take("find").unwrap(), "x"); +} + +#[test] +fn recipe_multiline_without_trailing_newline() { + let parsed = + recipe::parse("expect in=\"f\" count=1 find=#\"\"\"\n\nvoid\nf(int x) {\n\"\"\"#\n") + .unwrap(); + let mut args = parsed.ops[0].take_args(); + assert_eq!(args.take("find").unwrap(), "\nvoid\nf(int x) {"); +} + +#[test] +fn recipe_comments_blanks_and_line_numbers() { + let parsed = recipe::parse("// header\n\nmove from=\"a\" to=\"b\"\n\n// tail\n").unwrap(); + assert_eq!(parsed.ops.len(), 1); + assert_eq!(parsed.ops[0].line, 3); +} + +#[test] +fn recipe_rejects_duplicate_and_unknown_args() { + let err = recipe::parse("delete in=\"a\" in=\"b\"\n") + .err() + .expect("duplicate arguments are rejected") + .to_string(); + assert!(err.contains("duplicate argument"), "{err}"); + let parsed = recipe::parse("delete in=\"a\" bogus=\"b\"\n").unwrap(); + let mut args = parsed.ops[0].take_args(); + args.take("in").unwrap(); + assert!(args.finish().is_err()); +} + +#[test] +fn recipe_rejects_foreign_kdl_shapes() { + // KDL allows these shapes; the recipe language does not. + for (text, what) in [ + ("delete \"a\"\n", "positional"), + ("delete in=\"a\" { child; }\n", "children"), + ("delete in=(glob)\"a\"\n", "type annotation"), + ("manifest version=1.5\n", "float value"), + ] { + assert!(recipe::parse(text).is_err(), "{what} should be rejected"); + } +} + +#[test] +fn recipe_version_header_is_consumed_not_run() { + let parsed = recipe::parse("recipe version=1\ndelete in=\"a\"\n").unwrap(); + assert_eq!(parsed.version, Some(1)); + assert_eq!(parsed.ops.len(), 1); + assert_eq!(parsed.ops[0].op, "delete"); + assert_eq!(parsed.ops[0].line, 2); +} + +#[test] +fn recipe_without_header_has_no_declared_version() { + let parsed = recipe::parse("delete in=\"a\"\n").unwrap(); + assert_eq!(parsed.version, None); + assert_eq!(parsed.effective_version(), recipe::VERSION); +} + +#[test] +fn recipe_version_rejects_unknown_and_misplaced_headers() { + let err = |text: &str| recipe::parse(text).err().unwrap().to_string(); + assert!(err("recipe version=99\n").contains("this kernel-port implements version 1")); + assert!(err("recipe version=0\n").contains("this kernel-port implements version 1")); + assert!(err("delete in=\"a\"\nrecipe version=1\n").contains("must be the first")); + assert!(err("recipe\n").contains("requires argument version")); + assert!(err("recipe version=1 name=\"x\"\n").contains("does not take an argument named")); +} + +#[test] +fn relativize_dots_formula() { + let src = "from pkg.core import base\nfrom pkg.core.base import Base\nfrom pkg.utils import x\nfrom os.path import join\n"; + let (out, n) = python::relativize_source("f.py", src, &["pkg", "core"]) + .unwrap() + .unwrap(); + assert_eq!(n, 3); + assert_eq!( + out, + "from . import base\nfrom .base import Base\nfrom ..utils import x\nfrom os.path import join\n" + ); +} + +#[test] +fn relativize_from_package_root_preserves_full_module_path() { + let src = "from pkg.ops import base\nfrom pkg.ops.base import Base\nfrom pkg.utils import x\n"; + let (out, n) = + python::relativize_source_from_root("pkg/ops/core/engine.py", src, &["pkg", "ops", "core"]) + .unwrap() + .unwrap(); + assert_eq!(n, 3); + assert_eq!( + out, + "from ...ops import base\nfrom ...ops.base import Base\nfrom ...utils import x\n" + ); +} + +#[test] +fn relativize_preserves_comments_and_layout() { + let src = "from pkg.a import (\n x, # keep\n)\n"; + let (out, _) = python::relativize_source("f.py", src, &["pkg"]) + .unwrap() + .unwrap(); + assert_eq!(out, "from .a import (\n x, # keep\n)\n"); +} + +#[test] +fn relativize_untouched_returns_none() { + let src = "import os\nfrom .a import b\n"; + assert!( + python::relativize_source("f.py", src, &["pkg"]) + .unwrap() + .is_none() + ); +} + +#[test] +// infer_device is a text prefix of infer_device_arch: each statement has to be +// spliced on its own boundaries. +fn remap_prefix_and_boundary_collision() { + let src = "from liger_kernel.utils import infer_device\nfrom liger_kernel.utils import infer_device_arch\n"; + let from: python::DottedPath = "liger_kernel.utils".parse().unwrap(); + let to: python::DottedPath = "liger_kernels._liger_utils".parse().unwrap(); + let (out, n) = python::remap_source("f.py", src, &from, &to) + .unwrap() + .unwrap(); + assert_eq!(n, 2); + assert_eq!( + out, + "from liger_kernels._liger_utils import infer_device\nfrom liger_kernels._liger_utils import infer_device_arch\n" + ); +} + +#[test] +fn remap_does_not_touch_other_prefixes() { + let src = "from liger_kernel.ops.rms_norm import f\n"; + let from: python::DottedPath = "liger_kernel.utils".parse().unwrap(); + let to: python::DottedPath = "x".parse().unwrap(); + assert!( + python::remap_source("f.py", src, &from, &to) + .unwrap() + .is_none() + ); +} + +#[test] +fn splice_refuses_string_literal_duplicates() { + let src = "from pkg.a import b\ns = \"from pkg.a import b\"\n"; + assert!(python::relativize_source("f.py", src, &["pkg"]).is_err()); +} + +#[test] +fn imports_inside_functions_are_rewritten() { + let src = "def f():\n from pkg.a import b\n return b\n"; + let (out, _) = python::relativize_source("f.py", src, &["pkg"]) + .unwrap() + .unwrap(); + assert_eq!(out, "def f():\n from .a import b\n return b\n"); +} + +#[test] +fn ensure_import_appends_after_module_initialization() { + let src = "\"\"\"docs\"\"\"\n\nVALUE = 1\nfrom .packing import pack\n"; + let (out, n) = python::ensure_import_source("pkg/__init__.py", src, ".", "array_api") + .unwrap() + .unwrap(); + assert_eq!(n, 1); + assert_eq!( + out, + "\"\"\"docs\"\"\"\n\nVALUE = 1\nfrom .packing import pack\nfrom . import array_api\n" + ); +} + +#[test] +fn ensure_import_recognizes_an_existing_name_in_a_group() { + let src = "from . import other, array_api # public modules\n"; + assert!( + python::ensure_import_source("pkg/__init__.py", src, ".", "array_api") + .unwrap() + .is_none() + ); +} + +#[test] +fn ensure_import_rejects_duplicate_names_in_one_group() { + let src = "from . import array_api, array_api\n"; + let err = python::ensure_import_source("pkg/__init__.py", src, ".", "array_api") + .unwrap_err() + .to_string(); + assert!(err.contains("satisfied by 2 top-level imports"), "{err}"); +} + +#[test] +fn ensure_import_ignores_nested_imports_and_preserves_no_final_newline() { + let src = "def load():\n from . import array_api\n"; + let (out, _) = python::ensure_import_source("pkg/__init__.py", src, ".", "array_api") + .unwrap() + .unwrap(); + assert_eq!( + out, + "def load():\n from . import array_api\nfrom . import array_api\n" + ); + + let (out, _) = python::ensure_import_source("pkg/__init__.py", "VALUE = 1", ".", "array_api") + .unwrap() + .unwrap(); + assert_eq!(out, "VALUE = 1\nfrom . import array_api"); +} + +#[test] +fn ensure_import_recipe_pins_changes() { + let original = BTreeMap::from([("pkg/__init__.py".into(), b"VALUE = 1\n".to_vec())]); + let mut ws = Workspace::from_files(original.clone()); + run_recipe( + &mut ws, + "ensure_import in=\"pkg/__init__.py\" from=\".\" name=\"array_api\" changes=1\n", + ); + assert_eq!( + ws.get_text("pkg/__init__.py").unwrap(), + "VALUE = 1\nfrom . import array_api\n" + ); + + let mut ws = Workspace::from_files(original); + let err = run_recipe_err( + &mut ws, + "ensure_import in=\"pkg/__init__.py\" from=\".\" name=\"array_api\" changes=0\n", + ); + assert!( + err.contains("expected exactly 0 change(s) but made 1"), + "{err}" + ); +} + +#[test] +fn kernelize_imports_preserves_bindings_and_scope() { + let src = concat!( + "import os\n", + "import einops\n", + "import einops as eo\n", + "import einops.layers\n", + "import einops.layers.torch as torch_layers\n", + "from einops import rearrange, reduce as red # public API\n", + "def load():\n", + " from einops.layers.torch import Rearrange as R, Reduce\n", + " return R, Reduce\n", + ); + let (out, n) = python::kernelize_imports_source( + "tests/test_x.py", + src, + "einops", + "kernels-community/einops", + 1, + ) + .unwrap() + .unwrap(); + assert_eq!(n, 6); + assert!( + out.starts_with( + "__kernel_port_einops_root = None\ndef __kernel_port_einops(module=\"\"):\n" + ) + ); + assert!(out.contains( + "root = __import__(\"kernels\").get_kernel(\"kernels-community/einops\", version=1)" + )); + assert!(out.contains("\nimport os\neinops = __kernel_port_einops()\n")); + assert!(out.contains("eo = __kernel_port_einops()")); + assert!(out.contains("einops = __kernel_port_einops(); __kernel_port_einops(\"layers\")")); + assert!(out.contains("torch_layers = __kernel_port_einops(\"layers.torch\")")); + assert!(out.contains("rearrange, red = getattr(")); + assert!(out.contains(" R, Reduce = getattr(")); + assert!(out.contains(" # public API\n")); + assert!( + python::absolute_self_imports("tests/test_x.py", &out, "einops") + .unwrap() + .is_empty() + ); +} + +#[test] +fn kernelize_imports_rejects_unsafe_static_forms() { + for (src, message) in [ + ("from einops import *\n", "wildcard import"), + ( + "from einops import (rearrange, reduce)\n", + "parenthesized import", + ), + ("import os, einops\n", "multi-name import"), + ] { + let err = python::kernelize_imports_source( + "tests/test_x.py", + src, + "einops", + "kernels-community/einops", + 1, + ) + .unwrap_err() + .to_string(); + assert!(err.contains(message), "{err}"); + } +} + +#[test] +fn kernelize_imports_places_a_collision_free_helper_after_future_imports() { + let src = concat!( + "\"\"\"module docs\"\"\"\n", + "from __future__ import annotations\n", + "__kernel_port_einops = \"upstream name\"\n", + "from einops import rearrange\n", + ); + let (out, _) = python::kernelize_imports_source( + "tests/test_x.py", + src, + "einops", + "kernels-community/einops", + 1, + ) + .unwrap() + .unwrap(); + assert!(out.starts_with( + "\"\"\"module docs\"\"\"\nfrom __future__ import annotations\n__kernel_port_einops_2_root = None\ndef __kernel_port_einops_2(module=\"\"):\n" + )); + assert!(out.contains("rearrange = getattr(__kernel_port_einops_2(), \"rearrange\")")); +} + +#[test] +fn kernelize_imports_recipe_pins_statements() { + let mut ws = Workspace::from_files(BTreeMap::from([( + "tests/test_x.py".into(), + b"from einops import rearrange\n".to_vec(), + )])); + run_recipe( + &mut ws, + "kernelize_imports in=\"tests/**\" package=\"einops\" kernel=\"kernels-community/einops\" version=1 changes=1\n", + ); + assert!( + ws.get_text("tests/test_x.py") + .unwrap() + .contains("get_kernel(\"kernels-community/einops\", version=1)") + ); +} + +fn run_recipe_err(ws: &mut Workspace, recipe_text: &str) -> String { + let parsed = recipe::parse(recipe_text).unwrap(); + let inputs = ops::Inputs::default(); + let mut facts = ops::Facts::default(); + for inv in &parsed.ops { + match ops::build(inv, Path::new(".")).and_then(|op| op.apply(ws, &inputs, &mut facts)) { + Ok(_) => {} + Err(e) => return e.to_string(), + } + } + panic!("recipe unexpectedly succeeded"); +} + +fn run_recipe(ws: &mut Workspace, recipe_text: &str) { + let parsed = recipe::parse(recipe_text).unwrap(); + let inputs = ops::Inputs::default(); + let mut facts = ops::Facts::default(); + for inv in &parsed.ops { + let op = ops::build(inv, Path::new(".")).unwrap(); + op.apply(ws, &inputs, &mut facts).unwrap(); + } +} + +#[test] +fn manifest_torch_mode() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\" capabilities=\"9.0\"\nmanifest name=\"k\" version=1 license=\"MIT\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("backends = [\"cuda\"]")); + assert!(toml.contains( + "[kernel.k]\nbackend = \"cuda\"\ncuda-capabilities = [\"9.0\"]\ndepends = [\"torch\"]\nsrc = [\n \"k/a.cu\",\n]" + )); +} + +#[test] +fn manifest_kernel_cuda_flags() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\" cuda_flags=\"-O3,--use_fast_math\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + assert!( + ws.get_text("build.toml") + .unwrap() + .contains("cuda-flags = [\n \"-O3\",\n \"--use_fast_math\",\n]") + ); +} + +#[test] +fn manifest_kernel_cuda_minver() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\" cuda_minver=\"12.9\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + assert!( + ws.get_text("build.toml") + .unwrap() + .contains("backend = \"cuda\"\ncuda-minver = \"12.9\"\ndepends") + ); +} + +#[test] +fn manifest_kernel_rocm_archs() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"rocm\" src=\"k/*\" rocm_archs=\"gfx90a,gfx942\"\nmanifest name=\"k\" backends=\"rocm\" torch_src=\"torch-ext/*.cpp\"\n", + ); + assert!(ws.get_text("build.toml").unwrap().contains( + "backend = \"rocm\"\ndepends = [\"torch\"]\nrocm-archs = [\n \"gfx90a\",\n \"gfx942\",\n]\n" + )); +} + +#[test] +fn manifest_kernel_repeat_src() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\" repeat_src=\"k/a.cu\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert_eq!(toml.matches("\"k/a.cu\"").count(), 2); +} + +#[test] +fn manifest_kernel_cuda_flags_preserve_doubled_comma() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\" cuda_flags=\"-O3,--ptxas-options=--verbose,,--warn-on-local-memory-usage\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + assert!(ws.get_text("build.toml").unwrap().contains( + "cuda-flags = [\n \"-O3\",\n \"--ptxas-options=--verbose,--warn-on-local-memory-usage\",\n]" + )); +} + +#[test] +fn manifest_cuda_version_bounds() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"k\" backends=\"cuda\" cuda_minver=\"12.0\" cuda_maxver=\"12.9\" noarch=#true\n", + ); + assert!( + ws.get_text("build.toml") + .unwrap() + .contains("[general.cuda]\nminver = \"12.0\"\nmaxver = \"12.9\"\n") + ); +} + +#[test] +fn glob_lists_keep_brace_alternates_whole() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.h".into(), b"x".to_vec()), + ("k/a.cpp".into(), b"x".to_vec()), + ("k/a.py".into(), b"x".to_vec()), + ("t/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*.{h,cpp},t/*\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"t/*\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("src = [\n \"k/a.cpp\",\n \"k/a.h\",\n \"t/b.cpp\",\n]")); + assert!(!toml.contains("a.py")); +} + +#[test] +fn kernel_accepts_workspace_root_include() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" include=\".\" src=\"k/*.cu\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*.cpp\"\n", + ); + assert!( + ws.get_text("build.toml") + .unwrap() + .contains("include = [\".\"]") + ); +} + +#[test] +fn manifest_edition_stable_abi_and_multi_glob_src() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("k/b.h".into(), b"x".to_vec()), + ("k/skip.py".into(), b"x".to_vec()), + ("torch-ext/b.cpp".into(), b"x".to_vec()), + ("torch-ext/b.h".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*.cu,k/*.h\"\nmanifest name=\"k\" version=1 license=\"MIT\" edition=5 backends=\"cuda\" torch_src=\"torch-ext/*.cpp,torch-ext/*.h\" stable_abi=\"cuda=2.10,rocm=2.10\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("license = \"MIT\"\nedition = 5\nbackends = [\"cuda\"]")); + assert!(toml.contains("[torch.stable-abi]\ncuda = \"2.10\"\nrocm = \"2.10\"\n")); + assert!(toml.contains("[torch]\nsrc = [\n \"torch-ext/b.cpp\",\n \"torch-ext/b.h\",\n]")); + assert!(toml.contains("src = [\n \"k/a.cu\",\n \"k/b.h\",\n]")); + assert!(!toml.contains("skip.py")); +} + +#[test] +fn manifest_scalar_stable_abi() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/a.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*\" stable_abi_version=\"2.9\"\n", + ); + assert!( + ws.get_text("build.toml") + .unwrap() + .contains("[torch]\nstable-abi = \"2.9\"\nsrc = [") + ); +} + +#[test] +fn manifest_torch_include() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/a.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*\" torch_include=\"k\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!( + toml.contains("[torch]\ninclude = [\"k\"]\nsrc = ["), + "{toml}" + ); +} + +#[test] +fn manifest_torch_include_must_exist() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/a.cpp".into(), b"x".to_vec()), + ])); + let err = run_recipe_err( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*\" torch_include=\"nope\"\n", + ); + assert!( + err.contains("include directory \"nope\" contains no files"), + "{err}" + ); +} + +#[test] +fn manifest_torch_pyext() { + let mut ws = Workspace::from_files(BTreeMap::from([ + ("k/a.cu".into(), b"x".to_vec()), + ("torch-ext/a.cpp".into(), b"x".to_vec()), + ])); + run_recipe( + &mut ws, + "kernel name=\"k\" backend=\"cuda\" src=\"k/*\"\nmanifest name=\"k\" backends=\"cuda\" torch_src=\"torch-ext/*\" torch_pyext=\"py,cuh,hpp,h\"\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!( + toml.contains( + "[torch]\npyext = [\n \"py\",\n \"cuh\",\n \"hpp\",\n \"h\",\n]\nsrc = [" + ), + "{toml}" + ); +} + +#[test] +fn manifest_noarch_mode() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"k\" backends=\"cuda,rocm\" noarch=#true\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("backends = [\n \"cuda\",\n \"rocm\",\n]")); + assert!(toml.ends_with("[torch-noarch]\n\n[kernel]\n")); +} + +#[test] +fn manifest_upstream_field() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"einops\" version=1 license=\"MIT\" edition=5 upstream=\"https://github.com/arogozhnikov/einops.git\" backends=\"cpu,cuda\" noarch=#true\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains( + "license = \"MIT\"\nedition = 5\nupstream = \"https://github.com/arogozhnikov/einops.git\"\nbackends = [" + )); +} + +#[test] +fn manifest_general_cuda_table() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"k\" version=0 backends=\"cuda\" python_depends=\"einops,tvm-ffi\" cuda_minver=\"12.8\" cuda_python_depends=\"nvidia-cutlass-dsl\" repo_id=\"x/k\" noarch=#true\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("python-depends = [\n \"einops\",\n \"tvm-ffi\",\n]\n\n[general.cuda]\nminver = \"12.8\"\npython-depends = [\"nvidia-cutlass-dsl\"]\n\n[general.hub]\n")); +} + +#[test] +fn manifest_hub_branch() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"k\" backends=\"cuda\" repo_id=\"x/k\" hub_branch=\"ep-support\" noarch=#true\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("[general.hub]\nrepo-id = \"x/k\"\nbranch = \"ep-support\"\n")); +} + +#[test] +fn expect_guards() { + let mut ws = Workspace::from_files(BTreeMap::from([("a.py".into(), b"x = 1\n".to_vec())])); + run_recipe( + &mut ws, + "expect in=\"*.py\" files=1\nexpect in=\"a.py\" find=\"x\" count=1\nexpect in=\"a.py\" find=\"gone\" count=0\n", + ); + let parsed = recipe::parse("expect in=\"*.py\" files=2\n").unwrap(); + let op = ops::build(&parsed.ops[0], Path::new(".")).unwrap(); + assert!( + op.apply(&mut ws, &ops::Inputs::default(), &mut ops::Facts::default()) + .is_err() + ); +} + +#[test] +fn strip_suffix_is_fully_pinned() { + let original = BTreeMap::from([ + ("a.h".into(), b"a\n".to_vec()), + ("b.h".into(), b"b\n".to_vec()), + ]); + let mut ws = Workspace::from_files(original.clone()); + run_recipe(&mut ws, "strip_suffix in=\"*.h\" suffix=\"\\n\" files=2\n"); + assert_eq!(ws.get_text("a.h").unwrap(), "a"); + assert_eq!(ws.get_text("b.h").unwrap(), "b"); + + let mut ws = Workspace::from_files(original.clone()); + let parsed = recipe::parse("strip_suffix in=\"*.h\" suffix=\"\\n\" files=1\n").unwrap(); + let op = ops::build(&parsed.ops[0], Path::new(".")).unwrap(); + assert!( + op.apply(&mut ws, &ops::Inputs::default(), &mut ops::Facts::default()) + .unwrap_err() + .to_string() + .contains("expected exactly 1 file(s)") + ); + + let mut missing = original; + missing.insert("b.h".into(), b"b".to_vec()); + let mut ws = Workspace::from_files(missing); + let parsed = recipe::parse("strip_suffix in=\"*.h\" suffix=\"\\n\" files=2\n").unwrap(); + let op = ops::build(&parsed.ops[0], Path::new(".")).unwrap(); + assert!( + op.apply(&mut ws, &ops::Inputs::default(), &mut ops::Facts::default()) + .unwrap_err() + .to_string() + .contains("does not end with pinned suffix") + ); + assert_eq!(ws.get_text("a.h").unwrap(), "a\n"); +} diff --git a/kernel-port/src/workspace.rs b/kernel-port/src/workspace.rs new file mode 100644 index 00000000..1b3b2bd8 --- /dev/null +++ b/kernel-port/src/workspace.rs @@ -0,0 +1,247 @@ +use anyhow::{Context, Result, bail}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +pub struct Pattern { + raw: String, + matcher: globset::GlobMatcher, +} + +impl std::str::FromStr for Pattern { + type Err = anyhow::Error; + + fn from_str(raw: &str) -> Result { + // literal_separator keeps `*` from crossing a `/`; only `**` does. + let matcher = globset::GlobBuilder::new(raw) + .literal_separator(true) + .build() + .with_context(|| format!("invalid glob {raw:?}"))? + .compile_matcher(); + Ok(Self { + raw: raw.to_string(), + matcher, + }) + } +} + +impl std::fmt::Debug for Pattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.raw, f) + } +} + +impl std::fmt::Display for Pattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.raw) + } +} + +pub struct Workspace { + files: BTreeMap>, + // Load-time snapshot. The change set is the diff against it. + initial: BTreeMap>, +} + +pub struct ChangeSet { + pub added: Vec, + pub modified: Vec, + pub deleted: Vec, +} + +impl Workspace { + pub fn load(root: &Path) -> Result { + let mut files = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let mut entries: Vec = std::fs::read_dir(&dir) + .with_context(|| format!("reading {}", dir.display()))? + .map(|e| Ok(e?.path())) + .collect::>()?; + entries.sort(); + for path in entries { + let name = path.file_name().unwrap().to_string_lossy(); + if name == ".git" { + continue; + } + if path.is_dir() { + stack.push(path); + } else { + let rel = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + let content = std::fs::read(&path) + .with_context(|| format!("reading {}", path.display()))?; + files.insert(rel, content); + } + } + } + Ok(Self { + initial: files.clone(), + files, + }) + } + + pub fn initial_bytes(&self, path: &str) -> Option<&[u8]> { + self.initial.get(path).map(std::vec::Vec::as_slice) + } + + pub fn current_bytes(&self, path: &str) -> Option<&[u8]> { + self.files.get(path).map(std::vec::Vec::as_slice) + } + + pub fn tree_hash(&self) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + // Path and length are hashed with the content so no two trees collide + // by shifting bytes across a boundary. + for (path, content) in &self.files { + hasher.update(path.as_bytes()); + hasher.update([0]); + hasher.update((content.len() as u64).to_le_bytes()); + hasher.update(content); + } + crate::hex(&hasher.finalize()) + } + + pub fn from_files(files: BTreeMap>) -> Self { + Self { + initial: files.clone(), + files, + } + } + + pub fn paths(&self) -> Vec { + self.files.keys().cloned().collect() + } + + pub fn glob(&self, pattern: &Pattern) -> Vec { + self.files + .keys() + .filter(|p| pattern.matcher.is_match(p.as_str())) + .cloned() + .collect() + } + + pub fn glob_str(&self, pattern: &str) -> Result> { + Ok(self.glob(&pattern.parse()?)) + } + + pub fn get_text(&self, path: &str) -> Result<&str> { + let bytes = self + .files + .get(path) + .with_context(|| format!("no such file in workspace: {path}"))?; + std::str::from_utf8(bytes).with_context(|| format!("{path} is not valid UTF-8")) + } + + pub fn set_text(&mut self, path: &str, content: String) { + self.files.insert(path.to_string(), content.into_bytes()); + } + + pub fn insert(&mut self, path: &str, content: Vec) { + self.files.insert(path.to_string(), content); + } + + pub fn delete(&mut self, path: &str) -> Result<()> { + if self.files.remove(path).is_none() { + bail!("no such file in workspace: {path}"); + } + Ok(()) + } + + pub fn rename(&mut self, from: &str, to: &str) -> Result> { + let mut pairs = Vec::new(); + if self.files.contains_key(from) { + pairs.push((from.to_string(), to.to_string())); + } else { + let prefix = format!("{from}/"); + for p in self.files.keys() { + if let Some(rest) = p.strip_prefix(&prefix) { + pairs.push((p.clone(), format!("{to}/{rest}"))); + } + } + if pairs.is_empty() { + bail!("{from:?} matches no file or directory in the workspace"); + } + } + for (_, new) in &pairs { + if self.files.contains_key(new) { + bail!("destination {new:?} already exists"); + } + } + for (old, new) in &pairs { + let content = self.files.remove(old).unwrap(); + self.files.insert(new.clone(), content); + } + Ok(pairs) + } + + pub fn changes(&self) -> ChangeSet { + let mut added = Vec::new(); + let mut modified = Vec::new(); + let mut deleted = Vec::new(); + for (path, content) in &self.files { + match self.initial.get(path) { + None => added.push(path.clone()), + Some(old) if old != content => modified.push(path.clone()), + Some(_) => {} + } + } + for path in self.initial.keys() { + if !self.files.contains_key(path) { + deleted.push(path.clone()); + } + } + ChangeSet { + added, + modified, + deleted, + } + } + + // Wiped first, so the output is a function of the recipe alone and never a + // merge with a previous run. + pub fn materialize_into(&self, out: &Path) -> Result<()> { + if out.exists() { + std::fs::remove_dir_all(out) + .with_context(|| format!("clearing previous output {}", out.display()))?; + } + for (path, content) in &self.files { + let disk = out.join(path); + if let Some(parent) = disk.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&disk, content) + .with_context(|| format!("writing {}", disk.display()))?; + } + Ok(()) + } + + pub fn materialize(&self, root: &Path) -> Result<()> { + let changes = self.changes(); + for path in changes.added.iter().chain(&changes.modified) { + let disk = root.join(path); + if let Some(parent) = disk.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&disk, &self.files[path]) + .with_context(|| format!("writing {}", disk.display()))?; + } + for path in &changes.deleted { + let disk = root.join(path); + std::fs::remove_file(&disk).with_context(|| format!("removing {}", disk.display()))?; + // remove_dir only succeeds on an empty directory, which is the + // stop condition for pruning what the delete emptied. + let mut dir = disk.parent().map(Path::to_path_buf); + while let Some(d) = dir { + if d == root || std::fs::remove_dir(&d).is_err() { + break; + } + dir = d.parent().map(Path::to_path_buf); + } + } + Ok(()) + } +} diff --git a/nix-builder/lib/cache.nix b/nix-builder/lib/cache.nix index a6318ed8..d0dab478 100644 --- a/nix-builder/lib/cache.nix +++ b/nix-builder/lib/cache.nix @@ -25,6 +25,7 @@ allOutputs buildSet.torch ++ lib.concatMap allOutputs buildSet.extension.extraBuildDeps ++ allOutputs kernel-builder + ++ allOutputs kernel-port ++ allOutputs python3.pkgs.einops ++ allOutputs python3.pkgs.jax ++ allOutputs python3.pkgs.jax-tvm-ffi diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index 4d114aae..c86db677 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -10,6 +10,8 @@ final: prev: # Local packages/hooks. kernel-builder = final.callPackage ./pkgs/kernel-builder { inherit builderProvenance; }; + kernel-port = final.callPackage ./pkgs/kernel-port { }; + cmakeNvccThreadsHook = final.callPackage ./pkgs/cmake-nvcc-threads-hook { }; fetchFromHuggingFace = final.callPackage ./pkgs/fetch-from-huggingface { }; diff --git a/nix-builder/pkgs/crate-dirs.nix b/nix-builder/pkgs/crate-dirs.nix index 22697a7f..7b99900b 100644 --- a/nix-builder/pkgs/crate-dirs.nix +++ b/nix-builder/pkgs/crate-dirs.nix @@ -8,6 +8,7 @@ toSource { ../../Cargo.toml (fileFilter sourceFiles ../../kernel-abi-check) (fileFilter sourceFiles ../../kernel-builder) + (fileFilter sourceFiles ../../kernel-port) (fileFilter sourceFiles ../../kernels-data) ]; } diff --git a/nix-builder/pkgs/kernel-port/default.nix b/nix-builder/pkgs/kernel-port/default.nix new file mode 100644 index 00000000..0072c04f --- /dev/null +++ b/nix-builder/pkgs/kernel-port/default.nix @@ -0,0 +1,65 @@ +{ + lib, + rustPlatform, +}: + +let + version = (builtins.fromTOML (builtins.readFile ../../../kernel-port/Cargo.toml)).package.version; + cargoFlags = [ + "-p" + "kernel-port" + ]; +in +rustPlatform.buildRustPackage { + inherit version; + pname = "kernel-port"; + + src = + let + sourceFiles = + file: + file.name == "Cargo.toml" + || file.name == "Cargo.lock" + || file.name == "flake.nix" + || file.name == "manylinux-policy.json" + || file.name == "pyproject.toml" + || file.name == "pyproject_universal.toml" + || file.name == "python_dependencies.json" + || file.name == "shim_function_versions.txt" + || file.name == "stable_abi.toml" + || file.name == ".gitattributes" + || file.name == ".gitignore" + || (builtins.any file.hasExt [ + "cmake" + "cpp" + "cu" + "h" + "in" + "kdl" + "md" + "metal" + "mm" + "py" + "rs" + "toml" + ]); + in + import ../crate-dirs.nix { + inherit lib sourceFiles; + }; + + cargoLock = { + lockFile = ../../../Cargo.lock; + outputHashes = { + "hf-hub-1.1.0" = "sha256-wClUTCmphrO4QM+IYwYrNxyvDp8qBGAPdP+Wca8TgRA="; + }; + }; + + cargoBuildFlags = cargoFlags; + cargoTestFlags = cargoFlags; + + meta = { + description = "Port third-party kernels to the Hugging Face Kernels layout"; + mainProgram = "kernel-port"; + }; +}