From e60ae775d0d548e14e627daa86d29cfaf85509ce Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Mon, 27 Apr 2026 21:25:51 +0200 Subject: [PATCH 01/17] feat: replace unmaintained hubcaps with octocrab --- Cargo.lock | 1087 +++++++++++---------- Cargo.toml | 1 - flake.nix | 1 - ofborg/Cargo.toml | 15 +- ofborg/src/bin/build-faker.rs | 3 +- ofborg/src/bin/evaluation-filter.rs | 4 +- ofborg/src/bin/github-comment-filter.rs | 4 +- ofborg/src/bin/github-comment-poster.rs | 4 +- ofborg/src/bin/github-webhook-receiver.rs | 5 +- ofborg/src/bin/log-message-collector.rs | 4 +- ofborg/src/checkout.rs | 4 +- ofborg/src/clone.rs | 3 +- ofborg/src/commitstatus.rs | 69 +- ofborg/src/config.rs | 143 +-- ofborg/src/easylapin.rs | 20 +- ofborg/src/evalchecker.rs | 4 +- ofborg/src/files.rs | 2 +- ofborg/src/github.rs | 100 ++ ofborg/src/lib.rs | 4 +- ofborg/src/locks.rs | 4 +- ofborg/src/maintainers.rs | 8 +- ofborg/src/message/buildresult.rs | 4 +- ofborg/src/nix.rs | 14 +- ofborg/src/tasks/build.rs | 208 ++-- ofborg/src/tasks/eval/mod.rs | 4 +- ofborg/src/tasks/eval/nixpkgs.rs | 53 +- ofborg/src/tasks/evaluate.rs | 316 +++--- ofborg/src/tasks/evaluationfilter.rs | 122 +-- ofborg/src/tasks/githubcommentfilter.rs | 241 +++-- ofborg/src/tasks/githubcommentposter.rs | 878 +++++++---------- ofborg/src/tasks/log_message_collector.rs | 10 +- ofborg/src/tasks/statscollector.rs | 4 +- ofborg/src/worker.rs | 6 +- ofborg/src/writetoline.rs | 2 +- 34 files changed, 1629 insertions(+), 1722 deletions(-) create mode 100644 ofborg/src/github.rs diff --git a/Cargo.lock b/Cargo.lock index 3429dcab..6b069e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "asn1-rs" version = "0.7.1" @@ -99,7 +108,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -268,16 +277,10 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" +name = "base16ct" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -355,6 +358,39 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cbc" version = "0.1.2" @@ -366,9 +402,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -376,24 +412,12 @@ dependencies = [ "shlex", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "chacha20" version = "0.10.0" @@ -412,7 +436,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "serde", + "wasm-bindgen", "windows-link", ] @@ -564,6 +591,18 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -592,11 +631,38 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der" @@ -661,6 +727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -694,12 +761,71 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -743,6 +869,22 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -763,7 +905,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -923,6 +1065,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -945,11 +1088,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -966,6 +1107,17 @@ dependencies = [ "wasip3", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.4.13" @@ -977,7 +1129,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http", "indexmap", "slab", "tokio", @@ -1027,9 +1179,9 @@ dependencies = [ "hickory-proto", "idna", "ipnet", - "jni 0.22.4", + "jni", "rand 0.10.1", - "thiserror 2.0.18", + "thiserror", "tinyvec", "tokio", "tracing", @@ -1045,12 +1197,12 @@ dependencies = [ "data-encoding", "idna", "ipnet", - "jni 0.22.4", + "jni", "once_cell", "prefix-trie", "rand 0.10.1", - "ring 0.17.14", - "thiserror 2.0.18", + "ring", + "thiserror", "tinyvec", "tracing", "url", @@ -1068,7 +1220,7 @@ dependencies = [ "hickory-proto", "ipconfig", "ipnet", - "jni 0.22.4", + "jni", "moka", "ndk-context", "once_cell", @@ -1077,11 +1229,20 @@ dependencies = [ "resolv-conf", "smallvec", "system-configuration", - "thiserror 2.0.18", + "thiserror", "tokio", "tracing", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1100,17 +1261,6 @@ dependencies = [ "digest 0.11.2", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.4.0" @@ -1128,7 +1278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http", ] [[package]] @@ -1139,7 +1289,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http", "http-body", "pin-project-lite", ] @@ -1156,32 +1306,11 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hubcaps" -version = "0.6.2" -source = "git+https://github.com/ofborg/hubcaps.git?rev=0d7466ef941a7a8e160c071e2846e56b90b6ea86#0d7466ef941a7a8e160c071e2846e56b90b6ea86" -dependencies = [ - "base64 0.22.1", - "data-encoding", - "futures", - "http 1.4.0", - "hyperx", - "jsonwebtoken", - "log", - "mime", - "percent-encoding", - "reqwest", - "serde", - "serde_derive", - "serde_json", - "url", -] - [[package]] name = "hybrid-array" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" dependencies = [ "typenum", ] @@ -1197,7 +1326,7 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http 1.4.0", + "http", "http-body", "httparse", "httpdate", @@ -1214,31 +1343,43 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", + "http", "hyper", "hyper-util", + "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", "bytes", "futures-channel", "futures-util", - "http 1.4.0", + "http", "http-body", "hyper", - "ipnet", "libc", - "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1246,21 +1387,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hyperx" -version = "1.4.0" -source = "git+https://github.com/chantra/hyperx.git?branch=semver#69f17cf858573db42c2baaf0bfead54521de32f9" -dependencies = [ - "base64 0.13.1", - "bytes", - "http 0.2.12", - "httpdate", - "language-tags", - "mime", - "percent-encoding", - "unicase", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1386,9 +1512,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1454,22 +1580,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jni" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" -dependencies = [ - "cesu8", - "cfg-if", - "combine", - "jni-sys 0.3.1", - "log", - "thiserror 1.0.69", - "walkdir", - "windows-sys 0.45.0", -] - [[package]] name = "jni" version = "0.22.4" @@ -1479,10 +1589,10 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys 0.4.1", + "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror", "walkdir", "windows-link", ] @@ -1500,15 +1610,6 @@ dependencies = [ "syn", ] -[[package]] -name = "jni-sys" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" -dependencies = [ - "jni-sys 0.4.1", -] - [[package]] name = "jni-sys" version = "0.4.1" @@ -1544,37 +1645,38 @@ version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ - "cfg-if", - "futures-util", "once_cell", "wasm-bindgen", ] [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ - "base64 0.21.7", + "base64", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac 0.12.1", + "js-sys", + "p256", + "p384", "pem", - "ring 0.16.20", + "rand 0.8.6", + "rsa", "serde", "serde_json", + "sha2 0.10.9", + "signature", "simple_asn1", ] -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - [[package]] name = "lapin" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46ebc501bdffc4d35133680f1ee6d620c131bba216c49704d850f18a6a6495e" +checksum = "53fe00373c0142f6122cb2efb41bc88f584f7e109f42569a0a87b5d061d4e436" dependencies = [ "amq-protocol", "async-rs", @@ -1592,6 +1694,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leb128fmt" @@ -1601,9 +1706,15 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linked-hash-map" @@ -1647,12 +1758,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "matchers" version = "0.2.0" @@ -1758,6 +1863,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.1" @@ -1773,6 +1894,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1780,31 +1912,74 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] -name = "ofborg" -version = "0.1.9" +name = "octocrab" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2ad8abffe4e2b05f9cdc7e061de63d305a6dca0af81ca1064a7d98e0b78267" dependencies = [ + "arc-swap", "async-trait", - "brace-expand", + "base64", + "bytes", + "cargo_metadata", + "cfg-if", "chrono", - "either", - "fs2", "futures", "futures-util", - "hex", - "hmac 0.13.0", - "http 1.4.0", + "getrandom 0.2.17", + "http", + "http-body", "http-body-util", - "hubcaps", "hyper", + "hyper-rustls", + "hyper-timeout", "hyper-util", - "lapin", - "lru-cache", - "md5", - "mime", - "nom 8.0.0", + "jsonwebtoken", + "once_cell", + "percent-encoding", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "snafu", + "tokio", + "tower", + "tower-http", + "tracing", + "url", + "web-time", +] + +[[package]] +name = "ofborg" +version = "0.1.9" +dependencies = [ + "async-trait", + "brace-expand", + "chrono", + "either", + "fs2", + "futures", + "futures-util", + "hex", + "hmac 0.13.0", + "http", + "http-body-util", + "hyper", + "hyper-util", + "jsonwebtoken", + "lapin", + "lru-cache", + "md5", + "mime", + "nom 8.0.0", + "octocrab", "parking_lot", "regex", "rustls-pki-types", @@ -1858,7 +2033,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb9bf5222606eb712d3bb30e01bc9420545b00859970897e70c682353a034f2" dependencies = [ - "base64 0.22.1", + "base64", "cbc", "cms", "der", @@ -1871,10 +2046,34 @@ dependencies = [ "rc2", "sha1", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror", "x509-parser", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + [[package]] name = "parking" version = "2.2.1" @@ -1916,11 +2115,12 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64", + "serde_core", ] [[package]] @@ -1938,6 +2138,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1955,6 +2175,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs12" version = "0.1.0" @@ -1985,6 +2216,16 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -2037,68 +2278,21 @@ dependencies = [ ] [[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quinn" -version = "0.11.9" +name = "primeorder" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.18", - "tokio", - "tracing", - "web-time", + "elliptic-curve", ] [[package]] -name = "quinn-proto" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.4", - "ring 0.17.14", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" +name = "proc-macro2" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", + "unicode-ident", ] [[package]] @@ -2124,12 +2318,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ + "libc", "rand_chacha", - "rand_core 0.9.5", + "rand_core 0.6.4", ] [[package]] @@ -2145,21 +2340,21 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.9.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.9.5", + "rand_core 0.6.4", ] [[package]] name = "rand_core" -version = "0.9.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.2.17", ] [[package]] @@ -2215,41 +2410,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "reqwest" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-core", - "http 1.4.0", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier 0.6.2", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "resolv-conf" version = "0.7.6" @@ -2257,18 +2417,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] -name = "ring" -version = "0.16.20" +name = "rfc6979" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -2281,15 +2436,29 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] [[package]] -name = "rustc-hash" -version = "2.1.2" +name = "rsa" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] [[package]] name = "rustc_version" @@ -2324,12 +2493,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2347,7 +2518,7 @@ dependencies = [ "log", "rustls", "rustls-pki-types", - "rustls-platform-verifier 0.7.0", + "rustls-platform-verifier", "rustls-webpki", ] @@ -2365,35 +2536,13 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "web-time", "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni 0.21.1", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -2402,7 +2551,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni 0.22.4", + "jni", "log", "once_cell", "rustls", @@ -2423,14 +2572,14 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -2439,6 +2588,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "salsa20" version = "0.10.2" @@ -2483,6 +2638,29 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2511,6 +2689,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -2555,6 +2737,29 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2603,6 +2808,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd_cesu8" version = "1.1.1" @@ -2627,7 +2842,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -2643,6 +2858,27 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "socket2" version = "0.6.3" @@ -2653,12 +2889,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -2706,9 +2936,6 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] [[package]] name = "synstructure" @@ -2774,33 +3001,13 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -2950,8 +3157,10 @@ dependencies = [ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2963,13 +3172,14 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.0", + "http", "http-body", "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2990,6 +3200,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3070,12 +3281,6 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -3088,12 +3293,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -3110,6 +3309,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3197,16 +3397,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.118" @@ -3273,16 +3463,6 @@ dependencies = [ "semver", ] -[[package]] -name = "web-sys" -version = "0.3.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "web-time" version = "1.1.0" @@ -3290,6 +3470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", + "serde", "wasm-bindgen", ] @@ -3409,31 +3590,13 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -3445,192 +3608,70 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -3755,7 +3796,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] diff --git a/Cargo.toml b/Cargo.toml index a1688cec..d1db03ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,5 +9,4 @@ resolver = "2" debug = true [patch.crates-io] -#hubcaps = { path = "../hubcaps" } #amq-proto = { path = "rust-amq-proto" } diff --git a/flake.nix b/flake.nix index 1a9f2b26..9c9263de 100644 --- a/flake.nix +++ b/flake.nix @@ -108,7 +108,6 @@ cargoLock = { lockFile = ./Cargo.lock; outputHashes = { - "hubcaps-0.6.2" = "sha256-Vl4wQIKQVRxkpQxL8fL9rndAN3TKLV4OjgnZOpT6HRo="; "hyperx-1.4.0" = "sha256-MW/KxxMYvj/DYVKrYa7rDKwrH6s8uQOCA0dR2W7GBeg="; }; }; diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index 04c6ad34..94f93e47 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -20,9 +20,18 @@ hex = "0.4.3" hmac = "0.13.0" http = "1" http-body-util = "0.1" -#hubcaps = "0.6" -# for Conclusion::Skipped which is in master -hubcaps = { git = "https://github.com/ofborg/hubcaps.git", rev = "0d7466ef941a7a8e160c071e2846e56b90b6ea86" } +octocrab = { version = "0.51.0", default-features = false, features = [ + "default-client", + "follow-redirect", + "jwt-rust-crypto", + "retry", + "rustls", + # Enabling `rustls-ring` leads to runtime panics: https://github.com/XAMPPRocky/octocrab/issues/855 + # "rustls-ring", + "timeout", + "tracing", +] } +jsonwebtoken = "10" hyper = { version = "1.0", features = ["full", "server", "http1"] } hyper-util = { version = "0.1", features = ["server", "tokio", "http1"] } lapin = "4.3.0" diff --git a/ofborg/src/bin/build-faker.rs b/ofborg/src/bin/build-faker.rs index e6f1cd25..8e942e3d 100644 --- a/ofborg/src/bin/build-faker.rs +++ b/ofborg/src/bin/build-faker.rs @@ -1,7 +1,8 @@ -use lapin::message::Delivery; use std::env; use std::error::Error; +use lapin::message::Delivery; + use ofborg::commentparser; use ofborg::config; use ofborg::easylapin; diff --git a/ofborg/src/bin/evaluation-filter.rs b/ofborg/src/bin/evaluation-filter.rs index eda925f9..f35d6001 100644 --- a/ofborg/src/bin/evaluation-filter.rs +++ b/ofborg/src/bin/evaluation-filter.rs @@ -3,10 +3,8 @@ use std::error::Error; use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] async fn main() -> Result<(), Box> { diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index 4d1427e0..cc745529 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -4,10 +4,8 @@ use std::error::Error; use ofborg::systems::System; use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] async fn main() -> Result<(), Box> { diff --git a/ofborg/src/bin/github-comment-poster.rs b/ofborg/src/bin/github-comment-poster.rs index 3b26b83c..9c19d65d 100644 --- a/ofborg/src/bin/github-comment-poster.rs +++ b/ofborg/src/bin/github-comment-poster.rs @@ -3,10 +3,8 @@ use std::error::Error; use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] async fn main() -> Result<(), Box> { diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index f31ac793..cff4e515 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -13,13 +13,14 @@ use hyper::{Request, Response}; use hyper_util::rt::TokioIo; use lapin::options::BasicPublishOptions; use lapin::{BasicProperties, Channel}; -use ofborg::ghevent::GenericWebhook; -use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; use sha2::Sha256; use tokio::net::TcpListener; use tokio::sync::Mutex; use tracing::{error, info, warn}; +use ofborg::ghevent::GenericWebhook; +use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; + /// Prepares the the exchange we will write to, the queues that are bound to it /// and binds them. async fn setup_amqp(chan: &mut Channel) -> Result<(), Box> { diff --git a/ofborg/src/bin/log-message-collector.rs b/ofborg/src/bin/log-message-collector.rs index 3ad668cb..1a284345 100644 --- a/ofborg/src/bin/log-message-collector.rs +++ b/ofborg/src/bin/log-message-collector.rs @@ -4,10 +4,8 @@ use std::path::PathBuf; use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] async fn main() -> Result<(), Box> { diff --git a/ofborg/src/checkout.rs b/ofborg/src/checkout.rs index 731e68d3..2c489da8 100644 --- a/ofborg/src/checkout.rs +++ b/ofborg/src/checkout.rs @@ -1,5 +1,3 @@ -use crate::clone::{self, GitClonable}; - use std::ffi::{OsStr, OsString}; use std::fs; use std::io::Error; @@ -8,6 +6,8 @@ use std::process::{Command, Stdio}; use tracing::info; +use crate::clone::{self, GitClonable}; + pub struct CachedCloner { root: PathBuf, } diff --git a/ofborg/src/clone.rs b/ofborg/src/clone.rs index 0dcb71c2..c678b0a5 100644 --- a/ofborg/src/clone.rs +++ b/ofborg/src/clone.rs @@ -1,11 +1,10 @@ -use fs2::FileExt; - use std::ffi::OsStr; use std::fs; use std::io::Error; use std::path::PathBuf; use std::process::{Command, Stdio}; +use fs2::FileExt as _; use tracing::{debug, info, warn}; pub struct Lock { diff --git a/ofborg/src/commitstatus.rs b/ofborg/src/commitstatus.rs index 6747f3b0..751ccb24 100644 --- a/ofborg/src/commitstatus.rs +++ b/ofborg/src/commitstatus.rs @@ -1,8 +1,10 @@ -use futures_util::future::TryFutureExt; +use octocrab::{self, models::StatusState}; use tracing::warn; +use crate::github::GithubRepo; + pub struct CommitStatus { - api: hubcaps::statuses::Statuses, + repo: GithubRepo, sha: String, context: String, description: String, @@ -11,23 +13,19 @@ pub struct CommitStatus { impl CommitStatus { pub fn new( - api: hubcaps::statuses::Statuses, + repo: GithubRepo, sha: String, context: String, description: String, url: Option, ) -> CommitStatus { - let mut stat = CommitStatus { - api, + CommitStatus { + repo, sha, context, description, - url: "".to_owned(), - }; - - stat.set_url(url); - - stat + url: url.unwrap_or_else(|| String::from("")), + } } pub fn set_url(&mut self, url: Option) { @@ -37,7 +35,7 @@ impl CommitStatus { pub async fn set_with_description( &mut self, description: &str, - state: hubcaps::statuses::State, + state: StatusState, ) -> Result<(), CommitStatusError> { self.set_description(description.to_owned()); self.set(state).await @@ -47,7 +45,7 @@ impl CommitStatus { self.description = description; } - pub async fn set(&self, state: hubcaps::statuses::State) -> Result<(), CommitStatusError> { + pub async fn set(&self, state: StatusState) -> Result<(), CommitStatusError> { let desc = if self.description.len() >= 140 { warn!( "description is over 140 char; truncating: {:?}", @@ -57,47 +55,28 @@ impl CommitStatus { } else { self.description.clone() }; - self.api - .create( - self.sha.as_ref(), - &hubcaps::statuses::StatusOptions::builder(state) - .context(self.context.clone()) - .description(desc) - .target_url(self.url.clone()) - .build(), - ) - .map_ok(|_| ()) - .map_err(|e| CommitStatusError::from(e)) + + self.repo + .repos() + .create_status(self.sha.clone(), state) + .context(self.context.clone()) + .description(desc) + .target(self.url.clone()) + .send() .await?; + Ok(()) } } #[derive(Debug)] pub enum CommitStatusError { - ExpiredCreds(hubcaps::Error), - MissingSha(hubcaps::Error), - Error(hubcaps::Error), + OctocrabError(octocrab::Error), InternalError(String), } -impl From for CommitStatusError { - fn from(e: hubcaps::Error) -> CommitStatusError { - use http::status::StatusCode; - use hubcaps::Error; - match &e { - Error::Fault { code, error } - if code == &StatusCode::UNAUTHORIZED && error.message == "Bad credentials" => - { - CommitStatusError::ExpiredCreds(e) - } - Error::Fault { code, error } - if code == &StatusCode::UNPROCESSABLE_ENTITY - && error.message.starts_with("No commit found for SHA:") => - { - CommitStatusError::MissingSha(e) - } - _otherwise => CommitStatusError::Error(e), - } +impl From for CommitStatusError { + fn from(e: octocrab::Error) -> CommitStatusError { + CommitStatusError::OctocrabError(e) } } diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 542eeedd..a5f33a5d 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -1,18 +1,18 @@ -use crate::acl; -use crate::nix::Nix; - -use std::collections::{HashMap, hash_map::Entry}; +use std::collections::HashMap; use std::fmt; use std::fs::File; -use std::io::Read; +use std::io::Read as _; use std::marker::PhantomData; use std::path::{Path, PathBuf}; -use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials}; -use rustls_pki_types::pem::PemObject as _; +use octocrab::models::InstallationId; +use octocrab::{Octocrab, auth::AppAuth}; use serde::de::{self, Deserializer}; use tracing::{debug, error, info, warn}; +use crate::acl; +use crate::nix::Nix; + /// Main ofBorg configuration #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct Config { @@ -25,7 +25,7 @@ pub struct Config { /// Configuration for the GitHub comment filter pub github_comment_filter: Option, /// Configuration for the GitHub comment poster - pub github_comment_poster: Option, + pub github_comment_poster: Option, /// Configuration for the mass rebuilder pub mass_rebuilder: Option, /// Configuration for the builder @@ -91,7 +91,7 @@ pub struct GithubCommentFilter { /// Configuration for the GitHub comment poster #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] -pub struct GithubCommentPoster { +pub struct GitHubCommentPoster { /// RabbitMQ broker to connect to pub rabbitmq: RabbitMqConfig, } @@ -169,6 +169,17 @@ pub struct GithubAppConfig { pub oauth_client_secret_file: PathBuf, } +impl GithubAppConfig { + fn app_auth(&self) -> AppAuth { + let pem = std::fs::read_to_string(&self.private_key).expect("Unable to read private key"); + AppAuth { + app_id: self.app_id.into(), + key: jsonwebtoken::EncodingKey::from_rsa_pem(pem.as_bytes()) + .expect("Invalid private key"), + } + } +} + const fn default_instance() -> u8 { 1 } @@ -226,26 +237,16 @@ impl Config { acl::Acl::new(repos, trusted_users) } - pub fn github(&self) -> Github { - let token = std::fs::read_to_string( - self.github_app - .clone() - .expect("No GitHub app configured") - .oauth_client_secret_file, - ) - .expect("Couldn't read from GitHub app token"); - let token = token.trim(); - Github::new( - "github.com/NixOS/ofborg", - Credentials::Client( - self.github_app - .clone() - .expect("No GitHub app configured") - .oauth_client_id, - token.to_owned(), - ), - ) - .expect("Unable to create a github client instance") + pub fn github(&self) -> Octocrab { + let app_auth = self + .github_app + .as_ref() + .map(|app| app.app_auth()) + .expect("No GitHub app configured"); + Octocrab::builder() + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create a github client instance") } pub fn github_app_vendingmachine(&self) -> GithubAppVendingMachine { @@ -307,8 +308,8 @@ pub fn load(filename: &Path) -> Config { pub struct GithubAppVendingMachine { conf: GithubAppConfig, - id_cache: HashMap<(String, String), Option>, - client_cache: HashMap, + id_cache: HashMap<(String, String), Option>, + client_cache: HashMap, } impl GithubAppVendingMachine { @@ -316,54 +317,56 @@ impl GithubAppVendingMachine { "github.com/NixOS/ofborg (app)" } - fn jwt(&self) -> JWTCredentials { - let pem = rustls_pki_types::PrivatePkcs1KeyDer::from_pem_file(&self.conf.private_key) - .expect("Unable to read private key"); - let private_key_der = pem.secret_pkcs1_der().to_vec(); - JWTCredentials::new(self.conf.app_id, private_key_der) - .expect("Unable to create JWTCredentials") - } + async fn install_id_for_repo(&mut self, owner: &str, repo: &str) -> Option { + let key = (owner.to_owned(), repo.to_owned()); - async fn install_id_for_repo(&mut self, owner: &str, repo: &str) -> Option { - let useragent = self.useragent(); - let jwt = self.jwt(); + if let Some(Some(id)) = self.id_cache.get(&key) { + return Some(*id); + } - let key = (owner.to_owned(), repo.to_owned()); + info!("Looking up install ID for {}/{}", owner, repo); + + let app_auth = self.conf.app_auth(); + let octocrab = Octocrab::builder() + .add_header(http::header::USER_AGENT, self.useragent().parse().unwrap()) + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create app client"); - match self.id_cache.entry(key) { - Entry::Occupied(entry) => *entry.get(), - Entry::Vacant(entry) => { - info!("Looking up install ID for {}/{}", owner, repo); - - let lookup_gh = Github::new(useragent, Credentials::JWT(jwt)).unwrap(); - - let v = match lookup_gh.app().find_repo_installation(owner, repo).await { - Ok(install_id) => { - debug!("Received install ID {:?}", install_id); - Some(install_id.id) - } - Err(e) => { - warn!("Error during install ID lookup: {:?}", e); - None - } - }; - *entry.insert(v) + match octocrab + .apps() + .get_repository_installation(owner, repo) + .await + { + Ok(installation) => { + debug!("Received install ID {:?}", installation.id); + let id = installation.id; + self.id_cache.insert(key, Some(id)); + Some(id) + } + Err(e) => { + warn!("Error during install ID lookup: {:?}", e); + None } } } - pub async fn for_repo<'a>(&'a mut self, owner: &str, repo: &str) -> Option<&'a Github> { - let useragent = self.useragent(); - let jwt = self.jwt(); + pub async fn for_repo<'a>(&'a mut self, owner: &str, repo: &str) -> Option<&'a Octocrab> { let install_id = self.install_id_for_repo(owner, repo).await?; - Some(self.client_cache.entry(install_id).or_insert_with(|| { - Github::new( - useragent, - Credentials::InstallationToken(InstallationTokenGenerator::new(install_id, jwt)), - ) - .expect("Unable to create a github client instance") - })) + if !self.client_cache.contains_key(&install_id) { + let app_auth = self.conf.app_auth(); + let client = Octocrab::builder() + .add_header(http::header::USER_AGENT, self.useragent().parse().unwrap()) + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create app client") + .installation(install_id) + .expect("Unable to create installation client"); + self.client_cache.insert(install_id, client); + } + + self.client_cache.get(&install_id) } } diff --git a/ofborg/src/easylapin.rs b/ofborg/src/easylapin.rs index 08b7f7a1..549b5abd 100644 --- a/ofborg/src/easylapin.rs +++ b/ofborg/src/easylapin.rs @@ -1,15 +1,6 @@ use std::pin::Pin; use std::sync::Arc; -use crate::config::RabbitMqConfig; -use crate::easyamqp::{ - BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, - QueueConfig, -}; -use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; -use crate::ofborg; -use crate::worker::{Action, SimpleWorker}; - use lapin::message::Delivery; use lapin::options::{ BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicPublishOptions, BasicQosOptions, @@ -17,9 +8,18 @@ use lapin::options::{ }; use lapin::types::FieldTable; use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind}; -use tokio_stream::StreamExt; +use tokio_stream::StreamExt as _; use tracing::{debug, trace}; +use crate::config::RabbitMqConfig; +use crate::easyamqp::{ + BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, + QueueConfig, +}; +use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; +use crate::ofborg; +use crate::worker::{Action, SimpleWorker}; + pub async fn from_config(cfg: &RabbitMqConfig) -> Result { let opts = ConnectionProperties::default() .with_client_property("ofborg_version".into(), ofborg::VERSION.into()); diff --git a/ofborg/src/evalchecker.rs b/ofborg/src/evalchecker.rs index d7af769b..7a56ae64 100644 --- a/ofborg/src/evalchecker.rs +++ b/ofborg/src/evalchecker.rs @@ -1,8 +1,8 @@ -use crate::nix; - use std::fs::File; use std::path::Path; +use crate::nix; + pub struct EvalChecker { name: String, op: nix::Operation, diff --git a/ofborg/src/files.rs b/ofborg/src/files.rs index 9e329d83..ee5a4207 100644 --- a/ofborg/src/files.rs +++ b/ofborg/src/files.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::Read; +use std::io::Read as _; pub fn file_to_str(f: &mut File) -> String { let mut buffer = Vec::new(); diff --git a/ofborg/src/github.rs b/ofborg/src/github.rs new file mode 100644 index 00000000..095a4cbb --- /dev/null +++ b/ofborg/src/github.rs @@ -0,0 +1,100 @@ +use octocrab::Octocrab; +use tracing::info; + +use crate::commitstatus::CommitStatusError; + +#[derive(Clone)] +pub struct GithubRepo { + octocrab: Octocrab, + owner: String, + repo: String, +} + +impl GithubRepo { + pub fn new(octocrab: Octocrab, owner: impl Into, repo: impl Into) -> Self { + Self { + octocrab, + owner: owner.into(), + repo: repo.into(), + } + } + + pub fn owner(&self) -> &str { + &self.owner + } + + pub fn repo(&self) -> &str { + &self.repo + } + + pub fn repos(&self) -> octocrab::repos::RepoHandler<'_> { + self.octocrab.repos(&self.owner, &self.repo) + } + + pub fn issues(&self) -> octocrab::issues::IssueHandler<'_> { + self.octocrab.issues(&self.owner, &self.repo) + } + + pub fn checks(&self) -> octocrab::checks::ChecksHandler<'_> { + self.octocrab.checks(&self.owner, &self.repo) + } + + pub fn pulls(&self) -> octocrab::pulls::PullRequestHandler<'_> { + self.octocrab.pulls(&self.owner, &self.repo) + } + + pub async fn update_labels( + &self, + issue_number: u64, + add: &[String], + remove: &[String], + ) -> Result<(), CommitStatusError> { + let issue = self.issues().get(issue_number).await?; + + let existing: Vec = issue.labels.iter().map(|l| l.name.clone()).collect(); + + let to_add: Vec = add + .iter() + .filter(|l| !existing.contains(l)) + .cloned() + .collect(); + + let to_remove: Vec = remove + .iter() + .filter(|l| existing.contains(l)) + .cloned() + .collect(); + + info!("Labeling issue #{issue_number}: + {to_add:?} , - {to_remove:?}, = {existing:?}"); + + if !to_add.is_empty() { + self.issues().add_labels(issue_number, &to_add).await?; + } + + for label in to_remove { + self.issues().remove_label(issue_number, &label).await?; + } + + Ok(()) + } + + pub async fn get_prefix(&self, sha: &str) -> Result<&'static str, CommitStatusError> { + let mut page: octocrab::Page = + self.repos().list_statuses(sha.to_string()).send().await?; + + loop { + if page.items.iter().any(|s| { + s.context + .as_ref() + .is_some_and(|c| c.starts_with("grahamcofborg-")) + }) { + return Ok("grahamcofborg"); + } + + match self.octocrab.get_page(&page.next).await? { + Some(next_page) => page = next_page, + None => return Ok("ofborg"), + } + } + } +} diff --git a/ofborg/src/lib.rs b/ofborg/src/lib.rs index eb5da459..85bb0053 100644 --- a/ofborg/src/lib.rs +++ b/ofborg/src/lib.rs @@ -7,8 +7,7 @@ use std::env; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::prelude::*; +use tracing_subscriber::{EnvFilter, prelude::*}; pub mod acl; pub mod asynccmd; @@ -22,6 +21,7 @@ pub mod easylapin; pub mod evalchecker; pub mod files; pub mod ghevent; +pub mod github; pub mod locks; pub mod maintainers; pub mod message; diff --git a/ofborg/src/locks.rs b/ofborg/src/locks.rs index d1d2ee47..7d99b069 100644 --- a/ofborg/src/locks.rs +++ b/ofborg/src/locks.rs @@ -1,9 +1,9 @@ -use fs2::FileExt; - use std::fs; use std::io::Error; use std::path::PathBuf; +use fs2::FileExt; + pub trait Lockable { fn lock_path(&self) -> PathBuf; diff --git a/ofborg/src/maintainers.rs b/ofborg/src/maintainers.rs index ff1bec0c..33cfe871 100644 --- a/ofborg/src/maintainers.rs +++ b/ofborg/src/maintainers.rs @@ -1,11 +1,11 @@ -use crate::nix::Nix; - -use tempfile::NamedTempFile; - use std::collections::{HashMap, HashSet}; use std::io::Write; use std::path::Path; +use tempfile::NamedTempFile; + +use crate::nix::Nix; + #[derive(serde::Deserialize, Debug, Eq, PartialEq)] pub struct ImpactedMaintainers(HashMap>); pub struct MaintainersByPackage(pub HashMap>); diff --git a/ofborg/src/message/buildresult.rs b/ofborg/src/message/buildresult.rs index 70d8d1ea..017780fb 100644 --- a/ofborg/src/message/buildresult.rs +++ b/ofborg/src/message/buildresult.rs @@ -1,6 +1,6 @@ -use crate::message::{Pr, Repo}; +use octocrab::models::workflows::Conclusion; -use hubcaps::checks::Conclusion; +use crate::message::{Pr, Repo}; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub enum BuildStatus { diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 77aece6d..31350520 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -1,18 +1,16 @@ -use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd}; -use crate::message::buildresult::BuildStatus; -use crate::ofborg::partition_result; - use std::collections::HashMap; -use std::env; use std::ffi::OsStr; -use std::fmt; -use std::fs; -use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::io::{BufRead as _, BufReader, Seek as _, SeekFrom}; use std::path::Path; use std::process::{Command, Stdio}; +use std::{env, fmt, fs}; use tempfile::tempfile; +use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd}; +use crate::message::buildresult::BuildStatus; +use crate::ofborg::partition_result; + #[allow(clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum File { diff --git a/ofborg/src/tasks/build.rs b/ofborg/src/tasks/build.rs index 9ea59de3..e7c01ded 100644 --- a/ofborg/src/tasks/build.rs +++ b/ofborg/src/tasks/build.rs @@ -1,3 +1,10 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tracing::{Instrument, debug, debug_span, error, info}; +use uuid::Uuid; + use crate::checkout; use crate::commentparser; use crate::message::buildresult::{BuildResult, BuildStatus, V1Tag}; @@ -6,13 +13,6 @@ use crate::nix; use crate::notifyworker; use crate::worker; -use std::collections::VecDeque; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; - -use tracing::{debug, debug_span, error, info}; -use uuid::Uuid; - pub struct BuildWorker { cloner: checkout::CachedCloner, nix: nix::Nix, @@ -303,114 +303,116 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { >, ) { let span = debug_span!("job", pr = ?job.pr.number); - let _enter = span.enter(); - - let actions = self.actions(job, notifier); - - if actions.job.attrs.is_empty() { - debug!("No attrs to build"); - actions.nothing_to_do().await; - return; - } + async { + let actions = self.actions(job, notifier); - info!( - "Working on https://github.com/{}/pull/{}", - actions.job.repo.full_name, actions.job.pr.number - ); - let project = self.cloner.project( - &actions.job.repo.full_name, - actions.job.repo.clone_url.clone(), - ); - let co = project - .clone_for("builder".to_string(), self.identity.clone()) - .unwrap(); - - let target_branch = match actions.job.pr.target_branch.clone() { - Some(x) => x, - None => String::from("origin/master"), - }; - - let buildfile = match actions.job.subset { - Some(commentparser::Subset::NixOS) => nix::File::ReleaseNixOS, - _ => nix::File::DefaultNixpkgs, - }; + if actions.job.attrs.is_empty() { + debug!("No attrs to build"); + actions.nothing_to_do().await; + return; + } - let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); + info!( + "Working on https://github.com/{}/pull/{}", + actions.job.repo.full_name, actions.job.pr.number + ); + let project = self.cloner.project( + &actions.job.repo.full_name, + actions.job.repo.clone_url.clone(), + ); + let co = project + .clone_for("builder".to_string(), self.identity.clone()) + .unwrap(); + + let target_branch = match actions.job.pr.target_branch.clone() { + Some(x) => x, + None => String::from("origin/master"), + }; + + let buildfile = match actions.job.subset { + Some(commentparser::Subset::NixOS) => nix::File::ReleaseNixOS, + _ => nix::File::DefaultNixpkgs, + }; + + let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); + + if co.fetch_pr(actions.job.pr.number).is_err() { + info!("Failed to fetch {}", actions.job.pr.number); + actions.pr_head_missing().await; + return; + } - if co.fetch_pr(actions.job.pr.number).is_err() { - info!("Failed to fetch {}", actions.job.pr.number); - actions.pr_head_missing().await; - return; - } + if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { + info!("Commit {} doesn't exist", actions.job.pr.head_sha); + actions.commit_missing().await; + return; + } - if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { - info!("Commit {} doesn't exist", actions.job.pr.head_sha); - actions.commit_missing().await; - return; - } + if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { + info!("Failed to merge {}", actions.job.pr.head_sha); + actions.merge_failed().await; + return; + } - if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { - info!("Failed to merge {}", actions.job.pr.head_sha); - actions.merge_failed().await; - return; - } + info!( + "Got path: {:?}, determining which ones we can build ", + refpath + ); + let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( + refpath.as_ref(), + buildfile, + actions.job.attrs.clone(), + ); + + let cannot_build_attrs: Vec = cannot_build + .clone() + .into_iter() + .map(|(attr, _)| attr) + .collect(); + + info!( + "Can build: '{}', Cannot build: '{}'", + can_build.join(", "), + cannot_build_attrs.join(", ") + ); + + actions + .log_started(can_build.clone(), cannot_build_attrs.clone()) + .await; + actions.log_instantiation_errors(cannot_build).await; - info!( - "Got path: {:?}, determining which ones we can build ", - refpath - ); - let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( - refpath.as_ref(), - buildfile, - actions.job.attrs.clone(), - ); + if can_build.is_empty() { + actions.build_not_attempted(cannot_build_attrs).await; + return; + } - let cannot_build_attrs: Vec = cannot_build - .clone() - .into_iter() - .map(|(attr, _)| attr) - .collect(); - - info!( - "Can build: '{}', Cannot build: '{}'", - can_build.join(", "), - cannot_build_attrs.join(", ") - ); + let mut spawned = + self.nix + .safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone()); - actions - .log_started(can_build.clone(), cannot_build_attrs.clone()) - .await; - actions.log_instantiation_errors(cannot_build).await; + while let Ok(line) = spawned.get_next_line() { + actions.log_line(line).await; + } - if can_build.is_empty() { - actions.build_not_attempted(cannot_build_attrs).await; - return; - } + let status = nix::wait_for_build_status(spawned); - let mut spawned = - self.nix - .safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone()); + info!("ok built ({:?}), building", status); + info!("Lines:"); + info!("-----8<-----"); + actions + .log_snippet() + .iter() + .inspect(|x| info!("{}", x)) + .next_back(); + info!("----->8-----"); - while let Ok(line) = spawned.get_next_line() { - actions.log_line(line).await; + actions + .build_finished(status, can_build, cannot_build_attrs) + .await; + info!("Build done!"); } - - let status = nix::wait_for_build_status(spawned); - - info!("ok built ({:?}), building", status); - info!("Lines:"); - info!("-----8<-----"); - actions - .log_snippet() - .iter() - .inspect(|x| info!("{}", x)) - .next_back(); - info!("----->8-----"); - - actions - .build_finished(status, can_build, cannot_build_attrs) - .await; - info!("Build done!"); + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/eval/mod.rs b/ofborg/src/tasks/eval/mod.rs index b312db9a..f7baa954 100644 --- a/ofborg/src/tasks/eval/mod.rs +++ b/ofborg/src/tasks/eval/mod.rs @@ -4,12 +4,14 @@ pub use self::nixpkgs::NixpkgsStrategy; use crate::checkout::CachedProjectCo; use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::evalchecker::EvalChecker; +use crate::github::GithubRepo; use crate::message::buildjob::BuildJob; use std::path::Path; pub trait EvaluationStrategy { - fn pre_clone(&mut self) -> impl std::future::Future>; + fn pre_clone(&mut self, repo: &GithubRepo) + -> impl std::future::Future>; fn on_target_branch( &mut self, diff --git a/ofborg/src/tasks/eval/nixpkgs.rs b/ofborg/src/tasks/eval/nixpkgs.rs index 8f0fd7d7..9b8f8858 100644 --- a/ofborg/src/tasks/eval/nixpkgs.rs +++ b/ofborg/src/tasks/eval/nixpkgs.rs @@ -1,17 +1,18 @@ +use std::path::Path; + +use octocrab::models::StatusState; +use regex::Regex; +use tracing::warn; +use uuid::Uuid; + use crate::checkout::CachedProjectCo; use crate::commentparser::Subset; use crate::commitstatus::CommitStatus; use crate::evalchecker::EvalChecker; +use crate::github::GithubRepo; use crate::message::buildjob::BuildJob; use crate::message::evaluationjob::EvaluationJob; use crate::tasks::eval::{EvaluationComplete, EvaluationStrategy, StepResult}; -use crate::tasks::evaluate::update_labels; - -use std::path::Path; - -use hubcaps::issues::IssueRef; -use regex::Regex; -use uuid::Uuid; const TITLE_LABELS: [(&str, &str); 4] = [ ("bsd", "6.topic: bsd"), @@ -35,23 +36,26 @@ fn label_from_title(title: &str) -> Vec { pub struct NixpkgsStrategy<'a> { job: &'a EvaluationJob, - issue_ref: &'a IssueRef, + issue: Option<&'a octocrab::models::issues::Issue>, touched_packages: Option>, } impl<'a> NixpkgsStrategy<'a> { - pub fn new(job: &'a EvaluationJob, issue_ref: &'a IssueRef) -> NixpkgsStrategy<'a> { + pub fn new( + job: &'a EvaluationJob, + issue: Option<&'a octocrab::models::issues::Issue>, + ) -> NixpkgsStrategy<'a> { Self { job, - issue_ref, + issue, touched_packages: None, } } - async fn tag_from_title(&self) { - let title = match self.issue_ref.get().await { - Ok(issue) => issue.title.to_lowercase(), - Err(_) => return, + async fn tag_from_title(&self, repo: &GithubRepo) { + let title = match self.issue { + Some(issue) => issue.title.to_lowercase(), + None => return, }; let labels = label_from_title(&title); @@ -60,7 +64,10 @@ impl<'a> NixpkgsStrategy<'a> { return; } - update_labels(self.issue_ref, &labels, &[]).await; + let issue_number = self.issue.map(|i| i.number).unwrap_or(self.job.pr.number); + if let Err(e) = repo.update_labels(issue_number, &labels, &[]).await { + warn!("Failed to update labels on #{issue_number}: {e:?}"); + } } fn check_outpaths_before(&mut self, _dir: &Path) -> StepResult<()> { @@ -104,17 +111,14 @@ impl<'a> NixpkgsStrategy<'a> { } impl EvaluationStrategy for NixpkgsStrategy<'_> { - async fn pre_clone(&mut self) -> StepResult<()> { - self.tag_from_title().await; + async fn pre_clone(&mut self, repo: &GithubRepo) -> StepResult<()> { + self.tag_from_title(repo).await; Ok(()) } async fn on_target_branch(&mut self, dir: &Path, status: &mut CommitStatus) -> StepResult<()> { status - .set_with_description( - "Checking original out paths", - hubcaps::statuses::State::Pending, - ) + .set_with_description("Checking original out paths", StatusState::Pending) .await?; self.check_outpaths_before(dir)?; @@ -132,7 +136,7 @@ impl EvaluationStrategy for NixpkgsStrategy<'_> { async fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> { status - .set_with_description("Checking new out paths", hubcaps::statuses::State::Pending) + .set_with_description("Checking new out paths", StatusState::Pending) .await?; self.check_outpaths_after()?; @@ -148,10 +152,7 @@ impl EvaluationStrategy for NixpkgsStrategy<'_> { status: &mut CommitStatus, ) -> StepResult { status - .set_with_description( - "Calculating Changed Outputs", - hubcaps::statuses::State::Pending, - ) + .set_with_description("Calculating Changed Outputs", StatusState::Pending) .await?; let builds = self.queue_builds()?; diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index f34cf2b5..31cc0997 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -1,21 +1,20 @@ /// This is what evaluates every pull-request +use std::path::Path; +use std::time::Instant; + +use futures::stream::StreamExt; +use octocrab::{Octocrab, models::StatusState}; +use tracing::{Instrument, debug_span, error, info, warn}; + use crate::acl::Acl; -use crate::checkout; use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::config::GithubAppVendingMachine; +use crate::github::GithubRepo; use crate::message::{buildjob, evaluationjob}; use crate::stats::{self, Event}; -use crate::systems; use crate::tasks::eval; use crate::tasks::eval::EvaluationStrategy; -use crate::worker; -use futures::stream::StreamExt; -use futures_util::TryFutureExt; - -use std::path::Path; -use std::time::Instant; - -use tracing::{debug_span, error, info, warn}; +use crate::{checkout, systems, worker}; pub struct EvaluationWorker { cloner: checkout::CachedCloner, @@ -71,57 +70,72 @@ impl worker::SimpleWorker for EvaluationWorker async fn consumer(&mut self, job: &evaluationjob::EvaluationJob) -> worker::Actions { let span = debug_span!("job", pr = ?job.pr.number); - let _enter = span.enter(); - - let mut vending_machine = self.github_vend.write().await; - - let github_client = vending_machine - .for_repo(&job.repo.owner, &job.repo.name) + async { + let github_client = { + let mut vending_machine = self.github_vend.write().await; + match vending_machine + .for_repo(&job.repo.owner, &job.repo.name) + .await + { + Some(client) => client.clone(), + None => { + error!( + "Failed to get a github client token for {}/{}", + job.repo.owner, job.repo.name + ); + return vec![worker::Action::NackRequeue]; + } + } + }; + + OneEval::new( + &github_client, + &self.acl, + &mut self.events, + &self.identity, + &self.cloner, + job, + ) + .worker_actions() .await - .expect("Failed to get a github client token"); - - OneEval::new( - github_client, - &self.acl, - &mut self.events, - &self.identity, - &self.cloner, - job, - ) - .worker_actions() + } + .instrument(span) .await } } struct OneEval<'a, E> { - client_app: &'a hubcaps::Github, - repo: hubcaps::repositories::Repository, + repo: GithubRepo, acl: &'a Acl, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, + prefix: Option<&'static str>, } impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { #[allow(clippy::too_many_arguments)] fn new( - client_app: &'a hubcaps::Github, + octocrab: &'a Octocrab, acl: &'a Acl, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, ) -> OneEval<'a, E> { - let repo = client_app.repo(job.repo.owner.clone(), job.repo.name.clone()); OneEval { - client_app, - repo, + repo: GithubRepo::new( + octocrab.clone(), + job.repo.owner.clone(), + job.repo.name.clone(), + ), acl, events, identity, cloner, job, + prefix: None, } } @@ -133,52 +147,45 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { &self, description: String, url: Option, - state: hubcaps::statuses::State, + state: StatusState, ) -> Result<(), CommitStatusError> { - let description = if description.len() >= 140 { - warn!( - "description is over 140 char; truncating: {:?}", - &description - ); - description.chars().take(140).collect() - } else { - description - }; - let repo = self - .client_app - .repo(self.job.repo.owner.clone(), self.job.repo.name.clone()); - let prefix = get_prefix(repo.statuses(), &self.job.pr.head_sha).await?; - - let mut builder = hubcaps::statuses::StatusOptions::builder(state); - builder.context(format!("{prefix}-eval")); - builder.description(description.clone()); - - if let Some(url) = url { - builder.target_url(url); - } + let prefix = self + .prefix + .expect("prefix should have been set in worker_actions"); info!( "Updating status on {}:{} -> {}", &self.job.pr.number, &self.job.pr.head_sha, &description ); - self.repo - .statuses() - .create(&self.job.pr.head_sha, &builder.build()) - .map_ok(|_| ()) - .map_err(|e| CommitStatusError::from(e)) - .await + let status = CommitStatus::new( + self.repo.clone(), + self.job.pr.head_sha.clone(), + format!("{prefix}-eval"), + description, + url, + ); + + status.set(state).await } async fn worker_actions(&mut self) -> worker::Actions { + self.prefix = Some(match self.repo.get_prefix(&self.job.pr.head_sha).await { + Ok(p) => p, + Err(e) => { + error!("Failed to determine commit status prefix: {:?}", e); + return self.actions().retry_later(self.job); + } + }); + let eval_result = match self.evaluate_job().await { Ok(v) => Ok(v), Err(eval_error) => match eval_error { // Handle error cases which expect us to post statuses // to github. Convert Eval Errors in to Result<_, CommitStatusWrite> - EvalWorkerError::EvalError(eval::Error::Fail(msg)) => Err(self - .update_status(msg, None, hubcaps::statuses::State::Failure) - .await), + EvalWorkerError::EvalError(eval::Error::Fail(msg)) => { + Err(self.update_status(msg, None, StatusState::Failure).await) + } EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e)) => Err(Err(e)), EvalWorkerError::CommitStatusWrite(e) => Err(Err(e)), }, @@ -186,8 +193,15 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { match eval_result { Ok(eval_actions) => { - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[], &[String::from("ofborg-internal-error")]).await; + let issue = self.repo.issues().get(self.job.pr.number).await; + if let Ok(issue) = issue + && let Err(e) = self + .repo + .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) + .await + { + warn!("Failed to update labels: {e:?}"); + } eval_actions } @@ -195,51 +209,38 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { // There was an error during eval, but we successfully // updated the PR. - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[], &[String::from("ofborg-internal-error")]).await; + let issue = self.repo.issues().get(self.job.pr.number).await; + if let Ok(issue) = issue + && let Err(e) = self + .repo + .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) + .await + { + warn!("Failed to update labels: {e:?}"); + } self.actions().skip(self.job) } - Err(Err(CommitStatusError::ExpiredCreds(e))) => { - error!("Failed writing commit status: creds expired: {:?}", e); + Err(Err(CommitStatusError::OctocrabError(e))) => { + error!("Failed writing commit status: {:?}", e); self.actions().retry_later(self.job) } Err(Err(CommitStatusError::InternalError(e))) => { error!("Failed writing commit status: internal error: {:?}", e); self.actions().retry_later(self.job) } - Err(Err(CommitStatusError::MissingSha(e))) => { - error!( - "Failed writing commit status: commit sha was force-pushed away: {:?}", - e - ); - self.actions().skip(self.job) - } - - Err(Err(CommitStatusError::Error(cswerr))) => { - error!( - "Internal error writing commit status: {:?}, marking internal error", - cswerr - ); - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[String::from("ofborg-internal-error")], &[]).await; - - self.actions().skip(self.job) - } } } async fn evaluate_job(&mut self) -> Result { let job = self.job; - let repo = self - .client_app - .repo(self.job.repo.owner.clone(), self.job.repo.name.clone()); - let issue_ref = repo.issue(job.pr.number); let auto_schedule_build_archs: Vec; - match issue_ref.get().await { + let issue = self.repo.issues().get(job.pr.number).await; + + match issue { Ok(iss) => { - if iss.state == "closed" { + if iss.state == octocrab::models::IssueState::Closed { self.events.notify(Event::IssueAlreadyClosed).await; info!("Skipping {} because it is closed", job.pr.number); return Ok(self.actions().skip(job)); @@ -263,12 +264,16 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { } }; - let mut evaluation_strategy = eval::NixpkgsStrategy::new(job, &issue_ref); + let issue = self.repo.issues().get(job.pr.number).await.ok(); + + let mut evaluation_strategy = eval::NixpkgsStrategy::new(job, issue.as_ref()); - let prefix = get_prefix(repo.statuses(), &job.pr.head_sha).await?; + let prefix = self + .prefix + .expect("prefix should have been set in worker_actions"); let mut overall_status = CommitStatus::new( - repo.statuses(), + self.repo.clone(), job.pr.head_sha.clone(), format!("{prefix}-eval"), "Starting".to_owned(), @@ -276,17 +281,17 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { ); overall_status - .set_with_description("Starting", hubcaps::statuses::State::Pending) + .set_with_description("Starting", StatusState::Pending) .await?; - evaluation_strategy.pre_clone().await?; + evaluation_strategy.pre_clone(&self.repo).await?; let project = self .cloner .project(&job.repo.full_name, job.repo.clone_url.clone()); overall_status - .set_with_description("Cloning project", hubcaps::statuses::State::Pending) + .set_with_description("Cloning project", StatusState::Pending) .await?; info!("Working on {}", job.pr.number); @@ -307,8 +312,8 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { overall_status .set_with_description( "The branch you have targeted is a read-only mirror for channels. \ - Please target release-* or master.", - hubcaps::statuses::State::Error, + Please target release-* or master.", + StatusState::Error, ) .await?; @@ -319,7 +324,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { overall_status .set_with_description( format!("Checking out {}", &target_branch).as_ref(), - hubcaps::statuses::State::Pending, + StatusState::Pending, ) .await?; info!("Checking out target branch {}", &target_branch); @@ -331,12 +336,12 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { ))) })?; + let target_branch_rebuild_sniff_start = Instant::now(); + evaluation_strategy .on_target_branch(Path::new(&refpath), &mut overall_status) .await?; - let target_branch_rebuild_sniff_start = Instant::now(); - self.events .notify(Event::EvaluationDuration( target_branch.clone(), @@ -348,7 +353,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .await; overall_status - .set_with_description("Fetching PR", hubcaps::statuses::State::Pending) + .set_with_description("Fetching PR", StatusState::Pending) .await?; co.fetch_pr(job.pr.number).map_err(|e| { @@ -359,7 +364,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { if !co.commit_exists(job.pr.head_sha.as_ref()) { overall_status - .set_with_description("Commit not found", hubcaps::statuses::State::Error) + .set_with_description("Commit not found", StatusState::Error) .await?; info!("Commit {} doesn't exist", job.pr.head_sha); @@ -369,12 +374,12 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { evaluation_strategy.after_fetch(&co)?; overall_status - .set_with_description("Merging PR", hubcaps::statuses::State::Pending) + .set_with_description("Merging PR", StatusState::Pending) .await?; if co.merge_commit(job.pr.head_sha.as_ref()).is_err() { overall_status - .set_with_description("Failed to merge", hubcaps::statuses::State::Failure) + .set_with_description("Failed to merge", StatusState::Failure) .await?; info!("Failed to merge {}", job.pr.head_sha); @@ -386,41 +391,38 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { info!("Got path: {:?}, building", refpath); overall_status - .set_with_description("Beginning Evaluations", hubcaps::statuses::State::Pending) + .set_with_description("Beginning Evaluations", StatusState::Pending) .await?; let eval_results: bool = futures::stream::iter(evaluation_strategy.evaluation_checks()) .map(|check| { - // We need to clone or move variables into the async block - let repo_statuses = repo.statuses(); + let repo = self.repo.clone(); let head_sha = job.pr.head_sha.clone(); let refpath = refpath.clone(); async move { let status = CommitStatus::new( - repo_statuses, + repo, head_sha, format!("{prefix}-eval-{}", check.name()), check.cli_cmd(), None, ); - status - .set(hubcaps::statuses::State::Pending) - .await - .expect("Failed to set status on eval strategy"); + if let Err(e) = status.set(StatusState::Pending).await { + warn!("Failed to set pending status on eval strategy: {e:?}"); + } let state = match check.execute(Path::new(&refpath)) { - Ok(_) => hubcaps::statuses::State::Success, - Err(_) => hubcaps::statuses::State::Failure, + Ok(_) => StatusState::Success, + Err(_) => StatusState::Failure, }; - status - .set(state.clone()) - .await - .expect("Failed to set status on eval strategy"); + if let Err(e) = status.set(state).await { + warn!("Failed to set status on eval strategy: {e:?}"); + } - if state == hubcaps::statuses::State::Success { + if state == StatusState::Success { Ok(()) } else { Err(()) @@ -442,11 +444,11 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { response.extend(schedule_builds(complete.builds, auto_schedule_build_archs)); overall_status - .set_with_description("^.^!", hubcaps::statuses::State::Success) + .set_with_description("^.^!", StatusState::Success) .await?; } else { overall_status - .set_with_description("Complete, with errors", hubcaps::statuses::State::Failure) + .set_with_description("Complete, with errors", StatusState::Failure) .await?; } @@ -489,70 +491,10 @@ fn schedule_builds( response } -pub async fn update_labels( - issueref: &hubcaps::issues::IssueRef, - add: &[String], - remove: &[String], -) { - let l = issueref.labels(); - let issue = issueref.get().await.expect("Failed to get issue"); - - let existing: Vec = issue.labels.iter().map(|l| l.name.clone()).collect(); - - let to_add: Vec<&str> = add - .iter() - .filter(|l| !existing.contains(l)) // Remove labels already on the issue - .map(|l| l.as_ref()) - .collect(); - - let to_remove: Vec = remove - .iter() - .filter(|l| existing.contains(l)) // Remove labels already on the issue - .cloned() - .collect(); - - let issue = issue.number; - - info!("Labeling issue #{issue}: + {to_add:?} , - {to_remove:?}, = {existing:?}"); - - l.add(to_add.clone()) - .await - .unwrap_or_else(|err| panic!("Failed to add labels {to_add:?} to issue #{issue}: {err:?}")); - - for label in to_remove { - l.remove(&label).await.unwrap_or_else(|err| { - panic!("Failed to remove label {label:?} from issue #{issue}: {err:?}") - }); - } -} - -fn issue_is_wip(issue: &hubcaps::issues::Issue) -> bool { +fn issue_is_wip(issue: &octocrab::models::issues::Issue) -> bool { issue.title.starts_with("WIP:") || issue.title.contains("[WIP]") } -/// Determine whether or not to use the "old" status prefix, `grahamcofborg`, or -/// the new one, `ofborg`. -/// -/// If the PR already has any `grahamcofborg`-prefixed statuses, continue to use -/// that (e.g. if someone used `@ofborg eval`, `@ofborg build`, `@ofborg test`). -/// Otherwise, if it's a new PR or was recently force-pushed (and therefore -/// doesn't have any old `grahamcofborg`-prefixed statuses), use the new prefix. -pub async fn get_prefix( - statuses: hubcaps::statuses::Statuses, - sha: &str, -) -> Result<&str, CommitStatusError> { - if statuses - .list(sha) - .await? - .iter() - .any(|s| s.context.starts_with("grahamcofborg-")) - { - Ok("grahamcofborg") - } else { - Ok("ofborg") - } -} - enum EvalWorkerError { EvalError(eval::Error), CommitStatusWrite(CommitStatusError), diff --git a/ofborg/src/tasks/evaluationfilter.rs b/ofborg/src/tasks/evaluationfilter.rs index 91ca4f49..07d5881d 100644 --- a/ofborg/src/tasks/evaluationfilter.rs +++ b/ofborg/src/tasks/evaluationfilter.rs @@ -1,9 +1,7 @@ -use crate::acl; -use crate::ghevent; -use crate::message::{Pr, Repo, evaluationjob}; -use crate::worker; +use tracing::{Instrument, debug_span, info}; -use tracing::{debug_span, info}; +use crate::message::{Pr, Repo, evaluationjob}; +use crate::{acl, ghevent, worker}; pub struct EvaluationFilterWorker { acl: acl::Acl, @@ -35,70 +33,76 @@ impl worker::SimpleWorker for EvaluationFilterWorker { async fn consumer(&mut self, job: &ghevent::PullRequestEvent) -> worker::Actions { let span = debug_span!("job", pr = ?job.number); - let _enter = span.enter(); - - if !self.acl.is_repo_eligible(&job.repository.full_name) { - info!("Repo not authorized ({})", job.repository.full_name); - return vec![worker::Action::Ack]; - } + async { + if !self.acl.is_repo_eligible(&job.repository.full_name) { + info!("Repo not authorized ({})", job.repository.full_name); + return vec![worker::Action::Ack]; + } - if job.pull_request.state != ghevent::PullRequestState::Open { - info!( - "PR is not open ({}#{})", - job.repository.full_name, job.number - ); - return vec![worker::Action::Ack]; - } + if job.pull_request.state != ghevent::PullRequestState::Open { + info!( + "PR is not open ({}#{})", + job.repository.full_name, job.number + ); + return vec![worker::Action::Ack]; + } - let interesting: bool = match job.action { - ghevent::PullRequestAction::Opened => true, - ghevent::PullRequestAction::Synchronize => true, - ghevent::PullRequestAction::Reopened => true, - ghevent::PullRequestAction::Edited => { - if let Some(ref changes) = job.changes { - changes.base.is_some() - } else { - false + let interesting: bool = match job.action { + ghevent::PullRequestAction::Opened => true, + ghevent::PullRequestAction::Synchronize => true, + ghevent::PullRequestAction::Reopened => true, + ghevent::PullRequestAction::Edited => { + if let Some(ref changes) = job.changes { + changes.base.is_some() + } else { + false + } } + _ => false, + }; + + if !interesting { + info!( + "Not interesting: {}#{} because of {:?}", + job.repository.full_name, job.number, job.action + ); + + return vec![worker::Action::Ack]; } - _ => false, - }; - if !interesting { info!( - "Not interesting: {}#{} because of {:?}", + "Found {}#{} to be interesting because of {:?}", job.repository.full_name, job.number, job.action ); + let repo_msg = Repo { + clone_url: job.repository.clone_url.clone(), + full_name: job.repository.full_name.clone(), + owner: job.repository.owner.login.clone(), + name: job.repository.name.clone(), + }; + + let pr_msg = Pr { + number: job.number, + head_sha: job.pull_request.head.sha.clone(), + target_branch: Some(job.pull_request.base.git_ref.clone()), + }; + + let msg = evaluationjob::EvaluationJob { + repo: repo_msg, + pr: pr_msg, + }; - return vec![worker::Action::Ack]; + vec![ + worker::publish_serde_action( + None, + Some("mass-rebuild-check-jobs".to_owned()), + &msg, + ), + worker::Action::Ack, + ] } - - info!( - "Found {}#{} to be interesting because of {:?}", - job.repository.full_name, job.number, job.action - ); - let repo_msg = Repo { - clone_url: job.repository.clone_url.clone(), - full_name: job.repository.full_name.clone(), - owner: job.repository.owner.login.clone(), - name: job.repository.name.clone(), - }; - - let pr_msg = Pr { - number: job.number, - head_sha: job.pull_request.head.sha.clone(), - target_branch: Some(job.pull_request.base.git_ref.clone()), - }; - - let msg = evaluationjob::EvaluationJob { - repo: repo_msg, - pr: pr_msg, - }; - - vec![ - worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg), - worker::Action::Ack, - ] + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/githubcommentfilter.rs b/ofborg/src/tasks/githubcommentfilter.rs index 0656b1d4..decbc5a5 100644 --- a/ofborg/src/tasks/githubcommentfilter.rs +++ b/ofborg/src/tasks/githubcommentfilter.rs @@ -1,19 +1,18 @@ -use crate::acl; -use crate::commentparser; -use crate::ghevent; -use crate::message::{Pr, Repo, buildjob, evaluationjob}; -use crate::worker; - -use tracing::{debug_span, error, info}; +use octocrab::Octocrab; +use tracing::{Instrument, debug_span, error, info}; use uuid::Uuid; +use crate::github::GithubRepo; +use crate::message::{Pr, Repo, buildjob, evaluationjob}; +use crate::{acl, commentparser, ghevent, worker}; + pub struct GitHubCommentWorker { acl: acl::Acl, - github: hubcaps::Github, + github: Octocrab, } impl GitHubCommentWorker { - pub fn new(acl: acl::Acl, github: hubcaps::Github) -> GitHubCommentWorker { + pub fn new(acl: acl::Acl, github: Octocrab) -> GitHubCommentWorker { GitHubCommentWorker { acl, github } } } @@ -43,129 +42,127 @@ impl worker::SimpleWorker for GitHubCommentWorker { #[allow(clippy::cognitive_complexity)] async fn consumer(&mut self, job: &ghevent::IssueComment) -> worker::Actions { let span = debug_span!("job", pr = ?job.issue.number); - let _enter = span.enter(); - - if job.action == ghevent::IssueCommentAction::Deleted - || job.action == ghevent::IssueCommentAction::Pinned - || job.action == ghevent::IssueCommentAction::Unpinned - { - return vec![worker::Action::Ack]; - } + async { + if job.action == ghevent::IssueCommentAction::Deleted + || job.action == ghevent::IssueCommentAction::Pinned + || job.action == ghevent::IssueCommentAction::Unpinned + { + return vec![worker::Action::Ack]; + } - let instructions = commentparser::parse(&job.comment.body); - if instructions.is_none() { - return vec![worker::Action::Ack]; - } + let instructions = commentparser::parse(&job.comment.body); + if instructions.is_none() { + return vec![worker::Action::Ack]; + } - let build_destinations = self.acl.build_job_architectures_for_user_repo( - &job.comment.user.login, - &job.repository.full_name, - ); + let build_destinations = self.acl.build_job_architectures_for_user_repo( + &job.comment.user.login, + &job.repository.full_name, + ); - if build_destinations.is_empty() { - info!("No build destinations for: {:?}", job); - // Don't process comments if they can't build anything - return vec![worker::Action::Ack]; - } + if build_destinations.is_empty() { + info!("No build destinations for: {:?}", job); + // Don't process comments if they can't build anything + return vec![worker::Action::Ack]; + } - info!("Got job: {:?}", job); - - let instructions = commentparser::parse(&job.comment.body); - info!("Instructions: {:?}", instructions); - - let pr = self - .github - .repo( - job.repository.owner.login.clone(), - job.repository.name.clone(), - ) - .pulls() - .get(job.issue.number) - .get() - .await; - - if let Err(x) = pr { - info!( - "fetching PR {}#{} from GitHub yielded error {}", - job.repository.full_name, job.issue.number, x - ); - return vec![worker::Action::Ack]; - } + info!("Got job: {:?}", job); - let pr = pr.unwrap(); - - let repo_msg = Repo { - clone_url: job.repository.clone_url.clone(), - full_name: job.repository.full_name.clone(), - owner: job.repository.owner.login.clone(), - name: job.repository.name.clone(), - }; - - let pr_msg = Pr { - number: job.issue.number, - head_sha: pr.head.sha.clone(), - target_branch: Some(pr.base.commit_ref), - }; - - let mut response: Vec = vec![]; - if let Some(instructions) = instructions { - for instruction in instructions { - match instruction { - commentparser::Instruction::Build(subset, attrs) => { - let build_destinations = match subset { - commentparser::Subset::NixOS => build_destinations - .clone() - .into_iter() - .filter(|x| x.can_run_nixos_tests()) - .collect(), - _ => build_destinations.clone(), - }; - - let msg = buildjob::BuildJob::new( - repo_msg.clone(), - pr_msg.clone(), - subset, - attrs, - None, - None, - Uuid::new_v4().to_string(), - ); - - for arch in build_destinations.iter() { - let (exchange, routingkey) = arch.as_build_destination(); - response.push(worker::publish_serde_action(exchange, routingkey, &msg)); - } + let instructions = commentparser::parse(&job.comment.body); + info!("Instructions: {:?}", instructions); - response.push(worker::publish_serde_action( - Some("build-results".to_string()), - None, - &buildjob::QueuedBuildJobs { - job: msg, - architectures: build_destinations - .iter() - .cloned() - .map(|arch| arch.to_string()) + let github_repo = GithubRepo::new( + self.github.clone(), + &job.repository.owner.login, + &job.repository.name, + ); + let pr = match github_repo.pulls().get(job.issue.number).await { + Ok(pr) => pr, + Err(x) => { + info!( + "fetching PR {}#{} from GitHub yielded error: {}", + job.repository.full_name, job.issue.number, x + ); + return vec![worker::Action::Ack]; + } + }; + + let repo_msg = Repo { + clone_url: job.repository.clone_url.clone(), + full_name: job.repository.full_name.clone(), + owner: job.repository.owner.login.clone(), + name: job.repository.name.clone(), + }; + + let pr_msg = Pr { + number: job.issue.number, + head_sha: pr.head.sha.clone(), + target_branch: Some(pr.base.ref_field.clone()), + }; + + let mut response: Vec = vec![]; + if let Some(instructions) = instructions { + for instruction in instructions { + match instruction { + commentparser::Instruction::Build(subset, attrs) => { + let build_destinations = match subset { + commentparser::Subset::NixOS => build_destinations + .clone() + .into_iter() + .filter(|x| x.can_run_nixos_tests()) .collect(), - }, - )); - } - commentparser::Instruction::Eval => { - let msg = evaluationjob::EvaluationJob { - repo: repo_msg.clone(), - pr: pr_msg.clone(), - }; - - response.push(worker::publish_serde_action( - None, - Some("mass-rebuild-check-jobs".to_owned()), - &msg, - )); + _ => build_destinations.clone(), + }; + + let msg = buildjob::BuildJob::new( + repo_msg.clone(), + pr_msg.clone(), + subset, + attrs, + None, + None, + Uuid::new_v4().to_string(), + ); + + for arch in build_destinations.iter() { + let (exchange, routingkey) = arch.as_build_destination(); + response + .push(worker::publish_serde_action(exchange, routingkey, &msg)); + } + + response.push(worker::publish_serde_action( + Some("build-results".to_string()), + None, + &buildjob::QueuedBuildJobs { + job: msg, + architectures: build_destinations + .iter() + .cloned() + .map(|arch| arch.to_string()) + .collect(), + }, + )); + } + commentparser::Instruction::Eval => { + let msg = evaluationjob::EvaluationJob { + repo: repo_msg.clone(), + pr: pr_msg.clone(), + }; + + response.push(worker::publish_serde_action( + None, + Some("mass-rebuild-check-jobs".to_owned()), + &msg, + )); + } } } } - } - response.push(worker::Action::Ack); - response + response.push(worker::Action::Ack); + response + } + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/githubcommentposter.rs b/ofborg/src/tasks/githubcommentposter.rs index 12c7c7fd..3a137897 100644 --- a/ofborg/src/tasks/githubcommentposter.rs +++ b/ofborg/src/tasks/githubcommentposter.rs @@ -1,13 +1,14 @@ +use chrono::Utc; +use octocrab::params::checks::{CheckRunConclusion, CheckRunOutput, CheckRunStatus}; +use tracing::{Instrument, debug_span, info, warn}; + use crate::config::GithubAppVendingMachine; +use crate::github::GithubRepo; use crate::message::Repo; use crate::message::buildjob::{BuildJob, QueuedBuildJobs}; use crate::message::buildresult::{BuildResult, BuildStatus, LegacyBuildResult}; use crate::worker; -use chrono::{DateTime, Utc}; -use hubcaps::checks::{CheckRunOptions, CheckRunState, Conclusion, Output}; -use tracing::{debug, debug_span, info, warn}; - pub struct GitHubCommentPoster { github_vend: GithubAppVendingMachine, } @@ -18,6 +19,15 @@ impl GitHubCommentPoster { } } +pub struct CheckRunInfo { + pub name: String, + pub details_url: String, + pub output: CheckRunOutput, + pub conclusion: Option, + pub status: Option, + pub completed_at: Option>, +} + pub enum PostableEvent { BuildQueued(QueuedBuildJobs), BuildFinished(BuildResult), @@ -52,58 +62,82 @@ impl worker::SimpleWorker for GitHubCommentPoster { } async fn consumer(&mut self, job: &PostableEvent) -> worker::Actions { - let mut checks: Vec = vec![]; + let mut check_runs: Vec = vec![]; let repo: Repo; + let pr_number: u64; + let head_sha: String; - let pr = match job { + match job { PostableEvent::BuildQueued(queued_job) => { repo = queued_job.job.repo.clone(); + pr_number = queued_job.job.pr.number; + head_sha = queued_job.job.pr.head_sha.clone(); for architecture in queued_job.architectures.iter() { - checks.push(job_to_check(&queued_job.job, architecture, Utc::now())); + check_runs.push(job_to_check_info(&queued_job.job, architecture)); } - queued_job.job.pr.to_owned() } PostableEvent::BuildFinished(finished_job) => { let result = finished_job.legacy(); repo = result.repo.clone(); - checks.push(result_to_check(&result, Utc::now())); - finished_job.pr() + pr_number = result.pr.number; + head_sha = result.pr.head_sha.clone(); + check_runs.push(result_to_check_info(&result)); } }; - let span = debug_span!("job", pr = ?pr.number); - let _enter = span.enter(); - - for check in checks { - info!( - "check {:?} {} {}", - check.status, - check.name, - check.details_url.as_ref().unwrap_or(&String::from("-")) - ); - debug!("{:?}", check); - - let check_create_attempt = self - .github_vend - .for_repo(&repo.owner, &repo.name) - .await - .unwrap() - .repo(repo.owner.clone(), repo.name.clone()) - .checkruns() - .create(&check) - .await; - - match check_create_attempt { - Ok(_) => info!("Successfully sent."), - Err(err) => warn!("Failed to send check {:?}", err), + let span = debug_span!("job", pr = ?pr_number); + async { + let octocrab_ref = match self.github_vend.for_repo(&repo.owner, &repo.name).await { + Some(client) => client.clone(), + None => { + warn!( + "No GitHub installation found for {}/{}, skipping checks", + repo.owner, repo.name + ); + return vec![worker::Action::Ack]; + } + }; + + for check in check_runs { + info!( + "check {:?} {} {}", + check.status, check.name, check.details_url, + ); + + let github_repo = GithubRepo::new(octocrab_ref.clone(), &repo.owner, &repo.name); + let checks_handler = github_repo.checks(); + let mut builder = checks_handler + .create_check_run(check.name, head_sha.clone()) + .details_url(check.details_url); + + builder = builder.output(check.output); + + if let Some(completed_at) = check.completed_at { + builder = builder.completed_at(completed_at); + } + + if let Some(conclusion) = check.conclusion { + builder = builder.conclusion(conclusion); + } + + if let Some(status) = check.status { + builder = builder.status(status); + } + + match builder.send().await { + Ok(_) => info!("Successfully sent check."), + Err(err) => warn!("Failed to send check {:?}", err), + } } - } - vec![worker::Action::Ack] + vec![worker::Action::Ack] + } + .instrument(span) + .await } } -fn job_to_check(job: &BuildJob, architecture: &str, timestamp: DateTime) -> CheckRunOptions { +fn job_to_check_info(job: &BuildJob, architecture: &str) -> CheckRunInfo { let mut all_attrs: Vec = job.attrs.clone(); all_attrs.sort(); @@ -111,26 +145,31 @@ fn job_to_check(job: &BuildJob, architecture: &str, timestamp: DateTime) -> all_attrs = vec![String::from("(unknown attributes)")]; } - CheckRunOptions { - name: format!("{} on {architecture}", all_attrs.join(", ")), - actions: None, - completed_at: None, - started_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), + let name = format!("{} on {architecture}", all_attrs.join(", ")); + let details_url = format!( + "https://logs.ofborg.org/?key={}/{}.{}", + &job.repo.owner.to_lowercase(), + &job.repo.name.to_lowercase(), + job.pr.number, + ); + + CheckRunInfo { + name, + details_url, + output: CheckRunOutput { + title: "Queued".to_string(), + summary: String::new(), + text: None, + annotations: vec![], + images: vec![], + }, conclusion: None, - details_url: Some(format!( - "https://logs.ofborg.org/?key={}/{}.{}", - &job.repo.owner.to_lowercase(), - &job.repo.name.to_lowercase(), - job.pr.number, - )), - external_id: None, - head_sha: job.pr.head_sha.clone(), - output: None, - status: Some(CheckRunState::Queued), + status: Some(CheckRunStatus::Queued), + completed_at: None, } } -fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> CheckRunOptions { +fn result_to_check_info(result: &LegacyBuildResult) -> CheckRunInfo { let mut all_attrs: Vec = vec![result.attempted_attrs.clone(), result.skipped_attrs.clone()] .into_iter() @@ -143,7 +182,14 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec all_attrs = vec![String::from("(unknown attributes)")]; } - let conclusion: Conclusion = result.status.clone().into(); + let conclusion: CheckRunConclusion = match result.status { + BuildStatus::Skipped => CheckRunConclusion::Skipped, + BuildStatus::Success => CheckRunConclusion::Success, + BuildStatus::Failure => CheckRunConclusion::Neutral, + BuildStatus::TimedOut => CheckRunConclusion::Neutral, + BuildStatus::UnexpectedError { .. } => CheckRunConclusion::Neutral, + BuildStatus::HashMismatch => CheckRunConclusion::Failure, + }; let mut summary: Vec = vec![]; if let Some(ref attempted) = result.attempted_attrs { @@ -164,8 +210,6 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec )); } - // Allow the clippy violation for improved readability - #[allow(clippy::vec_init_then_push)] let text: String = if !result.output.is_empty() { let mut reply: Vec = vec![]; @@ -180,30 +224,28 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec String::from("No partial log is available.") }; - CheckRunOptions { - name: format!("{} on {}", all_attrs.join(", "), result.system), - actions: None, - completed_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), - started_at: None, - conclusion: Some(conclusion), - details_url: Some(format!( - "https://logs.ofborg.org/?key={}/{}.{}&attempt_id={}", - &result.repo.owner.to_lowercase(), - &result.repo.name.to_lowercase(), - result.pr.number, - result.attempt_id, - )), - external_id: Some(result.attempt_id.clone()), - head_sha: result.pr.head_sha.clone(), - - output: Some(Output { - annotations: None, - images: None, + let name = format!("{} on {}", all_attrs.join(", "), result.system); + let details_url = format!( + "https://logs.ofborg.org/?key={}/{}.{}&attempt_id={}", + &result.repo.owner.to_lowercase(), + &result.repo.name.to_lowercase(), + result.pr.number, + result.attempt_id, + ); + + CheckRunInfo { + name, + details_url, + output: CheckRunOutput { + title: result.status.clone().into(), summary: summary.join("\n"), text: Some(text), - title: result.status.clone().into(), - }), - status: Some(CheckRunState::Completed), + annotations: vec![], + images: vec![], + }, + conclusion: Some(conclusion), + status: Some(CheckRunStatus::Completed), + completed_at: Some(Utc::now()), } } @@ -221,545 +263,343 @@ fn list_segment(name: &str, things: &[String]) -> Vec { #[cfg(test)] mod tests { use super::*; + use crate::message::buildjob::BuildJob; use crate::message::{Pr, Repo}; - use chrono::TimeZone; + + fn check_passing_log() -> Vec { + vec![ + "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'" + .to_owned(), + "make[2]: Nothing to be done for 'install'.".to_owned(), + "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'" + .to_owned(), + "make[1]: Nothing to be done for 'install-target'.".to_owned(), + "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), + "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'" + .to_owned(), + "post-installation fixup".to_owned(), + "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip" + .to_owned(), + "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1" + .to_owned(), + "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), + ] + } + + fn partial_log_text(lines: &[String]) -> String { + let mut reply: Vec = vec![]; + reply.push("## Partial log".to_owned()); + reply.push("".to_owned()); + reply.push("```".to_owned()); + reply.extend(lines.iter().cloned()); + reply.push("```".to_owned()); + reply.join("\n") + } + + fn base_repo() -> Repo { + Repo { + clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), + full_name: "NixOS/nixpkgs".to_owned(), + owner: "NixOS".to_owned(), + name: "nixpkgs".to_owned(), + } + } + + fn base_pr() -> Pr { + Pr { + head_sha: "abc123".to_owned(), + number: 2345, + target_branch: Some("master".to_owned()), + } + } + + fn base_result() -> LegacyBuildResult { + LegacyBuildResult { + repo: base_repo(), + pr: base_pr(), + output: vec![], + attempt_id: "neatattemptid".to_owned(), + request_id: "bogus-request-id".to_owned(), + system: "x86_64-linux".to_owned(), + attempted_attrs: None, + skipped_attrs: None, + status: BuildStatus::Skipped, + } + } #[test] pub fn test_queued_build() { let job = BuildJob { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, + repo: base_repo(), + pr: base_pr(), logs: None, statusreport: None, subset: None, - request_id: "bogus-request-id".to_owned(), attrs: vec!["foo".to_owned(), "bar".to_owned()], }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); + let result = job_to_check_info(&job, "x86_64-linux"); + assert_eq!(result.name, "bar, foo on x86_64-linux"); assert_eq!( - job_to_check(&job, "x86_64-linux", timestamp), - CheckRunOptions { - name: "bar, foo on x86_64-linux".to_string(), - actions: None, - started_at: Some("2023-04-20T13:37:42Z".to_string()), - completed_at: None, - status: Some(CheckRunState::Queued), - conclusion: None, - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345".to_string()), - external_id: None, - head_sha: "abc123".to_string(), - output: None, - } + result.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345" ); + assert_eq!(result.output.title, "Queued"); + assert_eq!(result.output.summary, ""); + assert_eq!(result.output.text, None); + assert!(result.output.annotations.is_empty()); + assert!(result.output.images.is_empty()); + assert!(result.conclusion.is_none()); + assert!(matches!(result.status, Some(CheckRunStatus::Queued))); } #[test] pub fn test_check_passing_build() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: Some(vec!["bar".to_owned()]), status: BuildStatus::Success, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "bar, foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "bar, foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Success), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Success".to_string(), - summary: "Attempted: foo - -The following builds were skipped because they don't evaluate on x86_64-linux: bar -" - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Success) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Success"); + assert_eq!( + check.output.summary, + concat!( + "Attempted: foo", + "\n", + "\n", + "The following builds were skipped because they don't evaluate on x86_64-linux: bar", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_failing_build() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::Failure, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Failure".to_string(), - summary: "Attempted: foo -" - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Failure"); + assert_eq!(check.output.summary, "Attempted: foo\n"); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_timedout_build() { + let mut log = check_passing_log(); + log.push( + "building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' \ + timed out after 1 seconds" + .to_owned(), + ); + log.push( + "error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed" + .to_owned(), + ); + let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds".to_owned(), - "error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: log.clone(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::TimedOut, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Timed out, unknown build status".to_string(), - summary: "Attempted: foo - -Build timed out." - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds -error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Timed out, unknown build status"); + assert_eq!( + check.output.summary, + concat!("Attempted: foo", "\n", "\n", "Build timed out.") + ); + assert_eq!(check.output.text, Some(partial_log_text(&log))); } #[test] pub fn test_check_passing_build_unspecified_attributes() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Success, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "(unknown attributes) on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "(unknown attributes) on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Success), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Success".to_string(), - summary: "".to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Success) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Success"); + assert_eq!(check.output.summary, ""); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_failing_build_unspecified_attributes() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Failure, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "(unknown attributes) on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "(unknown attributes) on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Failure".to_string(), - summary: "".to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Failure"); + assert_eq!(check.output.summary, ""); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_no_attempt() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, output: vec!["foo".to_owned()], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "not-attempted on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "not-attempted on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "No attempt".to_string(), - summary: "The following builds were skipped because they don\'t evaluate on x86_64-linux: not-attempted -".to_string(), - text: Some("## Partial log - -``` -foo -```".to_string()), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Skipped) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "No attempt"); + assert_eq!( + check.output.summary, + concat!( + "The following builds were skipped because they don't evaluate on x86_64-linux: \ + not-attempted", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some( + concat!( + "## Partial log", + "\n", + "\n", + "```", + "\n", + "foo", + "\n", + "```" + ) + .to_string() + ) ); } #[test] pub fn test_check_no_attempt_no_log() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, output: vec![], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "not-attempted on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "not-attempted on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "No attempt".to_string(), - summary: "The following builds were skipped because they don\'t evaluate on x86_64-linux: not-attempted -".to_string(), - text: Some("No partial log is available.".to_string()), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Skipped) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "No attempt"); + assert_eq!( + check.output.summary, + concat!( + "The following builds were skipped because they don't evaluate on x86_64-linux: \ + not-attempted", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some("No partial log is available.".to_string()) ); } } diff --git a/ofborg/src/tasks/log_message_collector.rs b/ofborg/src/tasks/log_message_collector.rs index f359ffe7..be182642 100644 --- a/ofborg/src/tasks/log_message_collector.rs +++ b/ofborg/src/tasks/log_message_collector.rs @@ -1,8 +1,3 @@ -use crate::message::buildlogmsg::{BuildLogMsg, BuildLogStart}; -use crate::message::buildresult::BuildResult; -use crate::worker; -use crate::writetoline::LineWriter; - use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::path::{Component, Path, PathBuf}; @@ -10,6 +5,11 @@ use std::path::{Component, Path, PathBuf}; use lru_cache::LruCache; use tracing::warn; +use crate::message::buildlogmsg::{BuildLogMsg, BuildLogStart}; +use crate::message::buildresult::BuildResult; +use crate::worker; +use crate::writetoline::LineWriter; + #[derive(Eq, PartialEq, Hash, Debug, Clone)] pub struct LogFrom { routing_key: String, diff --git a/ofborg/src/tasks/statscollector.rs b/ofborg/src/tasks/statscollector.rs index fef23ad3..9b2d8067 100644 --- a/ofborg/src/tasks/statscollector.rs +++ b/ofborg/src/tasks/statscollector.rs @@ -1,8 +1,8 @@ +use tracing::error; + use crate::stats; use crate::worker; -use tracing::error; - pub struct StatCollectorWorker { events: E, collector: stats::MetricCollector, diff --git a/ofborg/src/worker.rs b/ofborg/src/worker.rs index 9569b450..eaae8e3b 100644 --- a/ofborg/src/worker.rs +++ b/ofborg/src/worker.rs @@ -1,7 +1,5 @@ use std::{marker::Send, sync::Arc}; -use serde::Serialize; - pub struct Response {} pub type Actions = Vec; @@ -24,7 +22,7 @@ pub struct QueueMsg { pub content: Vec, } -pub fn publish_serde_action( +pub fn publish_serde_action( exchange: Option, routing_key: Option, msg: &T, @@ -35,7 +33,7 @@ pub fn publish_serde_action( mandatory: false, immediate: false, content_type: Some("application/json".to_owned()), - content: serde_json::to_string(&msg).unwrap().into_bytes(), + content: serde_json::to_vec(&msg).expect("Failed to serialize message for publication"), })) } diff --git a/ofborg/src/writetoline.rs b/ofborg/src/writetoline.rs index 84846424..3ed3f2cf 100644 --- a/ofborg/src/writetoline.rs +++ b/ofborg/src/writetoline.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::{BufRead, BufReader, Seek, SeekFrom, Write}; +use std::io::{BufRead as _, BufReader, Seek, SeekFrom, Write as _}; pub struct LineWriter { file: File, From 11c4aaec6539e92d45703ffc29426ad2887da07a Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Wed, 22 Apr 2026 11:04:41 +0200 Subject: [PATCH 02/17] chore: drop unused dependency: log, either, mime --- Cargo.lock | 11 +---------- ofborg-simple-build/Cargo.toml | 2 +- ofborg-simple-build/src/main.rs | 6 ++---- ofborg/Cargo.toml | 3 --- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b069e70..96d579fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1779,12 +1779,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1963,7 +1957,6 @@ dependencies = [ "async-trait", "brace-expand", "chrono", - "either", "fs2", "futures", "futures-util", @@ -1977,12 +1970,10 @@ dependencies = [ "lapin", "lru-cache", "md5", - "mime", "nom 8.0.0", "octocrab", "parking_lot", "regex", - "rustls-pki-types", "serde", "serde_json", "sha2 0.11.0", @@ -1998,8 +1989,8 @@ dependencies = [ name = "ofborg-simple-build" version = "0.1.0" dependencies = [ - "log", "ofborg", + "tracing", ] [[package]] diff --git a/ofborg-simple-build/Cargo.toml b/ofborg-simple-build/Cargo.toml index 107744e0..12d7067d 100644 --- a/ofborg-simple-build/Cargo.toml +++ b/ofborg-simple-build/Cargo.toml @@ -7,4 +7,4 @@ edition = "2024" [dependencies] ofborg = { path = "../ofborg" } -log = "0.4.25" +tracing = "0.1.41" diff --git a/ofborg-simple-build/src/main.rs b/ofborg-simple-build/src/main.rs index b27aa040..edc24596 100644 --- a/ofborg-simple-build/src/main.rs +++ b/ofborg-simple-build/src/main.rs @@ -1,5 +1,3 @@ -extern crate log; - use std::env; use std::fs::File; use std::io::Read; @@ -11,11 +9,11 @@ use ofborg::nix; fn main() { ofborg::setup_log(); - log::info!("Loading config..."); + tracing::info!("Loading config..."); let cfg = config::load(env::args().nth(1).unwrap().as_ref()); let nix = cfg.nix(); - log::info!("Running build..."); + tracing::info!("Running build..."); match nix.safely_build_attrs( Path::new("./"), nix::File::DefaultNixpkgs, diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index 94f93e47..be2affea 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -12,7 +12,6 @@ chrono = { version = "0.4.38", default-features = false, features = [ "clock", "std", ] } -either = "1.13.0" fs2 = "0.4.3" futures = "0.3.31" futures-util = "0.3.31" @@ -37,11 +36,9 @@ hyper-util = { version = "0.1", features = ["server", "tokio", "http1"] } lapin = "4.3.0" lru-cache = "0.1.2" md5 = "0.8.0" -mime = "0.3" nom = "8" parking_lot = "0.12.4" regex = "1.11.1" -rustls-pki-types = "1.14" serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.135" sha2 = "0.11.0" From 8314f840d6ee27cbb04c7e2687c2ce0f006e8b21 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 29 May 2026 08:44:34 +0200 Subject: [PATCH 03/17] chore: cargo lockfile update --- Cargo.lock | 247 +++++++++++++++++++++++++++-------------------------- 1 file changed, 127 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96d579fa..6dd57a9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "amq-protocol" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95442d3faf08b9bed7491ceda36b0e0268126cb20d44e6e7573cf930d9aa5073" +checksum = "2eab68e8836c5812a01b34c5364d28db50bf686b442e902d9d93e4472318d86e" dependencies = [ "amq-protocol-tcp", "amq-protocol-types", @@ -38,9 +38,9 @@ dependencies = [ [[package]] name = "amq-protocol-tcp" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc754c77e62e58ea7323c3f257a97f747ac57f0ff3f7a91185f484dc1cb25bd2" +checksum = "8689f976dbd9864922f4f53e01ad2b43700ed3eb1bb5667b260522f291434dae" dependencies = [ "amq-protocol-uri", "async-rs", @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "amq-protocol-types" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8cb8fa210da5ec52799abf7ce45e2275b5a3f82ec4efafcd3e8161b7dbc9a8" +checksum = "27894e9e57d07f701251aeee3d2ab3d7d114bc830bf722d6626027eb55f3b222" dependencies = [ "cookie-factory", "nom 8.0.0", @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "amq-protocol-uri" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bc3e8b145641e20b57777a17b3bd7ba6bbdbb2b2ae2f26f6224be35e42c330" +checksum = "64fd5ca63f1b8cba2309aaec8595483c36f3a671225d5ab9fe8265adb9213514" dependencies = [ "amq-protocol-types", "percent-encoding", @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -201,14 +201,13 @@ dependencies = [ [[package]] name = "async-rs" -version = "0.8.4" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ebdd144e4199c67b5134fe2280e40d9656071f11df525d3322b43b6534c714" +checksum = "3cd5147201b63ba6883ffabca3a153822f71541748d7108e3e799beaeb283131" dependencies = [ "async-compat", "async-global-executor", "async-trait", - "cfg-if", "futures-core", "futures-io", "hickory-resolver", @@ -241,15 +240,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -257,9 +256,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -348,9 +347,9 @@ checksum = "c3adb80ee272c844254166ea32c8ae11c211b3639a293fdde41b1645b6be2c62" [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -402,9 +401,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.61" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -449,7 +448,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.1.6", "inout", ] @@ -464,9 +463,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "cms" @@ -605,9 +604,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -615,9 +614,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", ] @@ -728,27 +727,27 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.7", + "crypto-common 0.1.6", "subtle", ] [[package]] name = "digest" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", - "crypto-common 0.2.1", + "crypto-common 0.2.2", "ctutils", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -801,9 +800,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -1059,9 +1058,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -1120,9 +1119,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1148,9 +1147,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1166,9 +1165,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-net" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c61c8db47fae51ba9f8f2a2748bd87542acfbe22f2ec9cf9c8ec72d1ee6e9a6" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", @@ -1190,9 +1189,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" dependencies = [ "data-encoding", "idna", @@ -1210,9 +1209,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10bd64d950b4d38ca21e25c8ae230712e4955fb8290cfcb29a5e5dc6017e544" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", @@ -1258,14 +1257,14 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest 0.11.2", + "digest 0.11.3", ] [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1308,18 +1307,18 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" dependencies = [ "atomic-waker", "bytes", @@ -1527,7 +1526,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1564,16 +1563,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1641,19 +1630,21 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] [[package]] name = "jsonwebtoken" -version = "10.3.0" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ "base64", "ed25519-dalek", @@ -1670,19 +1661,21 @@ dependencies = [ "sha2 0.10.9", "signature", "simple_asn1", + "zeroize", ] [[package]] name = "lapin" -version = "4.6.0" +version = "4.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe00373c0142f6122cb2efb41bc88f584f7e109f42569a0a87b5d061d4e436" +checksum = "0fd20e01fd92597ca352ca7ceed3c589851ebad279dfcada48aa4d24fd3a7caa" dependencies = [ "amq-protocol", "async-rs", "async-trait", "backon", "cfg-if", + "event-listener", "flume", "futures-core", "futures-io", @@ -1745,9 +1738,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" [[package]] name = "lru-cache" @@ -1775,9 +1768,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "minimal-lexical" @@ -1787,9 +1780,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1875,9 +1868,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2131,18 +2124,18 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2249,9 +2242,9 @@ dependencies = [ [[package]] name = "prefix-trie" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23370be78b7e5bcbb0cab4a02047eb040279a693c78daad04c2c5f1c24a83503" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ "either", "ipnet", @@ -2484,9 +2477,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.39" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -2500,9 +2493,9 @@ dependencies = [ [[package]] name = "rustls-connector" -version = "0.23.0" +version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a288bf4b9d06a7c33e54e6879e2142fa45fc936017c3e1319147889daedf14d4" +checksum = "b7664e32b0ffbec386bdf1d7cbca51a89551a90d3278f135186cd6cb3cfa889c" dependencies = [ "futures-io", "futures-rustls", @@ -2717,9 +2710,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2781,7 +2774,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "digest 0.11.2", + "digest 0.11.3", ] [[package]] @@ -2795,9 +2788,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signature" @@ -2872,9 +2865,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2968,9 +2961,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tcp-stream" -version = "0.34.5" +version = "0.34.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6638f031787a4854f0ecc669514404d201a3f4eab1849e4151f01a28324972de" +checksum = "300d0735de48a565461c2ea14cc75b80ee5b6be3b4f5aeabe3553e4c2df8d23d" dependencies = [ "async-rs", "cfg-if", @@ -3079,9 +3072,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3156,21 +3149,21 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", "futures-util", "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3377,9 +3370,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3390,9 +3383,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3400,9 +3393,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3413,9 +3406,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -3816,18 +3809,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" dependencies = [ "proc-macro2", "quote", @@ -3836,9 +3829,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] @@ -3860,6 +3853,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "zerotrie" From 25ecc817dd195d5f7387d536431b27342efe51f1 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Wed, 22 Apr 2026 15:47:44 +0200 Subject: [PATCH 04/17] feat: testing/staging setup --- Cargo.lock | 341 +++++++++++++++++++++++++++++++++- Cargo.toml | 2 +- ofborg-send-event/Cargo.toml | 29 +++ ofborg-send-event/src/main.rs | 207 +++++++++++++++++++++ ofborg/src/ghevent/mod.rs | 2 +- 5 files changed, 578 insertions(+), 3 deletions(-) create mode 100644 ofborg-send-event/Cargo.toml create mode 100644 ofborg-send-event/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 6dd57a9e..65e10b92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -417,6 +467,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.10.0" @@ -452,6 +508,46 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -479,6 +575,12 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -825,6 +927,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1087,9 +1198,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1372,18 +1485,23 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1563,6 +1681,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -1751,6 +1875,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -1772,6 +1902,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1978,6 +2114,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "ofborg-send-event" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "hex", + "hmac 0.13.0", + "octocrab", + "ofborg", + "reqwest", + "serde_json", + "sha2 0.11.0", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "ofborg-simple-build" version = "0.1.0" @@ -2005,6 +2159,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2279,6 +2439,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -2307,10 +2523,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.1" @@ -2332,6 +2558,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2341,6 +2577,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -2394,6 +2639,46 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -2444,6 +2729,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2524,6 +2815,7 @@ version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ + "web-time", "zeroize", ] @@ -2792,6 +3084,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -2898,6 +3200,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2920,6 +3228,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -3079,7 +3390,9 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -3302,6 +3615,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3381,6 +3700,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.122" @@ -3447,6 +3776,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index d1db03ac..09f3c94d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ - "ofborg", + "ofborg", "ofborg-send-event", "ofborg-simple-build" ] resolver = "2" diff --git a/ofborg-send-event/Cargo.toml b/ofborg-send-event/Cargo.toml new file mode 100644 index 00000000..f355b4b2 --- /dev/null +++ b/ofborg-send-event/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "ofborg-send-event" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1" +clap = { version = "4", features = ["derive"] } +hex = "0.4" +hmac = "0.13" +reqwest = { version = "0.13", features = ["json"] } +sha2 = "0.11" +tokio = { version = "1", features = ["full"] } +uuid = { version = "1", features = ["v4"] } +serde_json = "1.0.135" +tracing = "0.1.41" + +octocrab = { version = "0.51.0", default-features = false, features = [ + "default-client", + "follow-redirect", + "jwt-rust-crypto", + "retry", + "rustls", + # Enabling `rustls-ring` leads to runtime panics: https://github.com/XAMPPRocky/octocrab/issues/855 + # "rustls-ring", + "timeout", + "tracing", +] } +ofborg = { path = "../ofborg" } diff --git a/ofborg-send-event/src/main.rs b/ofborg-send-event/src/main.rs new file mode 100644 index 00000000..7704da84 --- /dev/null +++ b/ofborg-send-event/src/main.rs @@ -0,0 +1,207 @@ +use anyhow::{Context as _, Result, bail}; +use clap::Parser; +use hmac::KeyInit; +use hmac::{Hmac, Mac as _}; +use octocrab::Octocrab; +use ofborg::ghevent::{ + PullRequest, PullRequestAction, PullRequestEvent, PullRequestRef, PullRequestState, Repository, + User, +}; +use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT}; +use sha2::Sha256; +use std::str::FromStr as _; +use uuid::Uuid; + +#[derive(Debug, Parser)] +#[command(about = "Send a GitHub-style webhook to a local or staging receiver")] +struct Args { + /// Destination webhook URL + #[arg(long, default_value = "http://localhost:9899")] + webhook_receiver_url: String, + + /// PR that should be fetched + #[arg(long)] + pr_nr: u64, + + /// Webhook event name, e.g. push, pull_request, ping + #[arg(long, default_value = "pull_request")] + event: String, + + /// Shared secret path used to generate X-Hub-Signature-256 + #[arg(long)] + secret_path: Option, + + /// Delivery ID header. Defaults to a random UUID. + #[arg(long)] + delivery_id: Option, + + /// Add arbitrary header(s), format: Name=Value + #[arg(long = "header")] + headers: Vec, + + /// Print response headers too + #[arg(long)] + verbose: bool, + + /// Timeout in seconds + #[arg(long, default_value_t = 30)] + timeout_secs: u64, + + /// Full Repo Name which should be used as repo + #[arg(long, default_value = "NixOS/nixpkgs")] + full_repo_name: String, +} + +async fn make_pull_request_body(full_repo_name: &str, number: u64) -> Result { + let (org_name, repo_name) = full_repo_name + .split_once("/") + .with_context(|| format!("Unexpected Full Repo Name! {full_repo_name}"))?; + + let octocrab = Octocrab::builder() + .build() + .context("failed to create Octocrab client")?; + let pr = octocrab + .pulls(org_name, repo_name) + .get(number) + .await + .with_context(|| format!("failed to fetch PR {number}"))?; + tracing::info!("Fetched PR {number}: {}", pr.title); + + Ok(PullRequestEvent { + action: PullRequestAction::Opened, + number, + repository: Repository { + owner: User { + login: org_name.to_owned(), + }, + name: repo_name.to_owned(), + full_name: format!("{org_name}/{repo_name}"), + clone_url: format!("https://github.com/{org_name}/{repo_name}.git"), + }, + pull_request: PullRequest { + state: PullRequestState::Open, + base: PullRequestRef { + git_ref: pr.base.ref_field, + sha: pr.base.sha, + }, + head: PullRequestRef { + git_ref: pr.head.ref_field, + sha: pr.head.sha, + }, + }, + changes: None, + }) +} + +#[tokio::main] +async fn main() -> Result<()> { + ofborg::setup_log(); + let args = Args::parse(); + + let event = make_pull_request_body(&args.full_repo_name, args.pr_nr).await?; + let body = serde_json::to_vec(&event)?; + + let delivery_id = args + .delivery_id + .unwrap_or_else(|| Uuid::new_v4().to_string()); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(args.timeout_secs)) + .build() + .context("failed to build reqwest client")?; + + let mut headers = HeaderMap::new(); + headers.insert( + USER_AGENT, + HeaderValue::from_static("ofborg-send-event/0.1"), + ); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + HeaderName::from_static("x-github-event"), + HeaderValue::from_str(&args.event).context("invalid X-GitHub-Event value")?, + ); + headers.insert( + HeaderName::from_static("x-github-delivery"), + HeaderValue::from_str(&delivery_id).context("invalid X-GitHub-Delivery value")?, + ); + + if let Some(secret_path) = &args.secret_path { + let signature = github_signature_256(&std::fs::read_to_string(secret_path)?.trim(), &body)?; + headers.insert( + HeaderName::from_static("x-hub-signature-256"), + HeaderValue::from_str(&signature).context("invalid X-Hub-Signature-256 value")?, + ); + } + + for raw in &args.headers { + let (name, value) = parse_header(raw)?; + headers.insert(name, value); + } + + let response = client + .post(&args.webhook_receiver_url) + .headers(headers) + .body(body) + .send() + .await + .with_context(|| format!("failed to POST to {}", args.webhook_receiver_url))?; + + let status = response.status(); + let resp_headers = response.headers().clone(); + let resp_body = response + .text() + .await + .context("failed to read response body")?; + + println!("status: {}", status); + + if args.verbose { + println!("response headers:"); + for (name, value) in resp_headers.iter() { + println!( + "{}: {}", + name.as_str(), + value.to_str().unwrap_or("") + ); + } + } + + if !resp_body.is_empty() { + println!(); + println!("{}", resp_body); + } + + Ok(()) +} + +fn github_verify_signature(secret: &str, body: &[u8], tag: &[u8]) -> Result { + let mut mac = + Hmac::::new_from_slice(secret.as_bytes()).context("invalid HMAC secret bytes")?; + mac.update(body); + Ok(mac.verify_slice(tag).is_ok()) +} + +fn github_signature_256(secret: &str, body: &[u8]) -> Result { + let mut mac = + Hmac::::new_from_slice(secret.as_bytes()).context("invalid HMAC secret bytes")?; + mac.update(body); + let tag = mac.finalize().into_bytes(); + assert!(github_verify_signature(secret, body, &tag)?); + Ok(format!("sha256={}", hex::encode(tag))) +} + +fn parse_header(input: &str) -> Result<(HeaderName, HeaderValue)> { + let Some((name, value)) = input.split_once('=') else { + bail!( + "invalid --header format: expected Name=Value, got {}", + input + ); + }; + + let name = HeaderName::from_str(name.trim()) + .with_context(|| format!("invalid header name: {}", name.trim()))?; + let value = HeaderValue::from_str(value.trim()) + .with_context(|| format!("invalid header value for {}", name))?; + + Ok((name, value)) +} diff --git a/ofborg/src/ghevent/mod.rs b/ofborg/src/ghevent/mod.rs index 24375880..98183242 100644 --- a/ofborg/src/ghevent/mod.rs +++ b/ofborg/src/ghevent/mod.rs @@ -5,5 +5,5 @@ mod pullrequestevent; pub use self::common::{Comment, GenericWebhook, Issue, Repository, User}; pub use self::issuecomment::{IssueComment, IssueCommentAction}; pub use self::pullrequestevent::{ - PullRequest, PullRequestAction, PullRequestEvent, PullRequestState, + PullRequest, PullRequestAction, PullRequestEvent, PullRequestRef, PullRequestState, }; From da654fdad3e5bb2eb79631b46aa911c14c6ab62b Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Thu, 23 Apr 2026 17:37:00 +0200 Subject: [PATCH 05/17] fix: stats not correctly starting a http_server --- ofborg/src/bin/stats.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofborg/src/bin/stats.rs b/ofborg/src/bin/stats.rs index 72b04e97..ddd1cd26 100644 --- a/ofborg/src/bin/stats.rs +++ b/ofborg/src/bin/stats.rs @@ -118,7 +118,7 @@ async fn main() -> Result<(), Box> { // Spawn HTTP server in a separate thread with its own tokio runtime let metrics_clone = metrics.clone(); - std::thread::spawn(async move || { + tokio::task::spawn(async move { let addr: SocketAddr = "0.0.0.0:9898".parse().unwrap(); if let Err(e) = run_http_server(addr, metrics_clone).await { error!("HTTP server error: {:?}", e); From 29faf84df236acd7be167ccf4668a3b1a0bcbb42 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 24 Apr 2026 16:33:13 +0200 Subject: [PATCH 06/17] chore: add anyhow::Result --- Cargo.lock | 1 + ofborg/Cargo.toml | 1 + ofborg/src/bin/build-faker.rs | 9 ++++----- ofborg/src/bin/builder.rs | 8 +++----- ofborg/src/bin/evaluation-filter.rs | 7 ++----- ofborg/src/bin/github-comment-filter.rs | 7 ++----- ofborg/src/bin/github-comment-poster.rs | 7 ++----- ofborg/src/bin/github-webhook-receiver.rs | 8 +++----- ofborg/src/bin/log-message-collector.rs | 10 +++------- ofborg/src/bin/logapi.rs | 19 ++++++++----------- ofborg/src/bin/mass-rebuilder.rs | 6 ++---- ofborg/src/bin/stats.rs | 8 +++----- 12 files changed, 34 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65e10b92..312a7397 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2083,6 +2083,7 @@ dependencies = [ name = "ofborg" version = "0.1.9" dependencies = [ + "anyhow", "async-trait", "brace-expand", "chrono", diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index be2affea..1739c939 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -6,6 +6,7 @@ build = "build.rs" edition = "2024" [dependencies] +anyhow = "1.0.0" async-trait = "0.1.89" brace-expand = "0.1.0" chrono = { version = "0.4.38", default-features = false, features = [ diff --git a/ofborg/src/bin/build-faker.rs b/ofborg/src/bin/build-faker.rs index 8e942e3d..c8475352 100644 --- a/ofborg/src/bin/build-faker.rs +++ b/ofborg/src/bin/build-faker.rs @@ -1,6 +1,3 @@ -use std::env; -use std::error::Error; - use lapin::message::Delivery; use ofborg::commentparser; @@ -11,10 +8,12 @@ use ofborg::notifyworker::NotificationReceiver; use ofborg::worker; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args().nth(1).expect("usage: build-faker "); + let arg = std::env::args() + .nth(1) + .expect("usage: build-faker "); let cfg = config::load(arg.as_ref()); let conn = easylapin::from_config(&cfg.builder.unwrap().rabbitmq).await?; diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs index 85c7b1e2..5f4c56aa 100644 --- a/ofborg/src/bin/builder.rs +++ b/ofborg/src/bin/builder.rs @@ -1,5 +1,3 @@ -use std::env; -use std::error::Error; use std::future::Future; use std::path::Path; use std::pin::Pin; @@ -12,10 +10,10 @@ use ofborg::easylapin; use ofborg::{checkout, config, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); @@ -44,7 +42,7 @@ async fn create_handle( conn: &lapin::Connection, cfg: &config::Config, system: String, -) -> Result + Send>>, Box> { +) -> anyhow::Result + Send>>> { let mut chan = conn.create_channel().await?; let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); diff --git a/ofborg/src/bin/evaluation-filter.rs b/ofborg/src/bin/evaluation-filter.rs index f35d6001..edba6848 100644 --- a/ofborg/src/bin/evaluation-filter.rs +++ b/ofborg/src/bin/evaluation-filter.rs @@ -1,16 +1,13 @@ -use std::env; -use std::error::Error; - use tracing::{error, info}; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index cc745529..3fca4424 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -1,6 +1,3 @@ -use std::env; -use std::error::Error; - use ofborg::systems::System; use tracing::{error, info}; @@ -8,10 +5,10 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/github-comment-poster.rs b/ofborg/src/bin/github-comment-poster.rs index 9c19d65d..6b3e8278 100644 --- a/ofborg/src/bin/github-comment-poster.rs +++ b/ofborg/src/bin/github-comment-poster.rs @@ -1,16 +1,13 @@ -use std::env; -use std::error::Error; - use tracing::{error, info}; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index cff4e515..cce398cd 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -1,5 +1,3 @@ -use std::env; -use std::error::Error; use std::net::SocketAddr; use std::sync::Arc; @@ -23,7 +21,7 @@ use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; /// Prepares the the exchange we will write to, the queues that are bound to it /// and binds them. -async fn setup_amqp(chan: &mut Channel) -> Result<(), Box> { +async fn setup_amqp(chan: &mut Channel) -> anyhow::Result<()> { chan.declare_exchange(easyamqp::ExchangeConfig { exchange: "github-events".to_owned(), exchange_type: easyamqp::ExchangeType::Topic, @@ -236,10 +234,10 @@ async fn handle_request( } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let Some(cfg) = config::load(arg.as_ref()).github_webhook_receiver else { diff --git a/ofborg/src/bin/log-message-collector.rs b/ofborg/src/bin/log-message-collector.rs index 1a284345..81715915 100644 --- a/ofborg/src/bin/log-message-collector.rs +++ b/ofborg/src/bin/log-message-collector.rs @@ -1,17 +1,13 @@ -use std::env; -use std::error::Error; -use std::path::PathBuf; - use tracing::{error, info}; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); @@ -58,7 +54,7 @@ async fn main() -> Result<(), Box> { let handle = chan .consume( tasks::log_message_collector::LogMessageCollector::new( - PathBuf::from(collector_cfg.logs_path), + std::path::PathBuf::from(collector_cfg.logs_path), 100, ), easyamqp::ConsumeConfig { diff --git a/ofborg/src/bin/logapi.rs b/ofborg/src/bin/logapi.rs index b47e1acf..bba4c6ce 100644 --- a/ofborg/src/bin/logapi.rs +++ b/ofborg/src/bin/logapi.rs @@ -1,6 +1,3 @@ -use std::net::SocketAddr; -use std::{collections::HashMap, error::Error, path::PathBuf, sync::Arc}; - use http::{Method, StatusCode}; use http_body_util::Full; use hyper::body::Bytes; @@ -21,12 +18,12 @@ struct Attempt { #[derive(serde::Serialize)] struct LogResponse { - attempts: HashMap, + attempts: std::collections::HashMap, } #[derive(Clone)] struct LogApiConfig { - logs_path: PathBuf, + logs_path: std::path::PathBuf, serve_root: String, } @@ -47,7 +44,7 @@ fn json_response(status: StatusCode, body: String) -> Response> { async fn handle_request( req: Request, - cfg: Arc, + cfg: std::sync::Arc, ) -> Result>, hyper::Error> { if req.method() != Method::GET { return Ok(response(StatusCode::METHOD_NOT_ALLOWED, "")); @@ -57,7 +54,7 @@ async fn handle_request( let Some(reqd) = uri.strip_prefix("/logs/").map(ToOwned::to_owned) else { return Ok(response(StatusCode::NOT_FOUND, "invalid uri")); }; - let path: PathBuf = cfg.logs_path.join(&reqd); + let path: std::path::PathBuf = cfg.logs_path.join(&reqd); let Ok(path) = std::fs::canonicalize(&path) else { return Ok(response(StatusCode::NOT_FOUND, "absent")); }; @@ -68,7 +65,7 @@ async fn handle_request( return Ok(response(StatusCode::NOT_FOUND, "non dir")); }; - let mut attempts = HashMap::::new(); + let mut attempts = std::collections::HashMap::::new(); for e in iter { let Ok(e) = e else { continue }; let e_metadata = e.metadata(); @@ -117,7 +114,7 @@ async fn handle_request( } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); let arg = std::env::args() @@ -131,12 +128,12 @@ async fn main() -> Result<(), Box> { let logs_path = std::fs::canonicalize(&cfg.logs_path) .expect("logs_path does not exist or is not accessible"); - let api_cfg = Arc::new(LogApiConfig { + let api_cfg = std::sync::Arc::new(LogApiConfig { logs_path, serve_root: cfg.serve_root, }); - let addr: SocketAddr = cfg.listen.parse()?; + let addr: std::net::SocketAddr = cfg.listen.parse()?; let listener = TcpListener::bind(addr).await?; info!("Listening on {}", addr); diff --git a/ofborg/src/bin/mass-rebuilder.rs b/ofborg/src/bin/mass-rebuilder.rs index 10877590..cead74c2 100644 --- a/ofborg/src/bin/mass-rebuilder.rs +++ b/ofborg/src/bin/mass-rebuilder.rs @@ -1,5 +1,3 @@ -use std::env; -use std::error::Error; use std::path::Path; use tracing::{error, info}; @@ -12,10 +10,10 @@ use ofborg::stats; use ofborg::tasks; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/stats.rs b/ofborg/src/bin/stats.rs index ddd1cd26..ed94530a 100644 --- a/ofborg/src/bin/stats.rs +++ b/ofborg/src/bin/stats.rs @@ -1,5 +1,3 @@ -use std::env; -use std::error::Error; use std::net::SocketAddr; use std::sync::Arc; @@ -26,7 +24,7 @@ fn response(body: String) -> Response> { async fn run_http_server( addr: SocketAddr, metrics: Arc, -) -> Result<(), Box> { +) -> anyhow::Result<()> { let listener = TcpListener::bind(addr).await?; info!("HTTP server listening on {}", addr); @@ -50,10 +48,10 @@ async fn run_http_server( } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); From 4e60e01e01b26cace741e8e1c2d9c17aa5f0060b Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 24 Apr 2026 16:43:45 +0200 Subject: [PATCH 07/17] fix: enable info log by default in shell --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 9c9263de..1f642f7e 100644 --- a/flake.nix +++ b/flake.nix @@ -69,7 +69,7 @@ RUSTFLAGS = "-D warnings"; RUST_BACKTRACE = "1"; - RUST_LOG = "ofborg=debug"; + RUST_LOG = "ofborg=debug,info"; NIX_PATH = "nixpkgs=${pkgs.path}"; }; } From 75b4a18b5a7207157a6724e5d3dbac3ac33460a7 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 24 Apr 2026 16:44:25 +0200 Subject: [PATCH 08/17] feat: optional github_app --- ofborg/src/bin/github-comment-poster.rs | 4 +- ofborg/src/commitstatus.rs | 10 +++++ ofborg/src/config.rs | 8 ++-- ofborg/src/tasks/evaluate.rs | 59 +++++++++++++++++-------- 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/ofborg/src/bin/github-comment-poster.rs b/ofborg/src/bin/github-comment-poster.rs index 6b3e8278..03e2d630 100644 --- a/ofborg/src/bin/github-comment-poster.rs +++ b/ofborg/src/bin/github-comment-poster.rs @@ -51,7 +51,9 @@ async fn main() -> anyhow::Result<()> { let handle = easylapin::WorkerChannel(chan) .consume( - tasks::githubcommentposter::GitHubCommentPoster::new(cfg.github_app_vendingmachine()), + tasks::githubcommentposter::GitHubCommentPoster::new( + cfg.github_app_vendingmachine().unwrap(), + ), easyamqp::ConsumeConfig { queue: "build-results".to_owned(), consumer_tag: format!("{}-github-comment-poster", cfg.whoami()), diff --git a/ofborg/src/commitstatus.rs b/ofborg/src/commitstatus.rs index 751ccb24..bcc86216 100644 --- a/ofborg/src/commitstatus.rs +++ b/ofborg/src/commitstatus.rs @@ -9,6 +9,7 @@ pub struct CommitStatus { context: String, description: String, url: String, + enable_publish: bool, } impl CommitStatus { @@ -25,9 +26,14 @@ impl CommitStatus { context, description, url: url.unwrap_or_else(|| String::from("")), + enable_publish: true, } } + pub fn set_enable_publish(&mut self, enable_publish: bool) { + self.enable_publish = enable_publish; + } + pub fn set_url(&mut self, url: Option) { self.url = url.unwrap_or_else(|| String::from("")) } @@ -46,6 +52,10 @@ impl CommitStatus { } pub async fn set(&self, state: StatusState) -> Result<(), CommitStatusError> { + if !self.enable_publish { + return Ok(()); + } + let desc = if self.description.len() >= 140 { warn!( "description is over 140 char; truncating: {:?}", diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index a5f33a5d..77143100 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -249,12 +249,12 @@ impl Config { .expect("Unable to create a github client instance") } - pub fn github_app_vendingmachine(&self) -> GithubAppVendingMachine { - GithubAppVendingMachine { - conf: self.github_app.clone().unwrap(), + pub fn github_app_vendingmachine(&self) -> Option { + Some(GithubAppVendingMachine { + conf: self.github_app.clone()?, id_cache: HashMap::new(), client_cache: HashMap::new(), - } + }) } pub fn nix(&self) -> Nix { diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index 31cc0997..eb595a92 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -18,7 +18,7 @@ use crate::{checkout, systems, worker}; pub struct EvaluationWorker { cloner: checkout::CachedCloner, - github_vend: tokio::sync::RwLock, + github_vend: Option>, acl: Acl, identity: String, events: E, @@ -27,14 +27,14 @@ pub struct EvaluationWorker { impl EvaluationWorker { pub fn new( cloner: checkout::CachedCloner, - github_vend: GithubAppVendingMachine, + github_vend: Option, acl: Acl, identity: String, events: E, ) -> EvaluationWorker { EvaluationWorker { cloner, - github_vend: tokio::sync::RwLock::new(github_vend), + github_vend: github_vend.map(tokio::sync::RwLock::new), acl, identity, events, @@ -71,13 +71,13 @@ impl worker::SimpleWorker for EvaluationWorker async fn consumer(&mut self, job: &evaluationjob::EvaluationJob) -> worker::Actions { let span = debug_span!("job", pr = ?job.pr.number); async { - let github_client = { - let mut vending_machine = self.github_vend.write().await; + let github_client = if let Some(github_vend) = self.github_vend.as_ref() { + let mut vending_machine = github_vend.write().await; match vending_machine .for_repo(&job.repo.owner, &job.repo.name) .await { - Some(client) => client.clone(), + Some(client) => Some(client.clone()), None => { error!( "Failed to get a github client token for {}/{}", @@ -86,10 +86,12 @@ impl worker::SimpleWorker for EvaluationWorker return vec![worker::Action::NackRequeue]; } } + } else { + None }; OneEval::new( - &github_client, + github_client, &self.acl, &mut self.events, &self.identity, @@ -106,6 +108,7 @@ impl worker::SimpleWorker for EvaluationWorker struct OneEval<'a, E> { repo: GithubRepo, + enable_publish: bool, acl: &'a Acl, events: &'a mut E, identity: &'a str, @@ -117,19 +120,28 @@ struct OneEval<'a, E> { impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { #[allow(clippy::too_many_arguments)] fn new( - octocrab: &'a Octocrab, + octocrab: Option, acl: &'a Acl, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, ) -> OneEval<'a, E> { + let (repo, enable_publish) = if let Some(octocrab) = octocrab { + ( + GithubRepo::new(octocrab, job.repo.owner.clone(), job.repo.name.clone()), + true, + ) + } else { + let octocrab = Octocrab::builder().build().unwrap(); + ( + GithubRepo::new(octocrab, job.repo.owner.clone(), job.repo.name.clone()), + false, + ) + }; OneEval { - repo: GithubRepo::new( - octocrab.clone(), - job.repo.owner.clone(), - job.repo.name.clone(), - ), + repo, + enable_publish, acl, events, identity, @@ -149,6 +161,10 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { url: Option, state: StatusState, ) -> Result<(), CommitStatusError> { + if !self.enable_publish { + return Ok(()); + } + let prefix = self .prefix .expect("prefix should have been set in worker_actions"); @@ -193,8 +209,8 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { match eval_result { Ok(eval_actions) => { - let issue = self.repo.issues().get(self.job.pr.number).await; - if let Ok(issue) = issue + if self.enable_publish + && let Ok(issue) = self.repo.issues().get(self.job.pr.number).await && let Err(e) = self .repo .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) @@ -209,8 +225,8 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { // There was an error during eval, but we successfully // updated the PR. - let issue = self.repo.issues().get(self.job.pr.number).await; - if let Ok(issue) = issue + if self.enable_publish + && let Ok(issue) = self.repo.issues().get(self.job.pr.number).await && let Err(e) = self .repo .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) @@ -279,12 +295,15 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { "Starting".to_owned(), None, ); + overall_status.set_enable_publish(self.enable_publish); overall_status .set_with_description("Starting", StatusState::Pending) .await?; - evaluation_strategy.pre_clone(&self.repo).await?; + if self.enable_publish { + evaluation_strategy.pre_clone(&self.repo).await?; + } let project = self .cloner @@ -394,6 +413,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .set_with_description("Beginning Evaluations", StatusState::Pending) .await?; + let enable_publish = self.enable_publish; let eval_results: bool = futures::stream::iter(evaluation_strategy.evaluation_checks()) .map(|check| { let repo = self.repo.clone(); @@ -401,13 +421,14 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { let refpath = refpath.clone(); async move { - let status = CommitStatus::new( + let mut status = CommitStatus::new( repo, head_sha, format!("{prefix}-eval-{}", check.name()), check.cli_cmd(), None, ); + status.set_enable_publish(enable_publish); if let Err(e) = status.set(StatusState::Pending).await { warn!("Failed to set pending status on eval strategy: {e:?}"); From 8792e3ae792ba8d784aedec9f55432fffb024f13 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 24 Apr 2026 18:00:53 +0200 Subject: [PATCH 09/17] feat: tests for github-webhook-receiver --- Cargo.lock | 2 + ofborg/Cargo.toml | 6 + ofborg/src/bin/github-webhook-receiver.rs | 379 ++++++++++++++++++++-- ofborg/src/lib.rs | 8 + ofborg/src/test_utils.rs | 42 +++ 5 files changed, 410 insertions(+), 27 deletions(-) create mode 100644 ofborg/src/test_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 312a7397..befd8f84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2086,6 +2086,7 @@ dependencies = [ "anyhow", "async-trait", "brace-expand", + "bytes", "chrono", "fs2", "futures", @@ -2093,6 +2094,7 @@ dependencies = [ "hex", "hmac 0.13.0", "http", + "http-body", "http-body-util", "hyper", "hyper-util", diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index 1739c939..cc38872a 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -9,6 +9,7 @@ edition = "2024" anyhow = "1.0.0" async-trait = "0.1.89" brace-expand = "0.1.0" +bytes = "1.11.1" chrono = { version = "0.4.38", default-features = false, features = [ "clock", "std", @@ -19,6 +20,7 @@ futures-util = "0.3.31" hex = "0.4.3" hmac = "0.13.0" http = "1" +http-body = "1" http-body-util = "0.1" octocrab = { version = "0.51.0", default-features = false, features = [ "default-client", @@ -49,3 +51,7 @@ tokio-stream = "0.1" tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = ["json", "env-filter"] } uuid = { version = "1.12", features = ["v4"] } + +[dev-dependencies] +bytes = "1.11.1" +http-body = "1.0.1" diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index cce398cd..0bd963fd 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -1,27 +1,55 @@ use std::net::SocketAddr; use std::sync::Arc; -use hmac::{Hmac, KeyInit as _, Mac}; +use async_trait::async_trait; +use hmac::{Hmac, KeyInit as _, Mac as _}; use http::{Method, StatusCode}; -use http_body_util::{BodyExt, Full}; -use hyper::body::{Bytes, Incoming}; +use http_body_util::{BodyExt as _, Full}; +use hyper::body::Bytes; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Request, Response}; use hyper_util::rt::TokioIo; -use lapin::options::BasicPublishOptions; -use lapin::{BasicProperties, Channel}; use sha2::Sha256; use tokio::net::TcpListener; use tokio::sync::Mutex; use tracing::{error, info, warn}; use ofborg::ghevent::GenericWebhook; -use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; +use ofborg::{MessagePublisher, config, easyamqp, easyamqp::ChannelExt, easylapin}; + +pub struct LapinPublisher { + chan: Arc>, +} + +impl LapinPublisher { + pub fn new(chan: Arc>) -> Self { + Self { chan } + } +} + +#[async_trait] +impl MessagePublisher for LapinPublisher { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()> { + let chan = self.chan.lock().await; + let _confirmation = chan + .basic_publish( + exchange.into(), + routing_key.into(), + lapin::options::BasicPublishOptions::default(), + body, + lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2), + ) + .await?; + Ok(()) + } +} /// Prepares the the exchange we will write to, the queues that are bound to it /// and binds them. -async fn setup_amqp(chan: &mut Channel) -> anyhow::Result<()> { +async fn setup_amqp(chan: &mut lapin::Channel) -> anyhow::Result<()> { chan.declare_exchange(easyamqp::ExchangeConfig { exchange: "github-events".to_owned(), exchange_type: easyamqp::ExchangeType::Topic, @@ -103,11 +131,16 @@ fn empty_response(status: StatusCode) -> Response> { .unwrap() } -async fn handle_request( - req: Request, +async fn handle_request( + req: Request, webhook_secret: Arc, - chan: Arc>, -) -> Result>, hyper::Error> { + publisher: Arc, +) -> Result>, hyper::Error> +where + B: http_body::Body + Send + Sync, + F: bytes::Buf, + B::Error: std::fmt::Debug + Send + Sync, +{ // HTTP 405 if req.method() != Method::POST { return Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED)); @@ -134,7 +167,7 @@ async fn handle_request( let raw = match req.collect().await { Ok(collected) => collected.to_bytes(), Err(e) => { - warn!("Failed to read body from client: {e}"); + warn!("Failed to read body from client: {e:?}"); return Ok(response( StatusCode::INTERNAL_SERVER_ERROR, "Failed to read body", @@ -217,18 +250,13 @@ async fn handle_request( let routing_key = format!("{event_type}.{}", input.repository.full_name.to_lowercase()); // Publish message - let chan = chan.lock().await; - let _confirmation = chan - .basic_publish( - "github-events".into(), - routing_key.as_str().into(), - BasicPublishOptions::default(), - &raw, - BasicProperties::default() - .with_content_type("application/json".into()) - .with_delivery_mode(2), // persistent - ) - .await; + if let Err(e) = publisher.publish("github-events", &routing_key, &raw).await { + error!("Failed to publish message: {e}"); + return Ok(response( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to publish message", + )); + } Ok(empty_response(StatusCode::NO_CONTENT)) } @@ -253,6 +281,7 @@ async fn main() -> anyhow::Result<()> { let mut chan = conn.create_channel().await?; setup_amqp(&mut chan).await?; let chan = Arc::new(Mutex::new(chan)); + let publisher: Arc = Arc::new(LapinPublisher::new(chan.clone())); let addr: SocketAddr = cfg.listen.parse()?; let listener = TcpListener::bind(addr).await?; @@ -263,11 +292,12 @@ async fn main() -> anyhow::Result<()> { let io = TokioIo::new(stream); let webhook_secret = webhook_secret.clone(); - let chan = chan.clone(); + let publisher = publisher.clone(); tokio::task::spawn(async move { - let service = - service_fn(move |req| handle_request(req, webhook_secret.clone(), chan.clone())); + let service = service_fn(move |req| { + handle_request(req, webhook_secret.clone(), publisher.clone()) + }); if let Err(err) = http1::Builder::new().serve_connection(io, service).await { warn!("Error serving connection: {:?}", err); @@ -275,3 +305,298 @@ async fn main() -> anyhow::Result<()> { }); } } + +mod github_webhook_receiver { + #[cfg(test)] + mod test { + use super::super::*; + use http::header; + + use hyper::body::Bytes; + use ofborg::test_utils::MockPublisher; + + fn create_request( + method: http::Method, + headers: Vec<(http::header::HeaderName, http::header::HeaderValue)>, + body: Bytes, + ) -> http::Request> { + let mut builder = http::Request::builder().method(method); + for (name, value) in headers { + builder = builder.header(name, value); + } + let body = http_body_util::Full::new(body); + builder.body(body).unwrap() + } + + fn hv(s: &str) -> http::header::HeaderValue { + http::header::HeaderValue::from_str(s).unwrap() + } + + fn hn(s: &'static str) -> http::header::HeaderName { + http::header::HeaderName::from_bytes(s.as_bytes()).unwrap() + } + + fn compute_signature(secret: &str, body: &[u8]) -> String { + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); + mac.update(body); + let result = mac.finalize(); + format!("sha256={}", hex::encode(result.into_bytes())) + } + + fn valid_headers( + secret: &str, + body: &[u8], + ) -> Vec<(header::HeaderName, header::HeaderValue)> { + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature(secret, body)), + ), + (hn("X-Github-Event"), hv("pull_request")), + ] + } + + fn minimal_valid_webhook() -> &'static str { + r#"{"repository":{"owner":{"login":"test"},"name":"test-repo","full_name":"test/test-repo","clone_url":"https://github.com/test/test-repo.git"}}"# + } + + fn pr_event_webhook() -> &'static str { + include_str!("../../test-srcs/events/pr-changed-base.json") + } + + #[tokio::test] + async fn test_method_not_allowed() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let req = create_request(http::Method::GET, vec![], Bytes::new()); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + } + + #[tokio::test] + async fn test_missing_signature_header() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![(header::CONTENT_TYPE, hv("application/json"))], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_signature_hash_method() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + (hn("X-Hub-Signature-256"), hv("sha1=abc123")), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_signature_verification_failed() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv( + "sha256=0000000000000000000000000000000000000000000000000000000000000000", + ), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_missing_content_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + )], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_content_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("text/plain")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_json() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from("not valid json {{{"); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", b"not valid json {{{")), + ), + (hn("X-Github-Event"), hv("pull_request")), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_missing_event_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_successful_webhook_with_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body_bytes = pr_event_webhook().as_bytes(); + let req = create_request( + http::Method::POST, + valid_headers("test-secret", body_bytes), + Bytes::from(pr_event_webhook()), + ); + + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published.len(), 1); + assert_eq!(published[0].exchange, "github-events"); + assert_eq!(published[0].routing_key, "pull_request.nixos/nixpkgs"); + assert_eq!(&published[0].body, pr_event_webhook().as_bytes()); + } + + #[tokio::test] + async fn test_issue_comment_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"test"},"name":"my-repo","full_name":"test/my-repo","clone_url":"https://github.com/test/my-repo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("issue_comment")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "issue_comment.test/my-repo"); + } + + #[tokio::test] + async fn test_routing_key_lowercases_repo_name() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"Test"},"name":"MyRepo","full_name":"Test/MyRepo","clone_url":"https://github.com/Test/MyRepo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("pull_request")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "pull_request.test/myrepo"); + } + } +} diff --git a/ofborg/src/lib.rs b/ofborg/src/lib.rs index 85bb0053..1383efc7 100644 --- a/ofborg/src/lib.rs +++ b/ofborg/src/lib.rs @@ -34,7 +34,15 @@ pub mod systems; pub mod tagger; pub mod tasks; pub mod test_scratch; +pub mod test_utils; pub mod worker; + +use async_trait::async_trait; + +#[async_trait] +pub trait MessagePublisher: Send + Sync { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()>; +} pub mod writetoline; pub mod ofborg { diff --git a/ofborg/src/test_utils.rs b/ofborg/src/test_utils.rs new file mode 100644 index 00000000..559f4588 --- /dev/null +++ b/ofborg/src/test_utils.rs @@ -0,0 +1,42 @@ +use async_trait::async_trait; + +#[derive(Debug, Clone)] +pub struct PublishedMessage { + pub exchange: String, + pub routing_key: String, + pub body: Vec, +} + +#[derive(Debug, Default)] +pub struct MockPublisher { + published: tokio::sync::Mutex>, +} + +impl MockPublisher { + pub fn new() -> Self { + Self { + published: tokio::sync::Mutex::new(Vec::new()), + } + } + + pub async fn get_published(&self) -> Vec { + self.published.lock().await.clone() + } + + pub async fn clear(&self) { + self.published.lock().await.clear(); + } +} + +#[async_trait] +impl crate::MessagePublisher for MockPublisher { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()> { + let mut guard = self.published.lock().await; + guard.push(PublishedMessage { + exchange: exchange.to_string(), + routing_key: routing_key.to_string(), + body: body.to_vec(), + }); + Ok(()) + } +} From dfd5012d4a7efac1b0478e4c550e14fa4de125dd Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Mon, 27 Apr 2026 09:39:32 +0200 Subject: [PATCH 10/17] feat: make stats socket bin configurable --- ofborg/src/bin/stats.rs | 9 +++++++-- ofborg/src/config.rs | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ofborg/src/bin/stats.rs b/ofborg/src/bin/stats.rs index ed94530a..3bd345f9 100644 --- a/ofborg/src/bin/stats.rs +++ b/ofborg/src/bin/stats.rs @@ -1,7 +1,8 @@ use std::net::SocketAddr; use std::sync::Arc; -use http::StatusCode; +use http::header::CONTENT_TYPE; +use http::{HeaderValue, StatusCode}; use http_body_util::Full; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -17,6 +18,10 @@ use ofborg::{config, easyamqp, easylapin, stats, tasks}; fn response(body: String) -> Response> { Response::builder() .status(StatusCode::OK) + .header( + CONTENT_TYPE, + HeaderValue::from_static("text/plain; version=0.0.4; charset=utf-8"), + ) .body(Full::new(Bytes::from(body))) .unwrap() } @@ -60,6 +65,7 @@ async fn main() -> anyhow::Result<()> { error!("No stats configuration found!"); panic!(); }; + let addr: SocketAddr = stats_cfg.listen.parse().unwrap(); let conn = easylapin::from_config(&stats_cfg.rabbitmq).await?; @@ -117,7 +123,6 @@ async fn main() -> anyhow::Result<()> { // Spawn HTTP server in a separate thread with its own tokio runtime let metrics_clone = metrics.clone(); tokio::task::spawn(async move { - let addr: SocketAddr = "0.0.0.0:9898".parse().unwrap(); if let Err(e) = run_http_server(addr, metrics_clone).await { error!("HTTP server error: {:?}", e); } diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 77143100..89fd5f06 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -126,6 +126,8 @@ pub struct LogMessageCollector { #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct Stats { + /// Listen host/port + pub listen: String, /// RabbitMQ broker to connect to pub rabbitmq: RabbitMqConfig, } From 5f4988f971490166a971d844ecebc65cbddb0422 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 11:02:06 +0200 Subject: [PATCH 11/17] feat: add mprocs for local setup --- .gitignore | 1 + flake.nix | 1 + mprocs.yaml | 47 +++++++++++++++++++++++ mprocs/bootstrap-ofborg.sh | 74 ++++++++++++++++++++++++++++++++++++ mprocs/bootstrap-rabbitmq.sh | 40 +++++++++++++++++++ mprocs/start-rabbitmq.sh | 30 +++++++++++++++ mprocs/wait-for-rabbitmq.sh | 16 ++++++++ 7 files changed, 209 insertions(+) create mode 100644 mprocs.yaml create mode 100755 mprocs/bootstrap-ofborg.sh create mode 100755 mprocs/bootstrap-rabbitmq.sh create mode 100755 mprocs/start-rabbitmq.sh create mode 100755 mprocs/wait-for-rabbitmq.sh diff --git a/.gitignore b/.gitignore index d5f18343..d8a90346 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ target *.nix.orig *.nix.rej /.direnv +/.ofborg-data diff --git a/flake.nix b/flake.nix index 1f642f7e..c0e5ca24 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,7 @@ rustfmt pkg-config git + mprocs ]; buildInputs = with pkgs; diff --git a/mprocs.yaml b/mprocs.yaml new file mode 100644 index 00000000..e4820607 --- /dev/null +++ b/mprocs.yaml @@ -0,0 +1,47 @@ +procs: + rabbitmq: + shell: mprocs/start-rabbitmq.sh + env: + RUST_LOG: info + stop: + send-keys: [""] + rabbitmq-init: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-rabbitmq.sh + env: + RUST_LOG: info + # no longer in use + # builder: + # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin builder .ofborg-data/local.json + # env: + # RUST_LOG: info + evaluation-filter: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin evaluation-filter .ofborg-data/local.json + env: + RUST_LOG: info + github-comment-filter: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-comment-filter .ofborg-data/local.json + autostart: false + env: + RUST_LOG: info + github-webhook-receiver: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-webhook-receiver .ofborg-data/local.json + env: + RUST_LOG: info + # no longer in use + # log-message-collector: + # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin log-message-collector .ofborg-data/local.json + # env: + # RUST_LOG: info + # no longer in use + # logapi: + # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin logapi .ofborg-data/local.json + # env: + # RUST_LOG: info + mass-rebuilder: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin mass-rebuilder .ofborg-data/local.json + env: + RUST_LOG: info + stats: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin stats .ofborg-data/local.json + env: + RUST_LOG: info diff --git a/mprocs/bootstrap-ofborg.sh b/mprocs/bootstrap-ofborg.sh new file mode 100755 index 00000000..fcb4ac3a --- /dev/null +++ b/mprocs/bootstrap-ofborg.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DATA_DIR="${DATA_DIR:-$(pwd)/.ofborg-data}" +mkdir -pv "${DATA_DIR}" + +gen_secret() { + openssl rand -base64 18 | tr -d '/+=' | head -c 24 +} +if [[ ! -f "${DATA_DIR}/.webhook-secret" ]]; then + gen_secret > "${DATA_DIR}/.webhook-secret" +fi + +cat < .ofborg-data/local.json +{ + "github_webhook_receiver": { + "listen": "[::1]:9899", + "webhook_secret_file": "${DATA_DIR}/.webhook-secret", + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "mass_rebuilder": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "evaluation_filter": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "stats": { + "listen": "[::1]:9898", + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "runner": { + "identity": "...", + "repos": [ + "nixos/nixpkgs", + "ofborg/testpkgs" + ], + "disable_trusted_users": true + }, + "checkout": { + "root": "${DATA_DIR}/checkout" + }, + "nix": { + "system": "x86_64-linux", + "remote": "daemon", + "build_timeout_seconds": 3600, + "initial_heap_size": "4g" + } +} +EOF diff --git a/mprocs/bootstrap-rabbitmq.sh b/mprocs/bootstrap-rabbitmq.sh new file mode 100755 index 00000000..63442d2e --- /dev/null +++ b/mprocs/bootstrap-rabbitmq.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RABBITMQ_HOST="${RABBITMQ_HOST:-localhost}" +RABBITMQ_MGMT_PORT="${RABBITMQ_MGMT_PORT:-15672}" +ADMIN_USER="${RABBITMQ_ADMIN_USER:-admin}" +ADMIN_PASS="${RABBITMQ_ADMIN_PASS:-admin}" +CONFIG_OUT="${CONFIG_OUT:-$(pwd)/.ofborg-data/rabbitmq-config.json}" + +BASE_URL="http://${RABBITMQ_HOST}:${RABBITMQ_MGMT_PORT}/api" + +rmq() { + set -x + local method="$1" path="$2"; shift 2 + curl -sf -u "${ADMIN_USER}:${ADMIN_PASS}" \ + -X "${method}" \ + -H "Content-Type: application/json" \ + "${BASE_URL}${path}" "$@" +} + +urlencode() { + python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1],safe=''))" "$1" +} + +gen_secret() { + openssl rand -base64 18 | tr -d '/+=' | head -c 24 +} + +mkdir -p "$(dirname "${CONFIG_OUT}")" "$(pwd)/.ofborg-data/rabbitmq" + +rmq PUT "/vhosts/$(urlencode "ofborg")" -d '{}' >/dev/null + +pass="$(gen_secret)" +rmq PUT "/users/ofborg" -d "{\"password\":\"${pass}\", \"tags\": \"\"}" >/dev/null + +rmq PUT "/permissions/$(urlencode "ofborg")/ofborg" \ + -d "{\"configure\":\".*\",\"write\":\".*\",\"read\":\".*\"}" >/dev/null + +echo "${pass}" > .ofborg-data/.amqp-password diff --git a/mprocs/start-rabbitmq.sh b/mprocs/start-rabbitmq.sh new file mode 100755 index 00000000..6161f256 --- /dev/null +++ b/mprocs/start-rabbitmq.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ADMIN_USER="${RABBITMQ_ADMIN_USER:-admin}" +ADMIN_PASS="${RABBITMQ_ADMIN_PASS:-admin}" +DATA_DIR="${DATA_DIR:-$(pwd)/.ofborg-data/rabbitmq}" +CONTAINER_NAME="${CONTAINER_NAME:-ofborg-rabbitmq}" + +mkdir -p "${DATA_DIR}" + +# Remove a stopped container with the same name if it exists +if docker inspect "${CONTAINER_NAME}" &>/dev/null; then + echo "Container '${CONTAINER_NAME}' already exists." + echo "Run 'docker rm -f ${CONTAINER_NAME}' to remove it first" + exit 1 +fi + +echo "Starting RabbitMQ..." +docker run \ + --name "${CONTAINER_NAME}" \ + --hostname ofborg-rabbitmq \ + --rm \ + -p 5672:5672 \ + -p 15672:15672 \ + -e RABBITMQ_DEFAULT_USER="${ADMIN_USER}" \ + -e RABBITMQ_DEFAULT_PASS="${ADMIN_PASS}" \ + -e RABBITMQ_DEFAULT_VHOST="/" \ + -v "${DATA_DIR}:/var/lib/rabbitmq" \ + rabbitmq:4-management diff --git a/mprocs/wait-for-rabbitmq.sh b/mprocs/wait-for-rabbitmq.sh new file mode 100755 index 00000000..e9d8679a --- /dev/null +++ b/mprocs/wait-for-rabbitmq.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +RABBIT_HOST=${RABBIT_HOST:-localhost} +RABBIT_USER=${RABBIT_USER:-admin} +RABBIT_PASS=${RABBIT_PASS:-admin} + +until curl -sf "http://${RABBIT_HOST}:15672/api/healthchecks/node" \ + -u "${RABBIT_USER}:${RABBIT_PASS}" &>/dev/null; do + echo "Waiting for RabbitMQ..." + sleep 1 +done + +echo "RabbitMQ is ready" From 2397962bbed8deff95487310c2cc4a4841ae786a Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 11:49:25 +0200 Subject: [PATCH 12/17] docs: local development documentation --- DEVELOPMENT.md | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..9d7e603a --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,104 @@ +# Local Development + +## Prerequisites + +- [Nix](https://nixos.org/download.html) (with flakes enabled) +- [Docker](https://docs.docker.com/engine/install/) +- [mprocs](https://github.com/pvolok/mprocs) — available in the dev shell + +## Quick start + +```shell +# Enter the development environment +nix-shell + +# Start all services +mprocs +``` + +This brings up: + +| Process | Autostart | Description | +|---------------------------|-----------|-----------------------------------------------------------------| +| `rabbitmq` | yes | Docker container (`rabbitmq:4-management`) on ports 5672, 15672 | +| `rabbitmq-init` | yes | Waits for RabbitMQ, then creates the `ofborg` vhost/user | +| `evaluation-filter` | yes | Processes evaluation events from the queue | +| `github-webhook-receiver` | yes | HTTP server on `[::1]:9899` — receives GitHub webhooks | +| `mass-rebuilder` | yes | Orchestrates mass rebuild jobs | +| `stats` | yes | Stats server on `[::1]:9898` | +| `github-comment-filter` | — | Disabled by default; start manually in mprocs | + +## What happens under the hood + +1. **`mprocs/start-rabbitmq.sh`** — Starts RabbitMQ via Docker (`rabbitmq:4-management`). Data is persisted in `.ofborg-data/rabbitmq/`. The admin user defaults to `admin` / `admin`. + +2. **`mprocs/wait-for-rabbitmq.sh`** — Polls the RabbitMQ management API healthcheck until the server is ready. + +3. **`mprocs/bootstrap-rabbitmq.sh`** — Creates the `ofborg` vhost, a restricted `ofborg` user with a random password, and writes the password to `.ofborg-data/.amqp-password`. + +4. **`mprocs/bootstrap-ofborg.sh`** — Generates a random webhook secret (`.ofborg-data/.webhook-secret`) and writes a complete config at `.ofborg-data/local.json`. The config wires all services to the local RabbitMQ instance. + +All services then start via `cargo r --bin .ofborg-data/local.json`, so changes to source code are reflected immediately (cargo recompiles on restart). + +## Sending test webhook events + +Use the `ofborg-send-event` crate to simulate GitHub pull_request webhooks: + +```shell +# Send PR #123456 from NixOS/nixpkgs to localhost:9899 +cargo run -p ofborg-send-event -- 123456 + +# Use a different repo +cargo run -p ofborg-send-event -- --full-repo-name "ofborg/testpkgs" 42 + +# Custom event type +cargo run -p ofborg-send-event -- --event push 123456 + +# Point at a different webhook receiver +cargo run -p ofborg-send-event -- --webhook-receiver-url http://localhost:9999 123456 + +# Supply the webhook secret (required if the receiver validates signatures) +cargo run -p ofborg-send-event -- --secret-path .ofborg-data/.webhook-secret 123456 + +# Custom delivery ID and extra headers +cargo run -p ofborg-send-event -- --delivery-id my-id --header X-Custom=val 123456 + +# Verbose mode (shows response headers) +cargo run -p ofborg-send-event -- --verbose 123456 +``` + +The tool fetches the real PR data from GitHub, constructs a `PullRequestEvent` payload, signs it with HMAC-SHA256 (if `--secret-path` is given), and POSTs it to the receiver. + +### CLI reference + +``` +Usage: ofborg-send-event [OPTIONS] + +Arguments: + PR that should be fetched + +Options: + --webhook-receiver-url [default: http://localhost:9899] + --event [default: pull_request] + --secret-path Shared secret for X-Hub-Signature-256 + --delivery-id X-GitHub-Delivery header (default: random UUID) + --header Add arbitrary header (repeatable) + --verbose Print response headers + --timeout-secs [default: 30] + --full-repo-name [default: NixOS/nixpkgs] +``` + +## Building and testing + +```shell +# All in the dev shell: + +cargo build # build all binaries +cargo test # run tests +cargo clippy # lint +cargo fmt # format code +cargo r --bin # run a specific binary + +# Full CI check (from repo root): +nix-shell --pure --run checkPhase +``` From 5d00e4fe8b57259faff299b20f92a1ae5c3a9993 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 12:19:42 +0200 Subject: [PATCH 13/17] docs: Architecture md --- Architecture.md | 257 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 Architecture.md diff --git a/Architecture.md b/Architecture.md new file mode 100644 index 00000000..20628586 --- /dev/null +++ b/Architecture.md @@ -0,0 +1,257 @@ +# Architecture + +## Overview + +ofborg is a Rust-based CI system for [Nixpkgs](https://github.com/nixos/nixpkgs). It runs as a +[GitHub App](https://docs.github.com/en/apps) that automatically processes pull requests and issue +comments. The system uses **RabbitMQ** for asynchronous message passing between components. Each +component runs as an independent process communicating exclusively through exchanges and queues. + +## Components + +| Binary | Purpose | Consumes From | Publishes To | +|--------|---------|---------------|--------------| +| `github-webhook-receiver` | HTTP server receiving GitHub webhooks. Validates HMAC-SHA256 signatures, parses the event type, and routes raw JSON to RabbitMQ. | GitHub HTTP POST | `github-events` (topic exchange) | +| `github-comment-filter` | Consumes issue comment events, parses `@ofborg` commands (`build` / `eval`), checks ACL (repo authorization, trusted users), fetches PR data from GitHub API, and schedules build/eval jobs. | `build-inputs` queue | `build-jobs` (fanout exchange), `mass-rebuild-check-jobs` (queue via default exchange), `build-results` (fanout exchange) | +| `evaluation-filter` | Filters pull request events for mass rebuild evaluation. Only processes Opened, Synchronize, Reopened, and Edited (base change) events on authorized repos. | `mass-rebuild-check-inputs` queue | `mass-rebuild-check-jobs` (queue via default exchange) | +| `mass-rebuilder` | Runs nixpkgs evaluation to determine which packages a PR impacts. Clones nixpkgs, merges the PR, runs nix-instantiate on release expressions, computes the diff of out-paths, and schedules builds for changed packages. | `mass-rebuild-check-jobs` queue | `build-jobs` (fanout exchange), `build-results` (fanout exchange) | +| `builder` | Performs actual Nix builds on a specific system architecture. Clones the repo, fetches the PR, merges, partitions attrs into buildable/non-buildable, runs nix-build, streams log lines, and publishes results. One instance per architecture. | `build-inputs-{system}` queue (via `build-jobs` fanout) | `logs` (topic exchange), `build-results` (fanout exchange) | +| `github-comment-poster` | Creates and updates GitHub Check Runs on pull requests. On receipt of `QueuedBuildJobs` creates an in-progress check; on receipt of `BuildResult` creates a completed check run. | `build-results` queue (via `build-results` fanout) | GitHub Check Runs API | +| `log-message-collector` | Persists build log messages to the filesystem. Receives `BuildLogStart`, `BuildLogMsg`, and `BuildResult` messages and writes metadata, log lines, and result JSON to disk. | Server-named exclusive queue (via `logs` topic exchange) | Filesystem (log storage) | +| `logapi` | HTTP server that serves build log metadata and results via JSON API at `GET /logs/{repo}`. | Filesystem (log storage) | HTTP JSON responses | +| `stats` | Dual-purpose binary: (1) Prometheus metrics HTTP server at `/metrics`, (2) consumer of `EventMessage` payloads from the stats exchange for metric collection. | `stats-events` queue (via `stats` fanout) | HTTP Prometheus `/metrics` | +| `build-faker` | Debug tool that injects a fake `BuildJob` directly into a system queue for testing. | CLI arguments | `build-inputs-{system}` queue | + +## Data Flow + +### 1. Webhook Reception + +GitHub sends webhook events to `github-webhook-receiver` via HTTP POST. The receiver validates the +`X-Hub-Signature-256` HMAC header, parses the `X-GitHub-Event` header, and publishes the raw JSON +body to the `github-events` **topic exchange** with a routing key of `{event}.{owner}/{repo}` (e.g., +`pull_request.nixos/nixpkgs`). + +``` +GitHub ──HTTP POST──▶ github-webhook-receiver ──publish──▶ github-events (topic) +``` + +### 2. Direct Build Flow (Issue Comments) + +When someone comments `@ofborg build` on a PR: + +1. **`github-comment-filter`** consumes the `issue_comment.*` event from `build-inputs` queue. +2. It parses the comment, checks the ACL (authorized repos, trusted users), fetches PR data from the + GitHub API, and publishes: + - A `BuildJob` message to the `build-jobs` **fanout exchange** (distributed to all per-system queues). + - A `QueuedBuildJobs` message to the `build-results` **fanout exchange** (for creating in-progress + check runs). + +``` +github-events ──issue_comment.*──▶ build-inputs ──▶ github-comment-filter + ├──▶ build-jobs (fanout) + └──▶ build-results (fanout) +``` + +### 3. Mass Rebuild Flow (Pull Requests) + +When a PR is opened or updated: + +1. **`evaluation-filter`** consumes the `pull_request.*` event from `mass-rebuild-check-inputs`. + This queue has two bindings to `github-events`: `pull_request.*` (from the webhook-receiver) and + `pull_request.nixos/*` (from the evaluation-filter itself). +2. It filters for Opened, Synchronize, Reopened, and Edited (base change) actions on authorized + repos (`nixos/*`). +3. Publishes an `EvaluationJob` to `mass-rebuild-check-jobs`. + +``` +github-events ──pull_request.*──▶ mass-rebuild-check-inputs ──▶ evaluation-filter + └──▶ mass-rebuild-check-jobs +``` + +4. **`mass-rebuilder`** consumes `EvaluationJob` from `mass-rebuild-check-jobs`. It clones nixpkgs, + checks out the target branch, merges the PR, runs nix-instantiate on `nixos/release.nix` and + `pkgs/top-level/release.nix`, computes the diff of out-paths between base and merge, and + publishes `BuildJob` messages for changed packages to the `build-jobs` fanout exchange. + +``` +mass-rebuild-check-jobs ──▶ mass-rebuilder ──▶ build-jobs (fanout) + └──▶ build-results (fanout) +``` + +### 4. Building + +**`builder`** instances consume `BuildJob` messages from per-system queues +(`build-inputs-x86_64-linux`, `build-inputs-aarch64-linux`, `build-inputs-x86_64-darwin`, +`build-inputs-aarch64-darwin`), all bound to the `build-jobs` fanout exchange. Each builder: + +1. Clones the repo, fetches the PR, and merges. +2. Partitions attrs into buildable/non-buildable via nix-instantiate. +3. Runs nix-build on buildable attrs. +4. Streams log lines to the `logs` topic exchange with routing key `{repo}.{pr}`. +5. Publishes `BuildResult` to the `build-results` fanout exchange. + +``` +build-jobs (fanout) ──▶ build-inputs-x86_64-linux ──▶ builder + ──▶ build-inputs-aarch64-linux ──▶ builder + ──▶ build-inputs-x86_64-darwin ──▶ builder + ──▶ build-inputs-aarch64-darwin ──▶ builder + +builder ──▶ logs (topic) + └──▶ build-results (fanout) +``` + +### 5. Build Results + +**`github-comment-poster`** consumes from the `build-results` queue (bound to the `build-results` +fanout exchange). It receives two message types: +- `QueuedBuildJobs` — creates an in-progress GitHub Check Run on the PR. +- `BuildResult` — creates a completed GitHub Check Run (success/failure). + +``` +build-results (fanout) ──▶ build-results queue ──▶ github-comment-poster ──▶ GitHub Checks API +``` + +### 6. Log Collection + +**`log-message-collector`** consumes from an exclusive queue bound to the `logs` topic exchange with +routing key `*.*`. It receives `BuildLogStart`, `BuildLogMsg`, and `BuildResult` messages and +persists them to disk: +- Metadata: `{logs_path}/{routing_key}/{attempt_id}.metadata.json` +- Log lines: `{logs_path}/{routing_key}/{attempt_id}` +- Build result: `{logs_path}/{routing_key}/{attempt_id}.result.json` + +**`logapi`** is an HTTP server that serves this stored data via `GET /logs/{repo}` as JSON. + +``` +logs (topic) ──▶ exclusive queue ──▶ log-message-collector ──▶ disk + │ + ▼ + logapi (HTTP) +``` + +### 7. Stats / Metrics + +Components publish `EventMessage` payloads to the `stats` fanout exchange. The `stats` binary +consumes them from the `stats-events` queue and exposes collected metrics as Prometheus text format +at `/metrics` over HTTP. + +``` +components ──▶ stats (fanout) ──▶ stats-events queue ──▶ stats (Prometheus HTTP /metrics) +``` + +## RabbitMQ Topology + +### Exchanges + +| Exchange | Type | Purpose | +|----------|------|---------| +| `github-events` | Topic | Ingress for all raw GitHub webhook events. Routing keys: `{event}.{owner}/{repo}` | +| `build-jobs` | Fanout | Distributes `BuildJob` messages to all per-system build queues | +| `build-results` | Fanout | Distributes `BuildResult` and `QueuedBuildJobs` to all interested consumers | +| `logs` | Topic | Distributes build log messages. Routing keys: `{repo}.{pr}` | +| `stats` | Fanout | Distributes `EventMessage` metrics payloads | + +### Queues + +| Queue | Type | Exchange Binding | Routing Key | Consumer | +|-------|------|------------------|-------------|----------| +| `build-inputs` | Durable | `github-events` | `issue_comment.*` | `github-comment-filter` | +| `github-events-unknown` | Durable | `github-events` | `unknown.*` | — (no consumer) | +| `mass-rebuild-check-inputs` | Durable | `github-events` | `pull_request.*` (webhook-receiver), `pull_request.nixos/*` (evaluation-filter) | `evaluation-filter` | +| `mass-rebuild-check-jobs` | Durable | Default exchange | `mass-rebuild-check-jobs` (routing key = queue name) | `mass-rebuilder` | +| `build-inputs-x86_64-linux` | Durable | `build-jobs` (fanout) | — | `builder` (x86_64-linux) | +| `build-inputs-aarch64-linux` | Durable | `build-jobs` (fanout) | — | `builder` (aarch64-linux) | +| `build-inputs-x86_64-darwin` | Durable | `build-jobs` (fanout) | — | `builder` (x86_64-darwin) | +| `build-inputs-aarch64-darwin` | Durable | `build-jobs` (fanout) | — | `builder` (aarch64-darwin) | +| `build-results` | Durable | `build-results` (fanout) | — | `github-comment-poster` | +| `logs` | Exclusive, auto-delete | `logs` (topic) | `*.*` | `log-message-collector` | +| `stats-events` | Durable | `stats` (fanout) | — | `stats` | + +## RabbitMQ Graph + +```mermaid +graph + classDef component fill:#f96 + classDef debug fill:#999999 + classDef exc fill:#08f108 + + subgraph Legend + app-example(This is an application):::component + debug-example(This is a debug application):::debug + exchange-example{{This is a RabbitMQ exchange}}:::exc + queue-example[/This is a RabbitMQ queue/] + end + + github-webhook-receiver(github-webhook-receiver):::component + github-events{{github-events}}:::exc + build-inputs[/build-inputs/] + github-events-unknown[/github-events-unknown/] + mass-rebuild-check-inputs[/mass-rebuild-check-inputs/] + github-comment-filter(github-comment-filter):::component + build-jobs{{build-jobs}}:::exc + evaluation-filter(evaluation-filter):::component + mass-rebuild-check-jobs[/mass-rebuild-check-jobs/] + mass-rebuilder(mass-rebuilder):::component + build-inputs-x86_64-linux[/build-inputs-x86_64-linux/] + build-inputs-aarch64-linux[/build-inputs-aarch64-linux/] + build-inputs-x86_64-darwin[/build-inputs-x86_64-darwin/] + build-inputs-aarch64-darwin[/build-inputs-aarch64-darwin/] + builder(builder):::component + logs{{logs}}:::exc + logs-queue[/logs/] + log-message-collector(log-message-collector):::component + log-storage[(Log Storage)] + logapi(logapi):::component + build-results{{build-results}}:::exc + build-results-queue[/build-results/] + github-comment-poster(github-comment-poster):::component + stats{{stats}}:::exc + stats-events[/stats-events/] + stats-rs(stats):::component + build-faker(build-faker):::debug + + github-webhook-receiver --> github-events + github-events -->|issue_comment.*| build-inputs + github-events -->|unknown.*| github-events-unknown + github-events -->|pull_request.*| mass-rebuild-check-inputs + build-inputs --> github-comment-filter + github-comment-filter --> build-jobs + github-comment-filter --> mass-rebuild-check-jobs + mass-rebuild-check-inputs --> evaluation-filter + evaluation-filter --> mass-rebuild-check-jobs + mass-rebuild-check-jobs --> mass-rebuilder + mass-rebuilder --> build-jobs + mass-rebuilder --> build-results + build-jobs --> build-inputs-x86_64-linux + build-jobs --> build-inputs-aarch64-linux + build-jobs --> build-inputs-x86_64-darwin + build-jobs --> build-inputs-aarch64-darwin + build-inputs-x86_64-linux --> builder + build-inputs-aarch64-linux --> builder + build-inputs-x86_64-darwin --> builder + build-inputs-aarch64-darwin --> builder + builder --> logs + builder --> build-results + logs --> logs-queue + logs-queue --> log-message-collector + log-message-collector --> log-storage + log-storage --> logapi + build-results --> build-results-queue + build-results-queue --> github-comment-poster + stats --> stats-events + stats-events --> stats-rs + build-faker --> build-inputs-aarch64-linux +``` + +## Message Types + +| Message | Serialization | Fields | Published To | +|---------|---------------|--------|--------------| +| `BuildJob` | JSON | `repo`, `pr`, `system`, `attrs`, `checkout` | `build-jobs` exchange | +| `EvaluationJob` | JSON | `repo`, `pr`, `system` | `mass-rebuild-check-jobs` queue | +| `QueuedBuildJobs` | JSON | `repo`, `pr`, `attempt_id`, `total_jobs` | `build-results` exchange | +| `BuildResult` | JSON | `repo`, `pr`, `attempt_id`, `status` (Success/Failure), `system`, `attrs` | `build-results` exchange, `logs` exchange | +| `BuildLogStart` | JSON | `repo`, `pr`, `attempt_id`, `system` | `logs` exchange | +| `BuildLogMsg` | JSON | `repo`, `pr`, `attempt_id`, `line`, `msg` | `logs` exchange | +| `EventMessage` | JSON | `event` (tagged enum), `repo`, `pr`, `value` | `stats` exchange | +| Raw webhook JSON | JSON | Raw GitHub webhook payload | `github-events` exchange | From aba0fcb33b3400d6ae9fa02653a39771b13a66dc Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 13:00:35 +0200 Subject: [PATCH 14/17] feat: implement hydra evaluator --- Cargo.lock | 854 ++++++++++++++++++++++++++- Cargo.toml | 6 +- flake.lock | 19 +- flake.nix | 14 + hydra-evaluator/Cargo.toml | 33 ++ hydra-evaluator/src/config.rs | 128 ++++ hydra-evaluator/src/grpc.rs | 107 ++++ hydra-evaluator/src/main.rs | 221 +++++++ mprocs.yaml | 4 + ofborg-send-event/src/main.rs | 3 +- ofborg/src/bin/mass-rebuilder.rs | 24 + ofborg/src/config.rs | 14 + ofborg/src/message/buildjob.rs | 2 +- ofborg/src/message/hydra_eval_job.rs | 14 + ofborg/src/message/mod.rs | 1 + ofborg/src/nix.rs | 18 +- ofborg/src/tasks/evaluate.rs | 106 +++- 17 files changed, 1547 insertions(+), 21 deletions(-) create mode 100644 hydra-evaluator/Cargo.toml create mode 100644 hydra-evaluator/src/config.rs create mode 100644 hydra-evaluator/src/grpc.rs create mode 100644 hydra-evaluator/src/main.rs create mode 100644 ofborg/src/message/hydra_eval_job.rs diff --git a/Cargo.lock b/Cargo.lock index befd8f84..3a66cee1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,6 +22,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amq-protocol" version = "10.6.2" @@ -146,6 +152,18 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "asn1-rs" version = "0.7.2" @@ -210,6 +228,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.14.0" @@ -316,6 +346,49 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -349,6 +422,20 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -406,6 +493,9 @@ name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] [[package]] name = "camino" @@ -575,6 +665,17 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -591,6 +692,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -612,6 +730,21 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie-factory" version = "0.3.3" @@ -759,6 +892,68 @@ dependencies = [ "syn", ] +[[package]] +name = "cxx" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -812,6 +1007,29 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + [[package]] name = "des" version = "0.8.1" @@ -1001,6 +1219,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flagset" version = "0.4.7" @@ -1030,6 +1254,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1039,6 +1269,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs2" version = "0.4.3" @@ -1249,13 +1489,151 @@ dependencies = [ "tracing", ] +[[package]] +name = "harmonia-store-aterm" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "bytes", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "memchr", + "serde_json", + "thiserror", +] + +[[package]] +name = "harmonia-store-content-address" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "derive_more", + "harmonia-store-path", + "harmonia-utils-hash", + "serde", + "thiserror", +] + +[[package]] +name = "harmonia-store-derivation" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "bytes", + "data-encoding", + "derive_more", + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", + "serde_json", + "thiserror", + "zerocopy", +] + +[[package]] +name = "harmonia-store-nar-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", +] + +[[package]] +name = "harmonia-store-path" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "derive_more", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "serde", + "thiserror", + "zerocopy", +] + +[[package]] +name = "harmonia-store-path-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", +] + +[[package]] +name = "harmonia-utils-base-encoding" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "data-encoding", + "derive_more", + "serde", +] + +[[package]] +name = "harmonia-utils-hash" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "blake3", + "data-encoding", + "derive_more", + "harmonia-utils-base-encoding", + "md5", + "serde", + "sha1 0.11.0", + "sha2 0.11.0", + "thiserror", + "tokio", +] + +[[package]] +name = "harmonia-utils-signature" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "data-encoding", + "ed25519-dalek", + "getrandom 0.4.2", + "harmonia-utils-base-encoding", + "serde", + "subtle", + "thiserror", + "zeroize", +] + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -1427,6 +1805,66 @@ dependencies = [ "typenum", ] +[[package]] +name = "hydra-evaluator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "fs-err", + "futures", + "harmonia-store-path", + "hydra-proto", + "hydra-tracing", + "hyper-rustls", + "hyper-util", + "lapin", + "nix-utils", + "ofborg", + "serde_json", + "store-transfer", + "tokio", + "tokio-stream", + "toml", + "tonic", + "tower", + "tracing", + "url", +] + +[[package]] +name = "hydra-proto" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "fs-err", + "harmonia-store-content-address", + "harmonia-store-nar-info", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "nix-support", + "prost", + "sha2 0.10.9", + "store-path-utils", + "thiserror", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + +[[package]] +name = "hydra-tracing" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "thiserror", + "tracing", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "hyper" version = "1.10.0" @@ -1451,16 +1889,18 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ + "futures-util", "http", "hyper", "hyper-util", "log", "rustls", "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1499,6 +1939,7 @@ dependencies = [ "socket2", "system-configuration", "tokio", + "tower-layer", "tower-service", "tracing", "windows-registry", @@ -1687,6 +2128,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1833,6 +2283,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1890,6 +2349,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md5" version = "0.8.0" @@ -1942,12 +2407,64 @@ dependencies = [ "uuid", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "nix-support" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "fs-err", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-hash", + "regex", + "sha2 0.10.9", + "store-path-utils", + "tokio", + "tracing", +] + +[[package]] +name = "nix-utils" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "bytes", + "cxx", + "cxx-build", + "fs-err", + "futures", + "harmonia-store-aterm", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "hashbrown 0.16.1", + "pkg-config", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + [[package]] name = "nom" version = "7.1.3" @@ -2191,7 +2708,7 @@ dependencies = [ "pkcs5", "rand 0.10.1", "rc2", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "thiserror", "x509-parser", @@ -2285,6 +2802,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -2373,6 +2901,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -2442,6 +2976,79 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quinn" version = "0.11.9" @@ -2778,7 +3385,6 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2906,6 +3512,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "scrypt" version = "0.11.0" @@ -3027,6 +3639,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3050,6 +3671,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3203,6 +3835,36 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "store-path-utils" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "harmonia-store-path", +] + +[[package]] +name = "store-transfer" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "async-compression", + "bytes", + "futures", + "harmonia-store-path", + "harmonia-store-path-info", + "hashbrown 0.16.1", + "hydra-proto", + "nix-utils", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tracing", + "zstd", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3299,6 +3961,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -3446,6 +4117,116 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", + "zstd", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" @@ -3454,7 +4235,9 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", "tokio-util", @@ -3581,12 +4364,30 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -3809,6 +4610,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "widestring" version = "1.2.1" @@ -3998,6 +4808,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4248,3 +5064,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 09f3c94d..218e585b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,9 @@ [workspace] members = [ - "ofborg", "ofborg-send-event", - "ofborg-simple-build" + "ofborg", + "ofborg-send-event", + "ofborg-simple-build", + "hydra-evaluator", ] resolver = "2" diff --git a/flake.lock b/flake.lock index 5d203907..5e773697 100644 --- a/flake.lock +++ b/flake.lock @@ -16,9 +16,26 @@ "type": "github" } }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1778729098, + "narHash": "sha256-17SbusskVZng4nwevRqsWNJf27nMG7UczvtgWTUJttg=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "39ea44cddd5060b8cd413ed5e13c6af61f302283", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" } } }, diff --git a/flake.nix b/flake.nix index c0e5ca24..ab569579 100644 --- a/flake.nix +++ b/flake.nix @@ -1,12 +1,14 @@ { inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-25.11"; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable-small"; }; outputs = { self, nixpkgs, + nixpkgs-unstable, ... }@inputs: let @@ -26,6 +28,10 @@ pkgs = import nixpkgs { inherit system; }; + unstable = import nixpkgs-unstable { + inherit system; + }; + nix = unstable.nixVersions.nix_2_34; in { default = pkgs.mkShell { @@ -41,6 +47,13 @@ pkg-config git mprocs + + zlib + protobuf + + nlohmann_json + libsodium + boost ]; buildInputs = with pkgs; @@ -72,6 +85,7 @@ RUST_BACKTRACE = "1"; RUST_LOG = "ofborg=debug,info"; NIX_PATH = "nixpkgs=${pkgs.path}"; + NIX_CFLAGS_COMPILE = "-Wno-error"; }; } ); diff --git a/hydra-evaluator/Cargo.toml b/hydra-evaluator/Cargo.toml new file mode 100644 index 00000000..19e2dcbd --- /dev/null +++ b/hydra-evaluator/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "hydra-evaluator" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" + +[dependencies] +tracing = "0.1" + +anyhow = "1.0.98" +clap = { version = "4", features = ["derive"] } +fs-err = { version = "3.0", features = ["tokio"] } + +tokio = { version = "1.50", features = ["full"] } +tokio-stream = "0.1" +futures = "0.3" +tonic = { version = "0.14", features = ["zstd", "tls-webpki-roots"] } +tower = "0.5" +hyper-util = { version = "0.1.10", features = ["full"] } + +lapin = "4.3.0" +serde_json = "1.0" + +url = "2.5.4" +toml = "1.0.0" + +harmonia-store-path = { git = "https://github.com/nix-community/harmonia.git" } +nix-utils = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +hydra-tracing = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +hydra-proto = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7", features = [ "client" ] } +store-transfer = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +ofborg = { path = "../ofborg" } +hyper-rustls = "=0.27.3" # required for ofborg::octocrab diff --git a/hydra-evaluator/src/config.rs b/hydra-evaluator/src/config.rs new file mode 100644 index 00000000..2d88e88a --- /dev/null +++ b/hydra-evaluator/src/config.rs @@ -0,0 +1,128 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap( + author, + version, + about, + long_about = "ofborg-evaluator: injects derivations into a queue-runner jobset" +)] +pub struct Cli { + /// Queue-runner gRPC endpoint + #[clap(short, long, default_value = "http://[::1]:50051")] + pub gateway_endpoint: String, + + /// File containing the bearer token for authentication + #[clap(long)] + pub authorization_file: Option, + + /// Whether to use mTLS + #[clap(long)] + pub mtls: bool, + + /// Path to Server root CA cert + #[clap(long)] + pub server_root_ca_cert_path: Option, + + /// Path to Client cert + #[clap(long)] + pub client_cert_path: Option, + + /// Path to Client key + #[clap(long)] + pub client_key_path: Option, + + /// Domain name for mTLS + #[clap(long)] + pub domain_name: Option, + + /// Config file path + #[clap()] + pub config_path: std::path::PathBuf, +} + +impl Cli { + #[must_use] + pub fn new() -> Self { + Self::parse() + } + + pub async fn get_authorization_token(&self) -> anyhow::Result> { + let Some(path) = &self.authorization_file else { + return Ok(None); + }; + + let content = fs_err::tokio::read_to_string(path).await?; + + // Try parsing as TOML and extracting the "token" field. + // This allows reusing the same token file format as the queue-runner. + if let Ok(value) = content.parse::() + && let Some(token) = value.get("token").and_then(toml::Value::as_str) + { + return Ok(Some(token.to_string())); + } + + // Fall back: treat entire file content as a plain-text token + Ok(Some(content.trim().to_string())) + } + + #[must_use] + pub const fn mtls_enabled(&self) -> bool { + self.mtls + } + + #[must_use] + pub fn mtls_configured_correctly(&self) -> bool { + if self.mtls { + self.server_root_ca_cert_path.is_some() + && self.client_cert_path.is_some() + && self.client_key_path.is_some() + && self.domain_name.is_some() + } else { + self.server_root_ca_cert_path.is_none() + && self.client_cert_path.is_none() + && self.client_key_path.is_none() + && self.domain_name.is_none() + } + } + + pub async fn get_mtls( + &self, + ) -> anyhow::Result<( + tonic::transport::Certificate, + tonic::transport::Identity, + String, + )> { + let server_root_ca_cert_path = self + .server_root_ca_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("server_root_ca_cert_path not provided"))?; + let client_cert_path = self + .client_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_cert_path not provided"))?; + let client_key_path = self + .client_key_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_key_path not provided"))?; + let domain_name = self + .domain_name + .as_deref() + .ok_or_else(|| anyhow::anyhow!("domain_name not provided"))?; + + let server_root_ca_cert = fs_err::tokio::read_to_string(server_root_ca_cert_path).await?; + let server_root_ca_cert = tonic::transport::Certificate::from_pem(server_root_ca_cert); + + let client_cert = fs_err::tokio::read_to_string(client_cert_path).await?; + let client_key = fs_err::tokio::read_to_string(client_key_path).await?; + let client_identity = tonic::transport::Identity::from_pem(client_cert, client_key); + + Ok((server_root_ca_cert, client_identity, domain_name.to_owned())) + } +} + +impl Default for Cli { + fn default() -> Self { + Self::new() + } +} diff --git a/hydra-evaluator/src/grpc.rs b/hydra-evaluator/src/grpc.rs new file mode 100644 index 00000000..ed0c152d --- /dev/null +++ b/hydra-evaluator/src/grpc.rs @@ -0,0 +1,107 @@ +use anyhow::Context as _; +use tonic::Request; +use tonic::transport::Channel; + +use hydra_proto::runner_service_client::RunnerServiceClient; + +#[derive(Debug, Clone)] +pub enum AuthInterceptor { + Token { + token: tonic::metadata::MetadataValue, + }, + Noop, +} + +impl tonic::service::Interceptor for AuthInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, tonic::Status> { + if let Self::Token { token } = self { + request + .metadata_mut() + .insert("authorization", token.clone()); + } + + Ok(request) + } +} + +pub type OfborgClient = + RunnerServiceClient>; + +#[tracing::instrument(err)] +pub async fn init_client(cli: &crate::config::Cli) -> anyhow::Result { + if !cli.mtls_configured_correctly() { + tracing::error!( + "mtls configured improperly, please pass all options: \ + server_root_ca_cert_path, client_cert_path, client_key_path and domain_name!" + ); + return Err(anyhow::anyhow!("Configuration issue")); + } + + tracing::info!("connecting to {}", cli.gateway_endpoint); + let channel = if cli.mtls_enabled() { + tracing::info!("mtls is enabled"); + let (server_root_ca_cert, client_identity, domain_name) = cli + .get_mtls() + .await + .context("Failed to get_mtls Certificate and Identity")?; + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name(domain_name) + .ca_certificate(server_root_ca_cert) + .identity(client_identity); + + Channel::builder(cli.gateway_endpoint.parse()?) + .tls_config(tls) + .context("Failed to attach tls config")? + .connect() + .await + .context("Failed to establish connection with Channel")? + } else if let Some(path) = cli.gateway_endpoint.strip_prefix("unix://") { + let path = path.to_owned(); + tonic::transport::Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + let path = path.clone(); + async move { + Ok::<_, std::io::Error>(hyper_util::rt::TokioIo::new( + tokio::net::UnixStream::connect(&path).await?, + )) + } + })) + .await + .context("Failed to establish unix socket connection with Channel")? + } else if cli.gateway_endpoint.starts_with("https://") { + let uri: url::Url = cli + .gateway_endpoint + .parse() + .context("Failed to parse gateway_endpoint")?; + + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name( + uri.domain() + .ok_or_else(|| anyhow::anyhow!("No domain_name found for gateway_endpoint"))?, + ) + .with_enabled_roots(); + Channel::builder(cli.gateway_endpoint.parse()?) + .tls_config(tls) + .context("Failed to attach tls config")? + .connect() + .await + .context("Failed to establish connection with Channel")? + } else { + Channel::builder(cli.gateway_endpoint.parse()?) + .connect() + .await + .context("Failed to establish connection with Channel")? + }; + + let interceptor = if let Some(t) = cli.get_authorization_token().await? { + AuthInterceptor::Token { + token: format!("Bearer {t}").parse()?, + } + } else { + AuthInterceptor::Noop + }; + + Ok(RunnerServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(50 * 1024 * 1024) + .max_encoding_message_size(50 * 1024 * 1024)) +} diff --git a/hydra-evaluator/src/main.rs b/hydra-evaluator/src/main.rs new file mode 100644 index 00000000..10b4f6b4 --- /dev/null +++ b/hydra-evaluator/src/main.rs @@ -0,0 +1,221 @@ +#![forbid(unsafe_code)] +#![deny( + clippy::all, + clippy::pedantic, + clippy::expect_used, + clippy::unwrap_used, + future_incompatible, + missing_debug_implementations, + nonstandard_style, + missing_copy_implementations, + unused_qualifications +)] +#![allow(clippy::missing_errors_doc)] + +mod config; +mod grpc; + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::Context as _; +use futures::TryFutureExt as _; +use harmonia_store_path::{FromStoreDirStr, StorePath}; +use hydra_proto::{CreateBuildRequest, ProtoStorePath}; +use lapin::options::{BasicAckOptions, BasicConsumeOptions, QueueDeclareOptions}; +use lapin::types::FieldTable; +use nix_utils::BaseStore as _; +use tokio::sync::mpsc; +use tokio_stream::StreamExt as _; + +use crate::grpc::OfborgClient; + +#[tracing::instrument(skip(client, drv_paths), err)] +async fn import_drvs(client: &mut OfborgClient, drv_paths: &[StorePath]) -> anyhow::Result<()> { + let (tx, rx) = + mpsc::unbounded_channel::>(); + + let store = nix_utils::LocalStore::init(); + let drv_paths = drv_paths.to_vec(); + let sender = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Runtime::new()?; + let infos = rt.block_on(store.query_path_infos(&drv_paths.iter().collect::>()))?; + store_transfer::export::export(&store, &drv_paths, &infos, &tx); + Ok::<(), anyhow::Error>(()) + }); + + let upload = client + .build_result(tokio_stream::StreamExt::filter_map( + tokio_stream::wrappers::UnboundedReceiverStream::new(rx), + Result::ok, + )) + .map_err(Into::::into); + + let (upload_result, sender_result) = futures::future::join(upload, sender).await; + upload_result?; + sender_result??; + + Ok(()) +} + +#[tracing::instrument(skip(client), err)] +async fn create_builds( + client: &mut OfborgClient, + jobset_id: i32, + drv_paths: &[StorePath], +) -> anyhow::Result> { + let response = client + .create_build(CreateBuildRequest { + jobset_id, + drv_paths: drv_paths.iter().map(ProtoStorePath::from).collect(), + }) + .await + .context("Failed to call CreateBuild")?; + + Ok(response.into_inner().build_ids) +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> anyhow::Result<()> { + hydra_tracing::init()?; + nix_utils::init_nix(); + + let cli = Arc::new(config::Cli::new()); + let Some(cfg) = ofborg::config::load(&cli.config_path).hydra_evaluator else { + tracing::error!("No ofborg/hydra evaluator configuration found!"); + panic!(); + }; + + tracing::info!( + "ofborg-evaluator starting endpoint={}", + cli.gateway_endpoint + ); + + tracing::info!("running in AMQP consumer mode"); + let conn = ofborg::easylapin::from_config(&cfg.rabbitmq).await?; + let chan = conn.create_channel().await?; + + // Declare the hydra-eval-jobs queue (must match what mass-rebuilder publishes to) + chan.queue_declare( + "hydra-eval-jobs".into(), + QueueDeclareOptions { + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + nowait: false, + }, + FieldTable::default(), + ) + .await?; + + tracing::info!("connecting to queue-runner gRPC"); + let mut client = grpc::init_client(&cli).await?; + + tracing::info!("consuming from hydra-eval-jobs"); + let mut consumer = chan + .basic_consume( + "hydra-eval-jobs".into(), + "ofborg-hydra-evaluator".into(), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await?; + + while let Some(Ok(delivery)) = consumer.next().await { + let body = &delivery.data; + let job: ofborg::message::hydra_eval_job::HydraEvalJob = match serde_json::from_slice(body) + { + Ok(job) => job, + Err(e) => { + tracing::error!( + "Failed to deserialize HydraEvalJob: {e}, body: {:?}", + std::str::from_utf8(body) + ); + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + continue; + } + }; + + tracing::info!( + "Processing HydraEvalJob for {}/{} PR #{} ({} drv paths, jobset_id={})", + job.repo.owner, + job.repo.name, + job.pr.number, + job.drv_paths.len(), + job.jobset_id, + ); + + if job.drv_paths.is_empty() { + tracing::warn!("Received HydraEvalJob with no drv paths, acking"); + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + continue; + } + + let store_dir = nix_utils::LocalStore::init().store_dir().clone(); + let drv_paths: Vec = job + .drv_paths + .iter() + .map(|s| { + StorePath::from_store_dir_str(&store_dir, s) + .unwrap_or_else(|e| panic!("Invalid store path '{s}': {e}")) + }) + .collect(); + + match import_drvs(&mut client, &drv_paths).await { + Ok(()) => { + tracing::info!("Successfully imported {} drv(s)", drv_paths.len()); + } + Err(e) => { + tracing::error!("Failed to import drvs: {e:?}"); + // Nack and requeue so another consumer can retry + let _ = chan + .basic_nack( + delivery.delivery_tag, + lapin::options::BasicNackOptions { + requeue: true, + ..Default::default() + }, + ) + .await; + continue; + } + } + + match create_builds(&mut client, job.jobset_id, &drv_paths).await { + Ok(build_ids) => { + tracing::info!("Created {} build(s)", build_ids.len()); + for (drv_path, build_id) in &build_ids { + tracing::info!(" {build_id} <- {drv_path}"); + } + } + Err(e) => { + tracing::error!("Failed to create builds: {e:?}"); + let _ = chan + .basic_nack( + delivery.delivery_tag, + lapin::options::BasicNackOptions { + requeue: true, + ..Default::default() + }, + ) + .await; + continue; + } + } + + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + tracing::info!("Finished processing job for PR #{}", job.pr.number); + } + + drop(conn); // Close connection. + tracing::info!("Closed the session... EOF"); + Ok(()) +} diff --git a/mprocs.yaml b/mprocs.yaml index e4820607..a9ba6dd3 100644 --- a/mprocs.yaml +++ b/mprocs.yaml @@ -45,3 +45,7 @@ procs: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin stats .ofborg-data/local.json env: RUST_LOG: info + hydra-evaluator: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r -p hydra-evaluator -- .ofborg-data/local.json + env: + RUST_LOG: info diff --git a/ofborg-send-event/src/main.rs b/ofborg-send-event/src/main.rs index 7704da84..b6ef9942 100644 --- a/ofborg-send-event/src/main.rs +++ b/ofborg-send-event/src/main.rs @@ -126,7 +126,8 @@ async fn main() -> Result<()> { ); if let Some(secret_path) = &args.secret_path { - let signature = github_signature_256(&std::fs::read_to_string(secret_path)?.trim(), &body)?; + let secret = std::fs::read_to_string(secret_path)?; + let signature = github_signature_256(secret.trim(), &body)?; headers.insert( HeaderName::from_static("x-hub-signature-256"), HeaderValue::from_str(&signature).context("invalid X-Hub-Signature-256 value")?, diff --git a/ofborg/src/bin/mass-rebuilder.rs b/ofborg/src/bin/mass-rebuilder.rs index cead74c2..97c4932c 100644 --- a/ofborg/src/bin/mass-rebuilder.rs +++ b/ofborg/src/bin/mass-rebuilder.rs @@ -42,6 +42,27 @@ async fn main() -> anyhow::Result<()> { }) .await?; + let hydra_eval_cfg = cfg.hydra_evaluator.clone(); + let (hydra_eval_queue, hydra_eval_nix, hydra_eval_jobset_id) = + if let Some(ref hec) = hydra_eval_cfg { + chan.declare_queue(easyamqp::QueueConfig { + queue: String::from("hydra-eval-jobs"), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + }) + .await?; + ( + Some("hydra-eval-jobs".to_owned()), + Some(cfg.nix()), + Some(hec.jobset_id), + ) + } else { + (None, None, None) + }; + let handle = easylapin::WorkerChannel(chan) .consume( tasks::evaluate::EvaluationWorker::new( @@ -50,6 +71,9 @@ async fn main() -> anyhow::Result<()> { cfg.acl(), cfg.runner.identity.clone(), events, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, ), easyamqp::ConsumeConfig { queue: queue_name.clone(), diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 89fd5f06..02bc86f2 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -28,6 +28,8 @@ pub struct Config { pub github_comment_poster: Option, /// Configuration for the mass rebuilder pub mass_rebuilder: Option, + /// Configuration for the hydra evaluator integration + pub hydra_evaluator: Option, /// Configuration for the builder pub builder: Option, /// Configuration for the log message collector @@ -104,6 +106,18 @@ pub struct MassRebuilder { pub rabbitmq: RabbitMqConfig, } +/// Configuration for the hydra evaluator integration +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct HydraEvaluatorConfig { + /// RabbitMQ broker to connect to + pub rabbitmq: RabbitMqConfig, + /// Queue-runner gRPC endpoint + pub gateway_endpoint: String, + /// Jobset ID to inject builds into + pub jobset_id: i32, +} + /// Configuration for the builder #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] diff --git a/ofborg/src/message/buildjob.rs b/ofborg/src/message/buildjob.rs index b09eae58..18c970e1 100644 --- a/ofborg/src/message/buildjob.rs +++ b/ofborg/src/message/buildjob.rs @@ -1,7 +1,7 @@ use crate::commentparser::Subset; use crate::message::{Pr, Repo}; -#[derive(serde::Serialize, serde::Deserialize, Debug)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct BuildJob { pub repo: Repo, pub pr: Pr, diff --git a/ofborg/src/message/hydra_eval_job.rs b/ofborg/src/message/hydra_eval_job.rs new file mode 100644 index 00000000..ed12ea88 --- /dev/null +++ b/ofborg/src/message/hydra_eval_job.rs @@ -0,0 +1,14 @@ +use crate::message::{Pr, Repo}; + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct HydraEvalJob { + pub repo: Repo, + pub pr: Pr, + pub drv_paths: Vec, + pub request_id: String, + pub jobset_id: i32, +} + +pub fn from(data: &[u8]) -> Result { + serde_json::from_slice(data) +} diff --git a/ofborg/src/message/mod.rs b/ofborg/src/message/mod.rs index 03551cd1..41efc579 100644 --- a/ofborg/src/message/mod.rs +++ b/ofborg/src/message/mod.rs @@ -3,5 +3,6 @@ pub mod buildlogmsg; pub mod buildresult; mod common; pub mod evaluationjob; +pub mod hydra_eval_job; pub use self::common::{Pr, Repo}; diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 31350520..e71723ca 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -58,8 +58,8 @@ impl Operation { "--no-out-link", "--keep-going", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::QueryPackagesJson => { @@ -68,8 +68,8 @@ impl Operation { "--available", "--json", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::QueryPackagesOutputs => { @@ -80,8 +80,8 @@ impl Operation { "--attr-path", "--out-path", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::NoOp { ref operation } => { @@ -93,12 +93,12 @@ impl Operation { "--strict", "--json", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::Instantiate => { - command.args(["--option", "extra-experimental-features", "no-url-literals"]); + command.args(["--option", "lint-url-literals", "fatal"]); } _ => (), }; diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index eb595a92..851a88a8 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -1,4 +1,5 @@ /// This is what evaluates every pull-request +use std::io::{BufRead as _, BufReader}; use std::path::Path; use std::time::Instant; @@ -10,11 +11,13 @@ use crate::acl::Acl; use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::config::GithubAppVendingMachine; use crate::github::GithubRepo; -use crate::message::{buildjob, evaluationjob}; +use crate::message::{buildjob, evaluationjob, hydra_eval_job}; +use crate::nix; use crate::stats::{self, Event}; use crate::tasks::eval; use crate::tasks::eval::EvaluationStrategy; use crate::{checkout, systems, worker}; +use uuid::Uuid; pub struct EvaluationWorker { cloner: checkout::CachedCloner, @@ -22,15 +25,22 @@ pub struct EvaluationWorker { acl: Acl, identity: String, events: E, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, } impl EvaluationWorker { + #[allow(clippy::too_many_arguments)] pub fn new( cloner: checkout::CachedCloner, github_vend: Option, acl: Acl, identity: String, events: E, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, ) -> EvaluationWorker { EvaluationWorker { cloner, @@ -38,6 +48,9 @@ impl EvaluationWorker { acl, identity, events, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, } } } @@ -97,6 +110,9 @@ impl worker::SimpleWorker for EvaluationWorker &self.identity, &self.cloner, job, + self.hydra_eval_queue.clone(), + self.hydra_eval_nix.clone(), + self.hydra_eval_jobset_id, ) .worker_actions() .await @@ -115,10 +131,14 @@ struct OneEval<'a, E> { cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, prefix: Option<&'static str>, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, } impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { #[allow(clippy::too_many_arguments)] + #[allow(clippy::borrow_as_ptr)] fn new( octocrab: Option, acl: &'a Acl, @@ -126,6 +146,9 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, ) -> OneEval<'a, E> { let (repo, enable_publish) = if let Some(octocrab) = octocrab { ( @@ -148,6 +171,9 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { cloner, job, prefix: None, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, } } @@ -462,7 +488,40 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .all_evaluations_passed(&mut overall_status) .await?; - response.extend(schedule_builds(complete.builds, auto_schedule_build_archs)); + response.extend(schedule_builds( + complete.builds.clone(), + auto_schedule_build_archs, + )); + + if let (Some(ref queue), Some(ref nix), Some(jobset_id)) = ( + self.hydra_eval_queue.clone(), + self.hydra_eval_nix.clone(), + self.hydra_eval_jobset_id, + ) { + let drv_paths = resolve_attrs_to_drv_paths( + nix, + std::path::Path::new(&refpath), + &complete.builds, + ); + if !drv_paths.is_empty() { + info!( + "Publishing {} drv paths to hydra-eval-jobs for PR #{}", + drv_paths.len(), + job.pr.number + ); + response.push(worker::publish_serde_action( + None, + Some(queue.clone()), + &hydra_eval_job::HydraEvalJob { + repo: job.repo.clone(), + pr: job.pr.clone(), + drv_paths, + request_id: Uuid::new_v4().to_string(), + jobset_id, + }, + )); + } + } overall_status .set_with_description("^.^!", StatusState::Success) @@ -512,6 +571,49 @@ fn schedule_builds( response } +fn resolve_attrs_to_drv_paths( + nix: &nix::Nix, + nixpkgs: &std::path::Path, + builds: &[buildjob::BuildJob], +) -> Vec { + let mut all_attrs: Vec = builds.iter().flat_map(|b| b.attrs.clone()).collect(); + all_attrs.sort(); + all_attrs.dedup(); + + if all_attrs.is_empty() { + return vec![]; + } + + let file = builds + .first() + .and_then(|b| b.subset.clone()) + .map(|s| match s { + crate::commentparser::Subset::NixOS => nix::File::ReleaseNixOS, + crate::commentparser::Subset::Nixpkgs => nix::File::DefaultNixpkgs, + }) + .unwrap_or(nix::File::DefaultNixpkgs); + + all_attrs + .into_iter() + .flat_map( + |attr| match nix.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) { + Ok(f) => BufReader::new(f) + .lines() + .map_while(Result::ok) + .filter(|line| line.trim().ends_with(".drv")) + .map(|line| line.trim().to_owned()) + .collect::>(), + Err(f) => { + let stderr: Vec = + BufReader::new(f).lines().map_while(Result::ok).collect(); + warn!("nix-instantiate failed for attrs: {:?}", stderr.join("\n")); + vec![] + } + }, + ) + .collect() +} + fn issue_is_wip(issue: &octocrab::models::issues::Issue) -> bool { issue.title.starts_with("WIP:") || issue.title.contains("[WIP]") } From 71ea30e4671d06691e11ba8867df081740d8da08 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 15:43:37 +0200 Subject: [PATCH 15/17] feat: configure hydra_evaluator --- DEVELOPMENT.md | 2 +- mprocs.yaml | 1 - mprocs/bootstrap-ofborg.sh | 21 ++++++++++++++++++++- ofborg-send-event/src/main.rs | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9d7e603a..5f9cad99 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -26,7 +26,7 @@ This brings up: | `github-webhook-receiver` | yes | HTTP server on `[::1]:9899` — receives GitHub webhooks | | `mass-rebuilder` | yes | Orchestrates mass rebuild jobs | | `stats` | yes | Stats server on `[::1]:9898` | -| `github-comment-filter` | — | Disabled by default; start manually in mprocs | +| `github-comment-filter` | yes | Disabled by default; start manually in mprocs | ## What happens under the hood diff --git a/mprocs.yaml b/mprocs.yaml index a9ba6dd3..dd016cdc 100644 --- a/mprocs.yaml +++ b/mprocs.yaml @@ -20,7 +20,6 @@ procs: RUST_LOG: info github-comment-filter: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-comment-filter .ofborg-data/local.json - autostart: false env: RUST_LOG: info github-webhook-receiver: diff --git a/mprocs/bootstrap-ofborg.sh b/mprocs/bootstrap-ofborg.sh index fcb4ac3a..431929f8 100755 --- a/mprocs/bootstrap-ofborg.sh +++ b/mprocs/bootstrap-ofborg.sh @@ -43,6 +43,26 @@ cat < .ofborg-data/local.json "virtualhost": "ofborg" } }, + "github_comment_filter": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "hydra_evaluator": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + }, + "gateway_endpoint": "[::1]:50051", + "jobset_id": 7 + }, "stats": { "listen": "[::1]:9898", "rabbitmq": { @@ -56,7 +76,6 @@ cat < .ofborg-data/local.json "runner": { "identity": "...", "repos": [ - "nixos/nixpkgs", "ofborg/testpkgs" ], "disable_trusted_users": true diff --git a/ofborg-send-event/src/main.rs b/ofborg-send-event/src/main.rs index b6ef9942..1d438055 100644 --- a/ofborg-send-event/src/main.rs +++ b/ofborg-send-event/src/main.rs @@ -20,7 +20,7 @@ struct Args { webhook_receiver_url: String, /// PR that should be fetched - #[arg(long)] + #[arg()] pr_nr: u64, /// Webhook event name, e.g. push, pull_request, ping From 0c28843b3a7713525858fc1fb051982b2e66cab5 Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 18:57:44 +0200 Subject: [PATCH 16/17] chore: drop no longer needed components - building in ofborg and some refactorings --- Architecture.md | 112 +++------------- Cargo.lock | 1 - README.md | 24 +--- mprocs.yaml | 15 --- ofborg/Cargo.toml | 3 +- ofborg/src/acl.rs | 11 -- ofborg/src/bin/build-faker.rs | 62 --------- ofborg/src/bin/builder.rs | 114 ---------------- ofborg/src/bin/github-webhook-receiver.rs | 37 +++++- ofborg/src/bin/log-message-collector.rs | 77 ----------- ofborg/src/bin/logapi.rs | 154 ---------------------- ofborg/src/bin/mass-rebuilder.rs | 1 - ofborg/src/config.rs | 10 -- ofborg/src/tasks/evaluate.rs | 132 +++++++------------ ofborg/src/tasks/mod.rs | 1 - 15 files changed, 102 insertions(+), 652 deletions(-) delete mode 100644 ofborg/src/bin/build-faker.rs delete mode 100644 ofborg/src/bin/builder.rs delete mode 100644 ofborg/src/bin/log-message-collector.rs delete mode 100644 ofborg/src/bin/logapi.rs diff --git a/Architecture.md b/Architecture.md index 20628586..cb10abbc 100644 --- a/Architecture.md +++ b/Architecture.md @@ -12,15 +12,12 @@ component runs as an independent process communicating exclusively through excha | Binary | Purpose | Consumes From | Publishes To | |--------|---------|---------------|--------------| | `github-webhook-receiver` | HTTP server receiving GitHub webhooks. Validates HMAC-SHA256 signatures, parses the event type, and routes raw JSON to RabbitMQ. | GitHub HTTP POST | `github-events` (topic exchange) | -| `github-comment-filter` | Consumes issue comment events, parses `@ofborg` commands (`build` / `eval`), checks ACL (repo authorization, trusted users), fetches PR data from GitHub API, and schedules build/eval jobs. | `build-inputs` queue | `build-jobs` (fanout exchange), `mass-rebuild-check-jobs` (queue via default exchange), `build-results` (fanout exchange) | +| `github-comment-filter` | Consumes issue comment events, parses `@ofborg` commands (`build` / `eval`), checks ACL (repo authorization, trusted users), fetches PR data from GitHub API, and schedules build/eval jobs. | `build-inputs` queue | `build-inputs-{system}` queues (via default exchange), `mass-rebuild-check-jobs` (queue via default exchange), `build-results` (fanout exchange) | | `evaluation-filter` | Filters pull request events for mass rebuild evaluation. Only processes Opened, Synchronize, Reopened, and Edited (base change) events on authorized repos. | `mass-rebuild-check-inputs` queue | `mass-rebuild-check-jobs` (queue via default exchange) | -| `mass-rebuilder` | Runs nixpkgs evaluation to determine which packages a PR impacts. Clones nixpkgs, merges the PR, runs nix-instantiate on release expressions, computes the diff of out-paths, and schedules builds for changed packages. | `mass-rebuild-check-jobs` queue | `build-jobs` (fanout exchange), `build-results` (fanout exchange) | -| `builder` | Performs actual Nix builds on a specific system architecture. Clones the repo, fetches the PR, merges, partitions attrs into buildable/non-buildable, runs nix-build, streams log lines, and publishes results. One instance per architecture. | `build-inputs-{system}` queue (via `build-jobs` fanout) | `logs` (topic exchange), `build-results` (fanout exchange) | +| `mass-rebuilder` | Runs nixpkgs evaluation to determine which packages a PR impacts. Clones nixpkgs, merges the PR, runs nix-instantiate on release expressions, computes the diff of out-paths, and optionally publishes evaluation results to the hydra evaluator. | `mass-rebuild-check-jobs` queue | `hydra-eval-jobs` (queue via default exchange) | | `github-comment-poster` | Creates and updates GitHub Check Runs on pull requests. On receipt of `QueuedBuildJobs` creates an in-progress check; on receipt of `BuildResult` creates a completed check run. | `build-results` queue (via `build-results` fanout) | GitHub Check Runs API | -| `log-message-collector` | Persists build log messages to the filesystem. Receives `BuildLogStart`, `BuildLogMsg`, and `BuildResult` messages and writes metadata, log lines, and result JSON to disk. | Server-named exclusive queue (via `logs` topic exchange) | Filesystem (log storage) | -| `logapi` | HTTP server that serves build log metadata and results via JSON API at `GET /logs/{repo}`. | Filesystem (log storage) | HTTP JSON responses | | `stats` | Dual-purpose binary: (1) Prometheus metrics HTTP server at `/metrics`, (2) consumer of `EventMessage` payloads from the stats exchange for metric collection. | `stats-events` queue (via `stats` fanout) | HTTP Prometheus `/metrics` | -| `build-faker` | Debug tool that injects a fake `BuildJob` directly into a system queue for testing. | CLI arguments | `build-inputs-{system}` queue | + ## Data Flow @@ -42,14 +39,17 @@ When someone comments `@ofborg build` on a PR: 1. **`github-comment-filter`** consumes the `issue_comment.*` event from `build-inputs` queue. 2. It parses the comment, checks the ACL (authorized repos, trusted users), fetches PR data from the GitHub API, and publishes: - - A `BuildJob` message to the `build-jobs` **fanout exchange** (distributed to all per-system queues). + - A `BuildJob` message directly to `build-inputs-{system}` queues (one per architecture, via default exchange). - A `QueuedBuildJobs` message to the `build-results` **fanout exchange** (for creating in-progress check runs). ``` github-events ──issue_comment.*──▶ build-inputs ──▶ github-comment-filter - ├──▶ build-jobs (fanout) - └──▶ build-results (fanout) + ├──▶ build-inputs-x86_64-linux + ├──▶ build-inputs-aarch64-linux + ├──▶ build-inputs-x86_64-darwin + ├──▶ build-inputs-aarch64-darwin + └──▶ build-results (fanout) ``` ### 3. Mass Rebuild Flow (Pull Requests) @@ -71,36 +71,13 @@ github-events ──pull_request.*──▶ mass-rebuild-check-inputs ──▶ 4. **`mass-rebuilder`** consumes `EvaluationJob` from `mass-rebuild-check-jobs`. It clones nixpkgs, checks out the target branch, merges the PR, runs nix-instantiate on `nixos/release.nix` and `pkgs/top-level/release.nix`, computes the diff of out-paths between base and merge, and - publishes `BuildJob` messages for changed packages to the `build-jobs` fanout exchange. - -``` -mass-rebuild-check-jobs ──▶ mass-rebuilder ──▶ build-jobs (fanout) - └──▶ build-results (fanout) -``` - -### 4. Building - -**`builder`** instances consume `BuildJob` messages from per-system queues -(`build-inputs-x86_64-linux`, `build-inputs-aarch64-linux`, `build-inputs-x86_64-darwin`, -`build-inputs-aarch64-darwin`), all bound to the `build-jobs` fanout exchange. Each builder: - -1. Clones the repo, fetches the PR, and merges. -2. Partitions attrs into buildable/non-buildable via nix-instantiate. -3. Runs nix-build on buildable attrs. -4. Streams log lines to the `logs` topic exchange with routing key `{repo}.{pr}`. -5. Publishes `BuildResult` to the `build-results` fanout exchange. + optionally publishes evaluation results to the `hydra-eval-jobs` queue. ``` -build-jobs (fanout) ──▶ build-inputs-x86_64-linux ──▶ builder - ──▶ build-inputs-aarch64-linux ──▶ builder - ──▶ build-inputs-x86_64-darwin ──▶ builder - ──▶ build-inputs-aarch64-darwin ──▶ builder - -builder ──▶ logs (topic) - └──▶ build-results (fanout) +mass-rebuild-check-jobs ──▶ mass-rebuilder ──▶ hydra-eval-jobs ``` -### 5. Build Results +### 4. Build Results **`github-comment-poster`** consumes from the `build-results` queue (bound to the `build-results` fanout exchange). It receives two message types: @@ -111,25 +88,7 @@ fanout exchange). It receives two message types: build-results (fanout) ──▶ build-results queue ──▶ github-comment-poster ──▶ GitHub Checks API ``` -### 6. Log Collection - -**`log-message-collector`** consumes from an exclusive queue bound to the `logs` topic exchange with -routing key `*.*`. It receives `BuildLogStart`, `BuildLogMsg`, and `BuildResult` messages and -persists them to disk: -- Metadata: `{logs_path}/{routing_key}/{attempt_id}.metadata.json` -- Log lines: `{logs_path}/{routing_key}/{attempt_id}` -- Build result: `{logs_path}/{routing_key}/{attempt_id}.result.json` - -**`logapi`** is an HTTP server that serves this stored data via `GET /logs/{repo}` as JSON. - -``` -logs (topic) ──▶ exclusive queue ──▶ log-message-collector ──▶ disk - │ - ▼ - logapi (HTTP) -``` - -### 7. Stats / Metrics +### 5. Stats / Metrics Components publish `EventMessage` payloads to the `stats` fanout exchange. The `stats` binary consumes them from the `stats-events` queue and exposes collected metrics as Prometheus text format @@ -146,9 +105,7 @@ components ──▶ stats (fanout) ──▶ stats-events queue ──▶ stats | Exchange | Type | Purpose | |----------|------|---------| | `github-events` | Topic | Ingress for all raw GitHub webhook events. Routing keys: `{event}.{owner}/{repo}` | -| `build-jobs` | Fanout | Distributes `BuildJob` messages to all per-system build queues | | `build-results` | Fanout | Distributes `BuildResult` and `QueuedBuildJobs` to all interested consumers | -| `logs` | Topic | Distributes build log messages. Routing keys: `{repo}.{pr}` | | `stats` | Fanout | Distributes `EventMessage` metrics payloads | ### Queues @@ -159,12 +116,7 @@ components ──▶ stats (fanout) ──▶ stats-events queue ──▶ stats | `github-events-unknown` | Durable | `github-events` | `unknown.*` | — (no consumer) | | `mass-rebuild-check-inputs` | Durable | `github-events` | `pull_request.*` (webhook-receiver), `pull_request.nixos/*` (evaluation-filter) | `evaluation-filter` | | `mass-rebuild-check-jobs` | Durable | Default exchange | `mass-rebuild-check-jobs` (routing key = queue name) | `mass-rebuilder` | -| `build-inputs-x86_64-linux` | Durable | `build-jobs` (fanout) | — | `builder` (x86_64-linux) | -| `build-inputs-aarch64-linux` | Durable | `build-jobs` (fanout) | — | `builder` (aarch64-linux) | -| `build-inputs-x86_64-darwin` | Durable | `build-jobs` (fanout) | — | `builder` (x86_64-darwin) | -| `build-inputs-aarch64-darwin` | Durable | `build-jobs` (fanout) | — | `builder` (aarch64-darwin) | | `build-results` | Durable | `build-results` (fanout) | — | `github-comment-poster` | -| `logs` | Exclusive, auto-delete | `logs` (topic) | `*.*` | `log-message-collector` | | `stats-events` | Durable | `stats` (fanout) | — | `stats` | ## RabbitMQ Graph @@ -172,12 +124,10 @@ components ──▶ stats (fanout) ──▶ stats-events queue ──▶ stats ```mermaid graph classDef component fill:#f96 - classDef debug fill:#999999 classDef exc fill:#08f108 subgraph Legend app-example(This is an application):::component - debug-example(This is a debug application):::debug exchange-example{{This is a RabbitMQ exchange}}:::exc queue-example[/This is a RabbitMQ queue/] end @@ -188,70 +138,38 @@ graph github-events-unknown[/github-events-unknown/] mass-rebuild-check-inputs[/mass-rebuild-check-inputs/] github-comment-filter(github-comment-filter):::component - build-jobs{{build-jobs}}:::exc evaluation-filter(evaluation-filter):::component mass-rebuild-check-jobs[/mass-rebuild-check-jobs/] mass-rebuilder(mass-rebuilder):::component - build-inputs-x86_64-linux[/build-inputs-x86_64-linux/] - build-inputs-aarch64-linux[/build-inputs-aarch64-linux/] - build-inputs-x86_64-darwin[/build-inputs-x86_64-darwin/] - build-inputs-aarch64-darwin[/build-inputs-aarch64-darwin/] - builder(builder):::component - logs{{logs}}:::exc - logs-queue[/logs/] - log-message-collector(log-message-collector):::component - log-storage[(Log Storage)] - logapi(logapi):::component build-results{{build-results}}:::exc build-results-queue[/build-results/] github-comment-poster(github-comment-poster):::component stats{{stats}}:::exc stats-events[/stats-events/] stats-rs(stats):::component - build-faker(build-faker):::debug github-webhook-receiver --> github-events github-events -->|issue_comment.*| build-inputs github-events -->|unknown.*| github-events-unknown github-events -->|pull_request.*| mass-rebuild-check-inputs build-inputs --> github-comment-filter - github-comment-filter --> build-jobs github-comment-filter --> mass-rebuild-check-jobs mass-rebuild-check-inputs --> evaluation-filter evaluation-filter --> mass-rebuild-check-jobs mass-rebuild-check-jobs --> mass-rebuilder - mass-rebuilder --> build-jobs - mass-rebuilder --> build-results - build-jobs --> build-inputs-x86_64-linux - build-jobs --> build-inputs-aarch64-linux - build-jobs --> build-inputs-x86_64-darwin - build-jobs --> build-inputs-aarch64-darwin - build-inputs-x86_64-linux --> builder - build-inputs-aarch64-linux --> builder - build-inputs-x86_64-darwin --> builder - build-inputs-aarch64-darwin --> builder - builder --> logs - builder --> build-results - logs --> logs-queue - logs-queue --> log-message-collector - log-message-collector --> log-storage - log-storage --> logapi build-results --> build-results-queue build-results-queue --> github-comment-poster stats --> stats-events stats-events --> stats-rs - build-faker --> build-inputs-aarch64-linux ``` ## Message Types | Message | Serialization | Fields | Published To | |---------|---------------|--------|--------------| -| `BuildJob` | JSON | `repo`, `pr`, `system`, `attrs`, `checkout` | `build-jobs` exchange | +| `BuildJob` | JSON | `repo`, `pr`, `system`, `attrs`, `checkout` | `build-inputs-{system}` queues (via default exchange) | | `EvaluationJob` | JSON | `repo`, `pr`, `system` | `mass-rebuild-check-jobs` queue | | `QueuedBuildJobs` | JSON | `repo`, `pr`, `attempt_id`, `total_jobs` | `build-results` exchange | -| `BuildResult` | JSON | `repo`, `pr`, `attempt_id`, `status` (Success/Failure), `system`, `attrs` | `build-results` exchange, `logs` exchange | -| `BuildLogStart` | JSON | `repo`, `pr`, `attempt_id`, `system` | `logs` exchange | -| `BuildLogMsg` | JSON | `repo`, `pr`, `attempt_id`, `line`, `msg` | `logs` exchange | +| `BuildResult` | JSON | `repo`, `pr`, `attempt_id`, `status` (Success/Failure), `system`, `attrs` | `build-results` exchange | | `EventMessage` | JSON | `event` (tagged enum), `repo`, `pr`, `value` | `stats` exchange | | Raw webhook JSON | JSON | Raw GitHub webhook payload | `github-events` exchange | diff --git a/Cargo.lock b/Cargo.lock index 3a66cee1..2b800c5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2607,7 +2607,6 @@ dependencies = [ "chrono", "fs2", "futures", - "futures-util", "hex", "hmac 0.13.0", "http", diff --git a/README.md b/README.md index 9e4dcc37..35b0036f 100644 --- a/README.md +++ b/README.md @@ -6,30 +6,13 @@ 2. Be gentle; try not to run mass rebuilds or massive builds (like Chromium) on it. -## Automatic Building +## Automatic Evaluation -All users will have their PRs automatically trigger builds if their commits +All users will have their PRs automatically evaluated if their commits follow the well-defined format of Nixpkgs. Specifically: prefixing the commit title with the package attribute. This includes package bumps as well as other changes. -Example commit titles and the builds they will start: - -| Message | Automatic Build | -|-----------------------------------------------------------------------|----------------------------------------------------------| -| `vim: 1.0.0 -> 2.0.0` | `vim` | -| `vagrant: Fix dependencies for version 2.0.2 ` | `vagrant` | -| `python36Packages.requests,python27Packages.requests: 1.0.0 -> 2.0.0` | `python36Packages.requests`, `python27Packages.requests` | -| `python{27,310}Packages.requests: 1.0.0 -> 2.0.0` | `python27Packages.requests`, `python310Packages.requests` | - -When opening a PR with multiple commits, ofborg creates a single build job for -all detected packages. If multiple commits get pushed to a PR one-by-one, each -detected package will get a separate build job. - -If the title of a PR begins with `WIP:` or contains `[WIP]` anywhere, its -packages are not built automatically. -**Note**: Marking a PR as a draft does not prevent automatic builds. - ## Commands The comment parser is line-based, so commentary can be interwoven with @@ -230,7 +213,4 @@ This will override the default of `-D warnings` set in [`shell.nix`](./shell.nix), which tells Rust to error if it detects any warnings. -# Running a builder -If you want to run a builder of your own, check out the [wiki page on operating -a builder](https://github.com/NixOS/ofborg/wiki/Operating-a-Builder/). diff --git a/mprocs.yaml b/mprocs.yaml index dd016cdc..718bae25 100644 --- a/mprocs.yaml +++ b/mprocs.yaml @@ -9,11 +9,6 @@ procs: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-rabbitmq.sh env: RUST_LOG: info - # no longer in use - # builder: - # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin builder .ofborg-data/local.json - # env: - # RUST_LOG: info evaluation-filter: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin evaluation-filter .ofborg-data/local.json env: @@ -26,16 +21,6 @@ procs: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-webhook-receiver .ofborg-data/local.json env: RUST_LOG: info - # no longer in use - # log-message-collector: - # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin log-message-collector .ofborg-data/local.json - # env: - # RUST_LOG: info - # no longer in use - # logapi: - # shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin logapi .ofborg-data/local.json - # env: - # RUST_LOG: info mass-rebuilder: shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin mass-rebuilder .ofborg-data/local.json env: diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index cc38872a..46e7340d 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -16,7 +16,6 @@ chrono = { version = "0.4.38", default-features = false, features = [ ] } fs2 = "0.4.3" futures = "0.3.31" -futures-util = "0.3.31" hex = "0.4.3" hmac = "0.13.0" http = "1" @@ -46,7 +45,7 @@ serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.135" sha2 = "0.11.0" tempfile = "3.15.0" -tokio = { version = "1", features = ["rt-multi-thread", "net", "macros", "sync"] } +tokio = { version = "1", features = ["full"] } tokio-stream = "0.1" tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = ["json", "env-filter"] } diff --git a/ofborg/src/acl.rs b/ofborg/src/acl.rs index 805fbb3b..cda86fde 100644 --- a/ofborg/src/acl.rs +++ b/ofborg/src/acl.rs @@ -35,17 +35,6 @@ impl Acl { } } - pub fn build_job_destinations_for_user_repo( - &self, - user: &str, - repo: &str, - ) -> Vec<(Option, Option)> { - self.build_job_architectures_for_user_repo(user, repo) - .iter() - .map(|system| system.as_build_destination()) - .collect() - } - pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool { if let Some(ref users) = self.trusted_users { if repo.to_lowercase() == "nixos/nixpkgs" { diff --git a/ofborg/src/bin/build-faker.rs b/ofborg/src/bin/build-faker.rs deleted file mode 100644 index c8475352..00000000 --- a/ofborg/src/bin/build-faker.rs +++ /dev/null @@ -1,62 +0,0 @@ -use lapin::message::Delivery; - -use ofborg::commentparser; -use ofborg::config; -use ofborg::easylapin; -use ofborg::message::{Pr, Repo, buildjob}; -use ofborg::notifyworker::NotificationReceiver; -use ofborg::worker; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - ofborg::setup_log(); - - let arg = std::env::args() - .nth(1) - .expect("usage: build-faker "); - let cfg = config::load(arg.as_ref()); - - let conn = easylapin::from_config(&cfg.builder.unwrap().rabbitmq).await?; - let chan = conn.create_channel().await?; - - let repo_msg = Repo { - clone_url: "https://github.com/nixos/ofborg.git".to_owned(), - full_name: "NixOS/ofborg".to_owned(), - owner: "NixOS".to_owned(), - name: "ofborg".to_owned(), - }; - - let pr_msg = Pr { - number: 42, - head_sha: "6dd9f0265d52b946dd13daf996f30b64e4edb446".to_owned(), - target_branch: Some("scratch".to_owned()), - }; - - let logbackrk = "NixOS/ofborg.42".to_owned(); - - let msg = buildjob::BuildJob { - repo: repo_msg, - pr: pr_msg, - subset: Some(commentparser::Subset::Nixpkgs), - attrs: vec!["success".to_owned()], - logs: Some((Some("logs".to_owned()), Some(logbackrk.to_lowercase()))), - statusreport: Some((None, Some("scratch".to_owned()))), - request_id: "bogus-request-id".to_owned(), - }; - - { - let deliver = Delivery::mock(0, "no-exchange".into(), "".into(), false, vec![]); - let recv = easylapin::ChannelNotificationReceiver::new(chan.clone(), deliver); - - for _i in 1..2 { - recv.tell(worker::publish_serde_action( - None, - Some("build-inputs-x86_64-darwin".to_owned()), - &msg, - )) - .await; - } - } - - Ok(()) -} diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs deleted file mode 100644 index 5f4c56aa..00000000 --- a/ofborg/src/bin/builder.rs +++ /dev/null @@ -1,114 +0,0 @@ -use std::future::Future; -use std::path::Path; -use std::pin::Pin; - -use futures_util::future; -use tracing::{error, info, warn}; - -use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::{checkout, config, tasks}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - ofborg::setup_log(); - - let arg = std::env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let cfg = config::load(arg.as_ref()); - - let Some(builder_cfg) = config::load(arg.as_ref()).builder else { - error!("No builder configuration found!"); - panic!(); - }; - - let conn = easylapin::from_config(&builder_cfg.rabbitmq).await?; - let mut handles: Vec + Send>>> = Vec::new(); - - for system in &cfg.nix.system { - handles.push(self::create_handle(&conn, &cfg, system.to_string()).await?); - } - - future::join_all(handles).await; - - drop(conn); // Close connection. - info!("Closed the session... EOF"); - Ok(()) -} - -#[allow(clippy::type_complexity)] -async fn create_handle( - conn: &lapin::Connection, - cfg: &config::Config, - system: String, -) -> anyhow::Result + Send>>> { - let mut chan = conn.create_channel().await?; - - let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); - let nix = cfg.nix().with_system(system.clone()); - - chan.declare_exchange(easyamqp::ExchangeConfig { - exchange: "build-jobs".to_owned(), - exchange_type: easyamqp::ExchangeType::Fanout, - passive: false, - durable: true, - auto_delete: false, - no_wait: false, - internal: false, - }) - .await?; - - let queue_name = if cfg.runner.build_all_jobs != Some(true) { - let queue_name = format!("build-inputs-{system}"); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: true, - exclusive: false, - auto_delete: false, - no_wait: false, - }) - .await?; - queue_name - } else { - warn!("Building all jobs, please don't use this unless you're"); - warn!("developing and have Graham's permission!"); - let queue_name = "".to_owned(); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: false, - exclusive: true, - auto_delete: true, - no_wait: false, - }) - .await?; - queue_name - }; - - chan.bind_queue(easyamqp::BindQueueConfig { - queue: queue_name.clone(), - exchange: "build-jobs".to_owned(), - routing_key: None, - no_wait: false, - }) - .await?; - - let handle = easylapin::NotifyChannel(chan) - .consume( - tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()), - easyamqp::ConsumeConfig { - queue: queue_name.clone(), - consumer_tag: format!("{}-builder", cfg.whoami()), - no_local: false, - no_ack: false, - no_wait: false, - exclusive: false, - }, - ) - .await?; - - info!("Fetching jobs from {}", &queue_name); - Ok(handle) -} diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index 0bd963fd..9cc2452f 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -247,7 +247,15 @@ where let Some(event_type) = event_type else { return Ok(response(StatusCode::BAD_REQUEST, "Missing event type")); }; - let routing_key = format!("{event_type}.{}", input.repository.full_name.to_lowercase()); + let routing_key = match event_type.as_str() { + "pull_request" | "issue_comment" => { + format!("{event_type}.{}", input.repository.full_name.to_lowercase()) + } + other => { + warn!("Received unknown event type: {other}"); + format!("unknown.{other}") + } + }; // Publish message if let Err(e) = publisher.publish("github-events", &routing_key, &raw).await { @@ -598,5 +606,32 @@ mod github_webhook_receiver { let published = publisher.get_published().await; assert_eq!(published[0].routing_key, "pull_request.test/myrepo"); } + + #[tokio::test] + async fn test_unknown_event_type_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"test"},"name":"my-repo","full_name":"test/my-repo","clone_url":"https://github.com/test/my-repo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("push")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "unknown.push"); + } } } diff --git a/ofborg/src/bin/log-message-collector.rs b/ofborg/src/bin/log-message-collector.rs deleted file mode 100644 index 81715915..00000000 --- a/ofborg/src/bin/log-message-collector.rs +++ /dev/null @@ -1,77 +0,0 @@ -use tracing::{error, info}; - -use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::{config, easylapin, tasks}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - ofborg::setup_log(); - - let arg = std::env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let cfg = config::load(arg.as_ref()); - - let Some(collector_cfg) = config::load(arg.as_ref()).log_message_collector else { - error!("No log message collector configuration found!"); - panic!(); - }; - - let conn = easylapin::from_config(&collector_cfg.rabbitmq).await?; - let mut chan = conn.create_channel().await?; - - chan.declare_exchange(easyamqp::ExchangeConfig { - exchange: "logs".to_owned(), - exchange_type: easyamqp::ExchangeType::Topic, - passive: false, - durable: true, - auto_delete: false, - no_wait: false, - internal: false, - }) - .await?; - - let queue_name = "logs".to_owned(); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: false, - exclusive: true, - auto_delete: true, - no_wait: false, - }) - .await?; - - chan.bind_queue(easyamqp::BindQueueConfig { - queue: queue_name.clone(), - exchange: "logs".to_owned(), - routing_key: Some("*.*".to_owned()), - no_wait: false, - }) - .await?; - - // Regular channel, we want prefetching here. - let handle = chan - .consume( - tasks::log_message_collector::LogMessageCollector::new( - std::path::PathBuf::from(collector_cfg.logs_path), - 100, - ), - easyamqp::ConsumeConfig { - queue: queue_name.clone(), - consumer_tag: format!("{}-log-collector", cfg.whoami()), - no_local: false, - no_ack: false, - no_wait: false, - exclusive: false, - }, - ) - .await?; - - info!("Fetching jobs from {}", &queue_name); - handle.await; - - drop(conn); // Close connection. - info!("Closed the session... EOF"); - Ok(()) -} diff --git a/ofborg/src/bin/logapi.rs b/ofborg/src/bin/logapi.rs deleted file mode 100644 index bba4c6ce..00000000 --- a/ofborg/src/bin/logapi.rs +++ /dev/null @@ -1,154 +0,0 @@ -use http::{Method, StatusCode}; -use http_body_util::Full; -use hyper::body::Bytes; -use hyper::server::conn::http1; -use hyper::service::service_fn; -use hyper::{Request, Response}; -use hyper_util::rt::TokioIo; -use ofborg::config; -use tokio::net::TcpListener; -use tracing::{error, info, warn}; - -#[derive(serde::Serialize, Default)] -struct Attempt { - metadata: Option, - result: Option, - log_url: Option, -} - -#[derive(serde::Serialize)] -struct LogResponse { - attempts: std::collections::HashMap, -} - -#[derive(Clone)] -struct LogApiConfig { - logs_path: std::path::PathBuf, - serve_root: String, -} - -fn response(status: StatusCode, body: &'static str) -> Response> { - Response::builder() - .status(status) - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -fn json_response(status: StatusCode, body: String) -> Response> { - Response::builder() - .status(status) - .header("Content-Type", "application/json") - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -async fn handle_request( - req: Request, - cfg: std::sync::Arc, -) -> Result>, hyper::Error> { - if req.method() != Method::GET { - return Ok(response(StatusCode::METHOD_NOT_ALLOWED, "")); - } - - let uri = req.uri().path().to_string(); - let Some(reqd) = uri.strip_prefix("/logs/").map(ToOwned::to_owned) else { - return Ok(response(StatusCode::NOT_FOUND, "invalid uri")); - }; - let path: std::path::PathBuf = cfg.logs_path.join(&reqd); - let Ok(path) = std::fs::canonicalize(&path) else { - return Ok(response(StatusCode::NOT_FOUND, "absent")); - }; - if !path.starts_with(&cfg.logs_path) { - return Ok(response(StatusCode::NOT_FOUND, "invalid path")); - } - let Ok(iter) = std::fs::read_dir(path) else { - return Ok(response(StatusCode::NOT_FOUND, "non dir")); - }; - - let mut attempts = std::collections::HashMap::::new(); - for e in iter { - let Ok(e) = e else { continue }; - let e_metadata = e.metadata(); - if e_metadata.as_ref().map(|v| v.is_dir()).unwrap_or(true) { - return Ok(response(StatusCode::INTERNAL_SERVER_ERROR, "dir found")); - } - - if e_metadata.as_ref().map(|v| v.is_file()).unwrap_or_default() { - let Ok(file_name) = e.file_name().into_string() else { - warn!("entry filename is not a utf-8 string: {:?}", e.file_name()); - continue; - }; - - if file_name.ends_with(".metadata.json") || file_name.ends_with(".result.json") { - let Ok(file) = std::fs::File::open(e.path()) else { - warn!("could not open file: {file_name}"); - continue; - }; - let Ok(json) = serde_json::from_reader::<_, serde_json::Value>(file) else { - warn!("file is not a valid json file: {file_name}"); - continue; - }; - let Some(attempt_id) = json - .get("attempt_id") - .and_then(|v| v.as_str()) - .map(ToOwned::to_owned) - else { - warn!("attempt_id not found in file: {file_name}"); - continue; - }; - let attempt_obj = attempts.entry(attempt_id).or_default(); - if file_name.ends_with(".metadata.json") { - attempt_obj.metadata = Some(json); - } else { - attempt_obj.result = Some(json); - } - } else { - let attempt_obj = attempts.entry(file_name.clone()).or_default(); - attempt_obj.log_url = Some(format!("{}/{reqd}/{file_name}", &cfg.serve_root)); - } - } - } - - let body = serde_json::to_string(&LogResponse { attempts }).unwrap_or_default(); - Ok(json_response(StatusCode::OK, body)) -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - ofborg::setup_log(); - - let arg = std::env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let Some(cfg) = config::load(arg.as_ref()).log_api_config else { - error!("No LogApi configuration found!"); - panic!(); - }; - - let logs_path = std::fs::canonicalize(&cfg.logs_path) - .expect("logs_path does not exist or is not accessible"); - - let api_cfg = std::sync::Arc::new(LogApiConfig { - logs_path, - serve_root: cfg.serve_root, - }); - - let addr: std::net::SocketAddr = cfg.listen.parse()?; - let listener = TcpListener::bind(addr).await?; - info!("Listening on {}", addr); - - loop { - let (stream, _) = listener.accept().await?; - let io = TokioIo::new(stream); - - let api_cfg = api_cfg.clone(); - - tokio::task::spawn(async move { - let service = service_fn(move |req| handle_request(req, api_cfg.clone())); - - if let Err(err) = http1::Builder::new().serve_connection(io, service).await { - warn!("Error serving connection: {:?}", err); - } - }); - } -} diff --git a/ofborg/src/bin/mass-rebuilder.rs b/ofborg/src/bin/mass-rebuilder.rs index 97c4932c..6507cfbf 100644 --- a/ofborg/src/bin/mass-rebuilder.rs +++ b/ofborg/src/bin/mass-rebuilder.rs @@ -68,7 +68,6 @@ async fn main() -> anyhow::Result<()> { tasks::evaluate::EvaluationWorker::new( cloner, cfg.github_app_vendingmachine(), - cfg.acl(), cfg.runner.identity.clone(), events, hydra_eval_queue, diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 02bc86f2..28cde14a 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -30,8 +30,6 @@ pub struct Config { pub mass_rebuilder: Option, /// Configuration for the hydra evaluator integration pub hydra_evaluator: Option, - /// Configuration for the builder - pub builder: Option, /// Configuration for the log message collector pub log_message_collector: Option, /// Configuration for the stats server @@ -118,14 +116,6 @@ pub struct HydraEvaluatorConfig { pub jobset_id: i32, } -/// Configuration for the builder -#[derive(serde::Serialize, serde::Deserialize, Debug)] -#[serde(deny_unknown_fields)] -pub struct Builder { - /// RabbitMQ broker to connect to - pub rabbitmq: RabbitMqConfig, -} - /// Configuration for the log message collector #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index 851a88a8..d4caf39a 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -1,5 +1,5 @@ /// This is what evaluates every pull-request -use std::io::{BufRead as _, BufReader}; +use std::io::{BufRead, BufReader}; use std::path::Path; use std::time::Instant; @@ -7,7 +7,6 @@ use futures::stream::StreamExt; use octocrab::{Octocrab, models::StatusState}; use tracing::{Instrument, debug_span, error, info, warn}; -use crate::acl::Acl; use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::config::GithubAppVendingMachine; use crate::github::GithubRepo; @@ -16,13 +15,12 @@ use crate::nix; use crate::stats::{self, Event}; use crate::tasks::eval; use crate::tasks::eval::EvaluationStrategy; -use crate::{checkout, systems, worker}; +use crate::{checkout, worker}; use uuid::Uuid; pub struct EvaluationWorker { cloner: checkout::CachedCloner, github_vend: Option>, - acl: Acl, identity: String, events: E, hydra_eval_queue: Option, @@ -35,7 +33,6 @@ impl EvaluationWorker { pub fn new( cloner: checkout::CachedCloner, github_vend: Option, - acl: Acl, identity: String, events: E, hydra_eval_queue: Option, @@ -45,7 +42,6 @@ impl EvaluationWorker { EvaluationWorker { cloner, github_vend: github_vend.map(tokio::sync::RwLock::new), - acl, identity, events, hydra_eval_queue, @@ -105,7 +101,6 @@ impl worker::SimpleWorker for EvaluationWorker OneEval::new( github_client, - &self.acl, &mut self.events, &self.identity, &self.cloner, @@ -125,7 +120,6 @@ impl worker::SimpleWorker for EvaluationWorker struct OneEval<'a, E> { repo: GithubRepo, enable_publish: bool, - acl: &'a Acl, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, @@ -141,7 +135,6 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { #[allow(clippy::borrow_as_ptr)] fn new( octocrab: Option, - acl: &'a Acl, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, @@ -165,7 +158,6 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { OneEval { repo, enable_publish, - acl, events, identity, cloner, @@ -212,12 +204,16 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { } async fn worker_actions(&mut self) -> worker::Actions { - self.prefix = Some(match self.repo.get_prefix(&self.job.pr.head_sha).await { - Ok(p) => p, - Err(e) => { - error!("Failed to determine commit status prefix: {:?}", e); - return self.actions().retry_later(self.job); + self.prefix = Some(if self.enable_publish { + match self.repo.get_prefix(&self.job.pr.head_sha).await { + Ok(p) => p, + Err(e) => { + error!("Failed to determine commit status prefix: {:?}", e); + return self.actions().retry_later(self.job); + } } + } else { + "ofborg" }); let eval_result = match self.evaluate_job().await { @@ -276,38 +272,30 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { async fn evaluate_job(&mut self) -> Result { let job = self.job; - let auto_schedule_build_archs: Vec; - - let issue = self.repo.issues().get(job.pr.number).await; - - match issue { - Ok(iss) => { - if iss.state == octocrab::models::IssueState::Closed { - self.events.notify(Event::IssueAlreadyClosed).await; - info!("Skipping {} because it is closed", job.pr.number); - return Ok(self.actions().skip(job)); + let issue = if self.enable_publish { + let issue_result = self.repo.issues().get(job.pr.number).await; + + match &issue_result { + Ok(iss) => { + if iss.state == octocrab::models::IssueState::Closed { + self.events.notify(Event::IssueAlreadyClosed).await; + info!("Skipping {} because it is closed", job.pr.number); + return Ok(self.actions().skip(job)); + } } - if issue_is_wip(&iss) { - auto_schedule_build_archs = vec![]; - } else { - auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo( - &iss.user.login, - &job.repo.full_name, - ); + Err(e) => { + self.events.notify(Event::IssueFetchFailed).await; + error!("Error fetching {}!", job.pr.number); + error!("E: {:?}", e); + return Ok(self.actions().skip(job)); } - } + }; - Err(e) => { - self.events.notify(Event::IssueFetchFailed).await; - error!("Error fetching {}!", job.pr.number); - error!("E: {:?}", e); - return Ok(self.actions().skip(job)); - } + issue_result.ok() + } else { + None }; - - let issue = self.repo.issues().get(job.pr.number).await.ok(); - let mut evaluation_strategy = eval::NixpkgsStrategy::new(job, issue.as_ref()); let prefix = self @@ -488,11 +476,6 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .all_evaluations_passed(&mut overall_status) .await?; - response.extend(schedule_builds( - complete.builds.clone(), - auto_schedule_build_archs, - )); - if let (Some(ref queue), Some(ref nix), Some(jobset_id)) = ( self.hydra_eval_queue.clone(), self.hydra_eval_nix.clone(), @@ -539,38 +522,6 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { } } -fn schedule_builds( - builds: Vec, - auto_schedule_build_archs: Vec, -) -> Vec { - let mut response = vec![]; - info!( - "Scheduling build jobs {:?} on arches {:?}", - builds, auto_schedule_build_archs - ); - for buildjob in builds { - for arch in auto_schedule_build_archs.iter() { - let (exchange, routingkey) = arch.as_build_destination(); - response.push(worker::publish_serde_action( - exchange, routingkey, &buildjob, - )); - } - response.push(worker::publish_serde_action( - Some("build-results".to_string()), - None, - &buildjob::QueuedBuildJobs { - job: buildjob, - architectures: auto_schedule_build_archs - .iter() - .map(|arch| arch.to_string()) - .collect(), - }, - )); - } - - response -} - fn resolve_attrs_to_drv_paths( nix: &nix::Nix, nixpkgs: &std::path::Path, @@ -593,6 +544,20 @@ fn resolve_attrs_to_drv_paths( }) .unwrap_or(nix::File::DefaultNixpkgs); + // Try batch instantiation first for performance (single nix-instantiate process) + match nix.safely_instantiate_attrs(nixpkgs, file, all_attrs.clone()) { + Ok(f) => { + return BufReader::new(f) + .lines() + .map_while(Result::ok) + .filter(|line| line.trim().ends_with(".drv")) + .map(|line| line.trim().to_owned()) + .collect(); + } + Err(_) => warn!("Batch instantiation failed, falling back to per-attr fallback"), + } + + // Fallback: try each attr individually all_attrs .into_iter() .flat_map( @@ -606,7 +571,10 @@ fn resolve_attrs_to_drv_paths( Err(f) => { let stderr: Vec = BufReader::new(f).lines().map_while(Result::ok).collect(); - warn!("nix-instantiate failed for attrs: {:?}", stderr.join("\n")); + warn!( + "nix-instantiate failed for attr '{attr}': {:?}", + stderr.join("\n") + ); vec![] } }, @@ -614,10 +582,6 @@ fn resolve_attrs_to_drv_paths( .collect() } -fn issue_is_wip(issue: &octocrab::models::issues::Issue) -> bool { - issue.title.starts_with("WIP:") || issue.title.contains("[WIP]") -} - enum EvalWorkerError { EvalError(eval::Error), CommitStatusWrite(CommitStatusError), diff --git a/ofborg/src/tasks/mod.rs b/ofborg/src/tasks/mod.rs index 5aab0fa6..e8c292fc 100644 --- a/ofborg/src/tasks/mod.rs +++ b/ofborg/src/tasks/mod.rs @@ -1,4 +1,3 @@ -pub mod build; pub mod eval; pub mod evaluate; pub mod evaluationfilter; From b83612d2bcda3f804c58aa84212f76b85df0f54d Mon Sep 17 00:00:00 2001 From: Simon Hauser Date: Fri, 15 May 2026 19:53:02 +0200 Subject: [PATCH 17/17] docs: document hydra-evaluator dev setup --- DEVELOPMENT.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5f9cad99..3f217db5 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -27,6 +27,7 @@ This brings up: | `mass-rebuilder` | yes | Orchestrates mass rebuild jobs | | `stats` | yes | Stats server on `[::1]:9898` | | `github-comment-filter` | yes | Disabled by default; start manually in mprocs | +| `hydra-evaluator` | yes | Consumes `HydraEvalJob` messages, imports derivations into queue-runner via gRPC | ## What happens under the hood @@ -38,7 +39,18 @@ This brings up: 4. **`mprocs/bootstrap-ofborg.sh`** — Generates a random webhook secret (`.ofborg-data/.webhook-secret`) and writes a complete config at `.ofborg-data/local.json`. The config wires all services to the local RabbitMQ instance. -All services then start via `cargo r --bin .ofborg-data/local.json`, so changes to source code are reflected immediately (cargo recompiles on restart). +All services then start via `cargo r --bin .ofborg-data/local.json`, so changes to source code are reflected immediately (cargo recompiles on restart). The `hydra-evaluator` crate runs via `cargo r -p hydra-evaluator -- .ofborg-data/local.json`. + +## Hydra evaluator flow + +When the `hydra_evaluator` config section is present, the mass-rebuilder publishes `HydraEvalJob` messages to the `hydra-eval-jobs` queue after a successful evaluation. The `hydra-evaluator` binary then: + +1. Consumes `HydraEvalJob` messages from `hydra-eval-jobs`. +2. Resolves each drv path against the local Nix store. +3. Streams the corresponding NARs (zstd-compressed) to the queue-runner via the `BuildResult` gRPC stream. +4. Calls `CreateBuild` on the queue-runner to register the builds under the configured `jobset_id`. + +If the `hydra_evaluator` config is absent, the mass-rebuilder skips the hydra integration step entirely. ## Sending test webhook events