diff --git a/.mise.toml b/.mise.toml index 0547868..0d10500 100644 --- a/.mise.toml +++ b/.mise.toml @@ -158,7 +158,7 @@ description = "Build the har1 workflow WASM module" dir = "services/ws-modules/har1" run = """ wasm-pack build . --target web -cargo run -p module-manifest-to-package-json +cargo run -p et-cli -- module-package-json """ [tasks.build-ws-face-detection-module] @@ -166,7 +166,7 @@ description = "Build the face detection workflow WASM module" dir = "services/ws-modules/face-detection" run = """ wasm-pack build . --target web -cargo run -p module-manifest-to-package-json +cargo run -p et-cli -- module-package-json """ [tasks.build-ws-comm1-module] @@ -235,7 +235,7 @@ description = "Build the pydata1 Python workflow module" dir = "services/ws-modules/pydata1" run = """ uv build --wheel --out-dir pkg -cargo run -p module-manifest-to-package-json +cargo run -p et-cli -- module-package-json """ [tasks.build-ws-pyface1-module] @@ -243,7 +243,7 @@ description = "Build the pyface1 Python face detection workflow module" dir = "services/ws-modules/pyface1" run = """ uv build --wheel --out-dir pkg -cargo run -p module-manifest-to-package-json +cargo run -p et-cli -- module-package-json """ [tasks.build-ws-java-data1-module] diff --git a/CLAUDE.md b/CLAUDE.md index a34166d..8505c38 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,11 +75,11 @@ Languages: ### Utilities (`utilities/`) -- **cli** (`et-cli`) — Deployment generator: reads scenario YAML, outputs `mise.toml` or `compose.yaml` +- **cli** (`et-cli`) — Scenario and module tooling. Reads scenario YAML and outputs `mise.toml` or `compose.yaml`; + also generates module `pkg/package.json` files with `et-cli module-package-json`. + Deployment-specific generators live under `utilities/cli/src/deployment_types/`. + Module package JSON generation lives under `utilities/cli/src/module_package_json/`. - **onnx** — ONNX model utilities -- **module-manifest-to-package-json** — Generates `pkg/package.json` from module metadata. - Reads `pyproject.toml` (Python modules, via `[tool.ws-module]`) - or `Cargo.toml` (Rust modules, via `[package.metadata.ws-module]`). ### Verification (`verification/`) @@ -91,9 +91,11 @@ and must stay in sync — `mise run check` will fail if they drift. Regenerate w - Most Rust modules: `wasm-pack build . --target web` from the module directory - WASM agent (nightly, MVP target): uses `RUSTFLAGS="-C target-cpu=mvp ..."` and `RUSTUP_TOOLCHAIN=nightly` - `har1` and `face-detection`: after wasm-pack, merge extra `package.json` fields with `yq` -- Python modules: `uv build --wheel` then `cargo run -p module-manifest-to-package-json` -- Rust modules needing dependency injection: `cargo run -p module-manifest-to-package-json` +- Python modules: `uv build --wheel` then `cargo run -p et-cli -- module-package-json` +- Rust modules needing dependency injection: `cargo run -p et-cli -- module-package-json` merges `[package.metadata.ws-module.dependencies]` from `Cargo.toml` into `pkg/package.json` +- `et-cli module-package-json` reads `pyproject.toml` (Python modules, via `[tool.ws-module]`) + or `Cargo.toml` (Rust modules, via `[package.metadata.ws-module]`). - Java: `mvn package` from repo root (uses `pom.xml`) ## Observability diff --git a/Cargo.lock b/Cargo.lock index 254194a..04f3c1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2038,15 +2038,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "module-manifest-to-package-json" -version = "0.1.0" -dependencies = [ - "serde", - "serde_json", - "toml", -] - [[package]] name = "multimap" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 2ee9119..8fa54a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ members = [ "services/ws-wasm-agent", "utilities/cli", "utilities/onnx", - "utilities/module-manifest-to-package-json", ] resolver = "2" diff --git a/utilities/cli/src/docker_compose.rs b/utilities/cli/src/deployment_types/docker_compose.rs similarity index 100% rename from utilities/cli/src/docker_compose.rs rename to utilities/cli/src/deployment_types/docker_compose.rs diff --git a/utilities/cli/src/mise.rs b/utilities/cli/src/deployment_types/mise.rs similarity index 100% rename from utilities/cli/src/mise.rs rename to utilities/cli/src/deployment_types/mise.rs diff --git a/utilities/cli/src/deployment_types/mod.rs b/utilities/cli/src/deployment_types/mod.rs new file mode 100644 index 0000000..30a4991 --- /dev/null +++ b/utilities/cli/src/deployment_types/mod.rs @@ -0,0 +1,5 @@ +mod docker_compose; +mod mise; + +pub use docker_compose::{docker_image_module_paths, generate_docker_compose_deployment}; +pub use mise::{generate_mise_deployment, scenario_module_paths}; diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index bf233f7..530cf20 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -8,11 +8,13 @@ use clap::ValueEnum; use edge_toolkit::input::ClusterInput; use serde::Deserialize; -mod docker_compose; -mod mise; +mod deployment_types; +mod module_package_json; -pub use docker_compose::{docker_image_module_paths, generate_docker_compose_deployment}; -pub use mise::{generate_mise_deployment, scenario_module_paths}; +pub use deployment_types::{ + docker_image_module_paths, generate_docker_compose_deployment, generate_mise_deployment, scenario_module_paths, +}; +pub use module_package_json::generate_module_package_json; #[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq, ValueEnum)] #[serde(rename_all = "lowercase")] diff --git a/utilities/cli/src/main.rs b/utilities/cli/src/main.rs index b2b2fbb..082c1bc 100644 --- a/utilities/cli/src/main.rs +++ b/utilities/cli/src/main.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use anyhow::Result; use clap::{Parser, Subcommand}; -use et_cli::{OutputType, generate_deployment, regenerate_verification}; +use et_cli::{OutputType, generate_deployment, generate_module_package_json, regenerate_verification}; #[derive(Parser)] struct Cli { @@ -26,6 +26,11 @@ enum Commands { #[arg(long, default_value = "verification")] verification_root: PathBuf, }, + /// Generate pkg/package.json from module metadata. + ModulePackageJson { + #[arg(long, default_value = ".")] + module_dir: PathBuf, + }, } fn main() -> Result<()> { @@ -64,6 +69,10 @@ fn main() -> Result<()> { } println!("Regenerated {} verification scenario output set(s).", regenerated.len()); } + Commands::ModulePackageJson { module_dir } => { + let output_path = generate_module_package_json(module_dir)?; + println!("Wrote {}", output_path.display()); + } } Ok(()) diff --git a/utilities/module-manifest-to-package-json/src/main.rs b/utilities/cli/src/module_package_json/mod.rs similarity index 52% rename from utilities/module-manifest-to-package-json/src/main.rs rename to utilities/cli/src/module_package_json/mod.rs index b17b937..a0dde01 100644 --- a/utilities/module-manifest-to-package-json/src/main.rs +++ b/utilities/cli/src/module_package_json/mod.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +use anyhow::{Context, Result, anyhow}; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -56,27 +57,33 @@ struct CargoWsModule { dependencies: BTreeMap, } -fn main() { - let out_path = PathBuf::from("pkg/package.json"); - let package_json = if Path::new("pyproject.toml").is_file() { - package_json_from_pyproject() - } else if Path::new("Cargo.toml").is_file() { - package_json_from_cargo(&out_path) +pub fn generate_module_package_json(module_dir: &Path) -> Result { + let out_path = module_dir.join("pkg/package.json"); + let package_json = if module_dir.join("pyproject.toml").is_file() { + package_json_from_pyproject(module_dir)? + } else if module_dir.join("Cargo.toml").is_file() { + package_json_from_cargo(module_dir, &out_path)? } else { - panic!("Expected pyproject.toml or Cargo.toml in the current directory"); + return Err(anyhow!( + "Expected pyproject.toml or Cargo.toml in module directory {:?}", + module_dir + )); }; - fs::create_dir_all(out_path.parent().unwrap()).unwrap(); - let mut out = serde_json::to_string_pretty(&package_json).unwrap(); + let parent = out_path + .parent() + .ok_or_else(|| anyhow!("Output path {:?} has no parent directory", out_path))?; + fs::create_dir_all(parent).with_context(|| format!("Failed to create output directory: {:?}", parent))?; + let mut out = serde_json::to_string_pretty(&package_json).context("Failed to serialize package JSON")?; out.push('\n'); - fs::write(&out_path, &out).unwrap_or_else(|e| panic!("Failed to write {}: {e}", out_path.display())); + fs::write(&out_path, &out).with_context(|| format!("Failed to write {}", out_path.display()))?; - println!("Wrote {}", out_path.display()); + Ok(out_path) } -fn package_json_from_pyproject() -> Value { - let pyproject_path = PathBuf::from("pyproject.toml"); - let pyproject: Pyproject = read_toml(&pyproject_path); +fn package_json_from_pyproject(module_dir: &Path) -> Result { + let pyproject_path = module_dir.join("pyproject.toml"); + let pyproject: Pyproject = read_toml(&pyproject_path)?; let p = &pyproject.project; let mut pkg = Map::from_iter([ ("name".to_string(), json!(p.name)), @@ -89,12 +96,12 @@ fn package_json_from_pyproject() -> Value { if !pyproject.tool.ws_module.dependencies.is_empty() { pkg.insert("dependencies".to_string(), json!(pyproject.tool.ws_module.dependencies)); } - Value::Object(pkg) + Ok(Value::Object(pkg)) } -fn package_json_from_cargo(out_path: &Path) -> Value { - let cargo_toml: CargoPackage = read_toml(Path::new("Cargo.toml")); - let mut pkg = read_package_json(out_path).unwrap_or_else(|| { +fn package_json_from_cargo(module_dir: &Path, out_path: &Path) -> Result { + let cargo_toml: CargoPackage = read_toml(&module_dir.join("Cargo.toml"))?; + let mut pkg = read_package_json(out_path)?.unwrap_or_else(|| { let mut pkg = Map::new(); pkg.insert("name".to_string(), json!(cargo_toml.package.name)); pkg.insert("type".to_string(), json!("module")); @@ -106,7 +113,7 @@ fn package_json_from_cargo(out_path: &Path) -> Value { } let Some(ws_module) = cargo_toml.package.metadata.and_then(|metadata| metadata.ws_module) else { - return Value::Object(pkg); + return Ok(Value::Object(pkg)); }; if !ws_module.dependencies.is_empty() { @@ -115,29 +122,33 @@ fn package_json_from_cargo(out_path: &Path) -> Value { .or_insert_with(|| Value::Object(Map::new())); let dependency_map = dependencies .as_object_mut() - .unwrap_or_else(|| panic!("{} contains a non-object dependencies field", out_path.display())); + .ok_or_else(|| anyhow!("{} contains a non-object dependencies field", out_path.display()))?; for (name, version) in ws_module.dependencies { dependency_map.insert(name, json!(version)); } } - Value::Object(pkg) + Ok(Value::Object(pkg)) } -fn read_toml(path: &Path) -> T +fn read_toml(path: &Path) -> Result where T: for<'de> Deserialize<'de>, { - let src = fs::read_to_string(path).unwrap_or_else(|e| panic!("Failed to read {}: {e}", path.display())); - toml::from_str(&src).unwrap_or_else(|e| panic!("Failed to parse {}: {e}", path.display())) + let src = fs::read_to_string(path).with_context(|| format!("Failed to read {}", path.display()))?; + toml::from_str(&src).with_context(|| format!("Failed to parse {}", path.display())) } -fn read_package_json(path: &Path) -> Option> { - let src = fs::read_to_string(path).ok()?; +fn read_package_json(path: &Path) -> Result>> { + let src = match fs::read_to_string(path) { + Ok(src) => src, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error).with_context(|| format!("Failed to read {}", path.display())), + }; let Value::Object(pkg) = - serde_json::from_str(&src).unwrap_or_else(|e| panic!("Failed to parse {}: {e}", path.display())) + serde_json::from_str(&src).with_context(|| format!("Failed to parse {}", path.display()))? else { - panic!("{} must contain a JSON object", path.display()); + return Err(anyhow!("{} must contain a JSON object", path.display())); }; - Some(pkg) + Ok(Some(pkg)) } diff --git a/utilities/cli/tests/module_package_json.rs b/utilities/cli/tests/module_package_json.rs new file mode 100644 index 0000000..46ba949 --- /dev/null +++ b/utilities/cli/tests/module_package_json.rs @@ -0,0 +1,79 @@ +#![cfg(test)] + +use std::fs; + +use et_cli::generate_module_package_json; +use serde_json::Value; +use tempfile::tempdir; + +#[test] +fn module_package_json_generates_from_pyproject_metadata() { + let test_root = tempdir().unwrap(); + let module_dir = test_root.path(); + fs::write( + module_dir.join("pyproject.toml"), + r#"[project] +name = "et-ws-python-module" +version = "0.1.0" +description = "Python module" +license = "Apache-2.0" + +[tool.ws-module] +js-main = "python_module.js" + +[tool.ws-module.dependencies] +et-model-face1 = "*" +"#, + ) + .unwrap(); + + let output_path = generate_module_package_json(module_dir).unwrap(); + let package: Value = serde_json::from_str(&fs::read_to_string(output_path).unwrap()).unwrap(); + + assert_eq!(package["name"], "et-ws-python-module"); + assert_eq!(package["type"], "module"); + assert_eq!(package["description"], "Python module"); + assert_eq!(package["version"], "0.1.0"); + assert_eq!(package["license"], "Apache-2.0"); + assert_eq!(package["main"], "python_module.js"); + assert_eq!(package["dependencies"]["et-model-face1"], "*"); +} + +#[test] +fn module_package_json_merges_cargo_ws_module_dependencies() { + let test_root = tempdir().unwrap(); + let module_dir = test_root.path(); + let package_dir = module_dir.join("pkg"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + module_dir.join("Cargo.toml"), + r#"[package] +name = "et-ws-rust-module" +version = "0.1.0" +edition = "2024" + +[package.metadata.ws-module.dependencies] +et-model-har-motion1 = "*" +"#, + ) + .unwrap(); + fs::write( + package_dir.join("package.json"), + r#"{ + "type": "module", + "dependencies": { + "existing-package": "1.0.0" + } +} +"#, + ) + .unwrap(); + + let output_path = generate_module_package_json(module_dir).unwrap(); + let package: Value = serde_json::from_str(&fs::read_to_string(output_path).unwrap()).unwrap(); + + assert_eq!(package["name"], "et-ws-rust-module"); + assert_eq!(package["type"], "module"); + assert_eq!(package["dependencies"]["existing-package"], "1.0.0"); + assert_eq!(package["dependencies"]["et-model-har-motion1"], "*"); +} diff --git a/utilities/cli/tests/test_scenario_generation.rs b/utilities/cli/tests/scenario_generation.rs similarity index 99% rename from utilities/cli/tests/test_scenario_generation.rs rename to utilities/cli/tests/scenario_generation.rs index 7b93def..9e3b4b5 100644 --- a/utilities/cli/tests/test_scenario_generation.rs +++ b/utilities/cli/tests/scenario_generation.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use std::fs; use et_cli::{ diff --git a/utilities/module-manifest-to-package-json/Cargo.toml b/utilities/module-manifest-to-package-json/Cargo.toml deleted file mode 100644 index 00696da..0000000 --- a/utilities/module-manifest-to-package-json/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "module-manifest-to-package-json" -description = "Generate pkg/package.json from ws-module project metadata" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[[bin]] -name = "module-manifest-to-package-json" -path = "src/main.rs" - -[dependencies] -serde.workspace = true -serde_json.workspace = true -toml.workspace = true