From 63eee41ad8ec91b88ba27a81565124b630956869 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 24 Apr 2026 07:37:19 -0700 Subject: [PATCH 1/4] Project bundling --- Cargo.lock | 1 + crates/icp-cli/Cargo.toml | 1 + crates/icp-cli/src/commands/project/bundle.rs | 41 +++ crates/icp-cli/src/commands/project/mod.rs | 4 +- crates/icp-cli/src/main.rs | 3 + crates/icp-cli/src/operations/bundle.rs | 249 ++++++++++++++++++ crates/icp-cli/src/operations/mod.rs | 1 + crates/icp-cli/tests/bundle_tests.rs | 238 +++++++++++++++++ crates/icp/src/canister/mod.rs | 10 +- crates/icp/src/manifest/canister.rs | 6 +- crates/icp/src/manifest/environment.rs | 38 ++- crates/icp/src/manifest/mod.rs | 18 +- crates/icp/src/manifest/network.rs | 47 +++- crates/icp/src/manifest/project.rs | 4 +- crates/icp/src/manifest/recipe.rs | 10 +- docs/reference/cli.md | 22 +- docs/schemas/canister-yaml-schema.json | 13 +- docs/schemas/environment-yaml-schema.json | 1 - docs/schemas/icp-yaml-schema.json | 16 +- 19 files changed, 674 insertions(+), 49 deletions(-) create mode 100644 crates/icp-cli/src/commands/project/bundle.rs create mode 100644 crates/icp-cli/src/operations/bundle.rs create mode 100644 crates/icp-cli/tests/bundle_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 04d300437..a76ec39fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3541,6 +3541,7 @@ dependencies = [ "shellwords", "snafu", "sysinfo", + "tar", "test-tag", "time", "tiny-bip39", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index f940f7034..9d3ac7b65 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -32,6 +32,7 @@ dunce.workspace = true elliptic-curve.workspace = true flate2.workspace = true futures.workspace = true +tar.workspace = true hex.workspace = true httptest.workspace = true ic-agent.workspace = true diff --git a/crates/icp-cli/src/commands/project/bundle.rs b/crates/icp-cli/src/commands/project/bundle.rs new file mode 100644 index 000000000..0b758cbca --- /dev/null +++ b/crates/icp-cli/src/commands/project/bundle.rs @@ -0,0 +1,41 @@ +use anyhow::Context as _; +use clap::Args; +use icp::context::Context; +use icp::prelude::*; + +use crate::operations::bundle::create_bundle; + +/// Bundle a project into a self-contained deployable archive. +/// +/// Builds all project canisters and packages them with a rewritten manifest +/// into a `.tar.gz` file. The rewritten manifest replaces all build steps +/// with pre-built steps referencing the bundled WASM files. Asset sync +/// directories are included in the archive. +/// +/// Projects with script sync steps cannot be bundled. +#[derive(Args, Debug)] +pub(crate) struct BundleArgs { + /// Output path for the bundle archive (e.g. bundle.tar.gz) + #[arg(long, short)] + pub(crate) output: PathBuf, +} + +pub(crate) async fn exec(ctx: &Context, args: &BundleArgs) -> Result<(), anyhow::Error> { + let project = ctx.project.load().await.context("failed to load project")?; + + let canisters: Vec<_> = project.canisters.into_values().collect(); + + create_bundle( + &project.dir, + canisters, + ctx.builder.clone(), + ctx.artifacts.clone(), + &ctx.dirs.package_cache()?, + ctx.debug, + &args.output, + ) + .await + .context("failed to create bundle")?; + + Ok(()) +} diff --git a/crates/icp-cli/src/commands/project/mod.rs b/crates/icp-cli/src/commands/project/mod.rs index 48ae1348a..99a4e84e6 100644 --- a/crates/icp-cli/src/commands/project/mod.rs +++ b/crates/icp-cli/src/commands/project/mod.rs @@ -1,9 +1,11 @@ use clap::Subcommand; +pub(crate) mod bundle; pub(crate) mod show; -/// Display information about the current project +/// Manage the current project #[derive(Debug, Subcommand)] pub(crate) enum Command { Show(show::ShowArgs), + Bundle(bundle::BundleArgs), } diff --git a/crates/icp-cli/src/main.rs b/crates/icp-cli/src/main.rs index 9d580708a..b883c6a91 100644 --- a/crates/icp-cli/src/main.rs +++ b/crates/icp-cli/src/main.rs @@ -406,6 +406,9 @@ async fn dispatch(ctx: &icp::context::Context, command: Command) -> Result<(), E commands::project::Command::Show(args) => { commands::project::show::exec(ctx, &args).await? } + commands::project::Command::Bundle(args) => { + commands::project::bundle::exec(ctx, &args).await? + } }, // Settings diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs new file mode 100644 index 000000000..95726e8c0 --- /dev/null +++ b/crates/icp-cli/src/operations/bundle.rs @@ -0,0 +1,249 @@ +use std::{ + fs::File, + io::{BufWriter, Cursor}, + sync::Arc, +}; + +use sha2::{Digest, Sha256}; + +use flate2::{Compression, write::GzEncoder}; +use icp::{ + Canister, InitArgs, + canister::build::Build, + manifest::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, Item, + LoadManifestFromPathError, ManifestInitArgs, PROJECT_MANIFEST, ProjectManifest, SyncStep, + SyncSteps, assets::DirField, prebuilt, + }, + package::PackageCache, + prelude::*, + store_artifact, +}; +use snafu::{ResultExt, Snafu}; +use tar::Builder; + +use crate::operations::build::{BuildManyError, build_many_with_progress_bar}; + +#[derive(Debug, Snafu)] +pub enum BundleError { + #[snafu(display( + "canister '{canister}' has a script sync step, which is not supported in bundles" + ))] + ScriptSyncStep { canister: String }, + + #[snafu(transparent)] + Build { source: BuildManyError }, + + #[snafu(display("failed to look up built artifact for canister '{canister}'"))] + LookupArtifact { + canister: String, + source: store_artifact::LookupArtifactError, + }, + + #[snafu(display("failed to load project manifest for bundle"))] + LoadManifest { source: LoadManifestFromPathError }, + + #[snafu(display("failed to serialize bundle manifest"))] + SerializeManifest { source: serde_yaml::Error }, + + #[snafu(display("failed to add '{path}' to bundle archive"))] + WriteArchiveEntry { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to create bundle output file at '{path}'"))] + CreateOutput { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to finalize bundle archive"))] + FlushArchive { source: std::io::Error }, +} + +pub(crate) async fn create_bundle( + project_dir: &Path, + canisters: Vec<(PathBuf, Canister)>, + builder: Arc, + artifacts: Arc, + pkg_cache: &PackageCache, + debug: bool, + output: &Path, +) -> Result<(), BundleError> { + for (_, canister) in &canisters { + for step in &canister.sync.steps { + if matches!(step, SyncStep::Script(_)) { + return ScriptSyncStepSnafu { + canister: canister.name.clone(), + } + .fail(); + } + } + } + + build_many_with_progress_bar( + canisters.clone(), + builder, + artifacts.clone(), + pkg_cache, + debug, + ) + .await?; + + // Re-read the raw manifest to preserve networks and environments verbatim. + let raw_manifest: ProjectManifest = + icp::manifest::load_manifest_from_path(&project_dir.join(PROJECT_MANIFEST)) + .await + .context(LoadManifestSnafu)?; + + let mut bundle_canisters = Vec::new(); + let mut canister_wasms: Vec<(String, Vec)> = Vec::new(); + // (canister_path, canister_name, asset_dirs) + let mut asset_dirs: Vec<(PathBuf, String, Vec)> = Vec::new(); + + for (canister_path, canister) in &canisters { + let wasm = artifacts + .lookup(&canister.name) + .await + .context(LookupArtifactSnafu { + canister: canister.name.clone(), + })?; + + let sha256 = hex::encode(Sha256::digest(&wasm)); + let wasm_filename = format!("{}.wasm", canister.name); + + // Collect asset dirs and rewrite their paths for the bundle. + let mut bundle_sync_steps = Vec::new(); + let mut raw_asset_dirs = Vec::new(); + + for step in &canister.sync.steps { + match step { + SyncStep::Script(_) => unreachable!("validated above"), + SyncStep::Assets(adapter) => { + let dirs = adapter.dir.as_vec(); + raw_asset_dirs.extend(dirs.clone()); + + let prefixed_dirs: Vec = dirs + .iter() + .map(|d| format!("{}/{d}", canister.name)) + .collect(); + + let new_dir = if prefixed_dirs.len() == 1 { + DirField::Dir(prefixed_dirs.into_iter().next().unwrap()) + } else { + DirField::Dirs(prefixed_dirs) + }; + + bundle_sync_steps.push(SyncStep::Assets(icp::manifest::assets::Adapter { + dir: new_dir, + })); + } + } + } + + if !raw_asset_dirs.is_empty() { + asset_dirs.push((canister_path.clone(), canister.name.clone(), raw_asset_dirs)); + } + + let init_args = canister.init_args.as_ref().map(convert_init_args); + + let sync = if bundle_sync_steps.is_empty() { + None + } else { + Some(SyncSteps { + steps: bundle_sync_steps, + }) + }; + + let canister_manifest = CanisterManifest { + name: canister.name.clone(), + settings: canister.settings.clone(), + init_args, + instructions: Instructions::BuildSync { + build: BuildSteps { + steps: vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(prebuilt::LocalSource { + path: wasm_filename.as_str().into(), + }), + sha256: Some(sha256), + })], + }, + sync, + }, + }; + + bundle_canisters.push(Item::Manifest(canister_manifest)); + canister_wasms.push((wasm_filename, wasm)); + } + + let bundle_manifest = ProjectManifest { + canisters: bundle_canisters, + networks: raw_manifest.networks, + environments: raw_manifest.environments, + }; + + let manifest_yaml = serde_yaml::to_string(&bundle_manifest).context(SerializeManifestSnafu)?; + + let file = File::create(output.as_std_path()).context(CreateOutputSnafu { + path: output.to_path_buf(), + })?; + let gz = GzEncoder::new(BufWriter::new(file), Compression::default()); + let mut archive = Builder::new(gz); + + // icp.yaml + let manifest_bytes = manifest_yaml.as_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "icp.yaml", Cursor::new(manifest_bytes)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from("icp.yaml"), + })?; + + // WASM files + for (filename, wasm) in &canister_wasms { + let mut header = tar::Header::new_gnu(); + header.set_size(wasm.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, filename, Cursor::new(wasm)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(filename), + })?; + } + + // Asset directories + for (canister_path, canister_name, dirs) in &asset_dirs { + for dir in dirs { + let src_path = canister_path.join(dir); + let archive_prefix = format!("{canister_name}/{dir}"); + archive + .append_dir_all(&archive_prefix, src_path.as_std_path()) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(&archive_prefix), + })?; + } + } + + let gz = archive.into_inner().context(FlushArchiveSnafu)?; + gz.finish().context(FlushArchiveSnafu)?; + + Ok(()) +} + +fn convert_init_args(args: &InitArgs) -> ManifestInitArgs { + match args { + InitArgs::Text { content, format } => ManifestInitArgs::Value { + value: content.clone(), + format: format.clone(), + }, + InitArgs::Binary(bytes) => ManifestInitArgs::Value { + value: hex::encode(bytes), + format: ArgsFormat::Hex, + }, + } +} diff --git a/crates/icp-cli/src/operations/mod.rs b/crates/icp-cli/src/operations/mod.rs index f1d8ba2b7..a53b619a0 100644 --- a/crates/icp-cli/src/operations/mod.rs +++ b/crates/icp-cli/src/operations/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod binding_env_vars; pub(crate) mod build; +pub(crate) mod bundle; pub(crate) mod candid_compat; pub(crate) mod canister_migration; pub(crate) mod create; diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs new file mode 100644 index 000000000..80f879cd2 --- /dev/null +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -0,0 +1,238 @@ +use std::{ + fs, + io::{BufReader, Read as _}, +}; + +use flate2::bufread::GzDecoder; +use icp::{ + fs::{create_dir_all, json, write_string}, + prelude::*, + store_id::IdMapping, +}; +use indoc::formatdoc; +use predicates::{ + ord::eq, + prelude::PredicateBooleanExt, + str::{PredicateStrExt, contains}, +}; +use tar::Archive; + +use crate::common::{ENVIRONMENT_RANDOM_PORT, NETWORK_RANDOM_PORT, TestContext, clients}; + +mod common; + +/// Bundle a standard frontend-backend project: a script-built backend canister and an +/// asset-canister-recipe frontend canister with an assets sync step. +/// Verify archive structure, manifest content, and that the bundle deploys successfully. +#[tokio::test] +async fn bundle_and_deploy() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + // A small WASM to serve as the pre-built artifact for the backend. + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Create asset directory for the frontend canister. + let asset_dir = project_dir.join("www"); + create_dir_all(&asset_dir).expect("failed to create asset dir"); + write_string(&asset_dir.join("index.html"), "hello").expect("failed to write asset file"); + + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.1.0" + configuration: + dir: www + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + assert!(bundle_path.exists(), "bundle file was not created"); + + // Extract and inspect archive contents. + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_manifest = false; + let mut found_backend_wasm = false; + let mut found_frontend_wasm = false; + let mut found_asset = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + found_manifest = true; + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "backend.wasm" => { + found_backend_wasm = true; + } + "frontend.wasm" => { + found_frontend_wasm = true; + } + p if p.starts_with("frontend/www/") => { + found_asset = true; + } + _ => {} + } + } + + assert!(found_manifest, "icp.yaml not found in bundle"); + assert!(found_backend_wasm, "backend.wasm not found in bundle"); + assert!(found_frontend_wasm, "frontend.wasm not found in bundle"); + assert!( + found_asset, + "asset file not found under frontend/www/ in bundle" + ); + + // Manifest must contain pre-built steps and no script or recipe steps. + assert!( + manifest_yaml.contains("pre-built"), + "bundle manifest should have pre-built build steps" + ); + assert!( + !manifest_yaml.contains("type: script"), + "bundle manifest should not contain script steps" + ); + assert!( + !manifest_yaml.contains("recipe:"), + "bundle manifest should not contain recipe sections" + ); + assert!( + manifest_yaml.contains("sha256:"), + "bundle manifest should include sha256 for pre-built wasms" + ); + + // Extract bundle to a fresh directory and deploy from it. + let bundle_dir = project_dir.join("bundle-extracted"); + create_dir_all(&bundle_dir).expect("failed to create bundle-extracted dir"); + + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + archive + .unpack(bundle_dir.as_std_path()) + .expect("failed to extract bundle"); + + let _g = ctx.start_network_in(&bundle_dir, "random-network").await; + ctx.ping_until_healthy(&bundle_dir, "random-network"); + + let network_port = ctx + .wait_for_network_descriptor(&bundle_dir, "random-network") + .gateway_port; + + clients::icp(&ctx, &bundle_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "deploy", + "--subnet", + common::SUBNET_ID, + "--environment", + "random-environment", + ]) + .assert() + .success(); + + // Verify the backend canister responds to a query call. + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "canister", + "call", + "--environment", + "random-environment", + "backend", + "greet", + "(\"world\")", + ]) + .assert() + .success() + .stdout(eq("(\"Hello, world!\")").trim()); + + // Verify the frontend canister serves the bundled asset. + let id_mapping: IdMapping = json::load( + &bundle_dir + .join(".icp") + .join("cache") + .join("mappings") + .join("random-environment.ids.json"), + ) + .expect("failed to read ID mapping"); + + let frontend_cid = id_mapping + .get("frontend") + .expect("frontend canister ID not found"); + + let resp = reqwest::get(format!( + "http://localhost:{network_port}/?canisterId={frontend_cid}" + )) + .await + .expect("request to frontend canister failed"); + + assert_eq!( + resp.text().await.expect("failed to read response body"), + "hello" + ); +} + +/// Projects with script sync steps must be rejected with a clear error. +#[test] +fn bundle_rejects_script_sync_step() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + let pm = r#" +canisters: + - name: my-canister + build: + steps: + - type: script + command: echo build + sync: + steps: + - type: script + command: echo sync +"#; + + write_string(&project_dir.join("icp.yaml"), pm).expect("failed to write project manifest"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", "bundle.tar.gz"]) + .assert() + .failure() + .stderr(contains("my-canister").and(contains("script sync step"))); +} diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index 791ca7888..e1f895854 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -144,40 +144,48 @@ impl From for LogVisibility { #[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Settings { /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] pub log_visibility: Option, /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] pub compute_allocation: Option, /// Memory allocation in bytes. If unset, memory is allocated dynamically. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub memory_allocation: Option, /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] pub freezing_threshold: Option, /// Upper limit on cycles reserved for future resource payments. /// Memory allocations that would push the reserved balance above this limit will fail. /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub reserved_cycles_limit: Option, /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_limit: Option, /// Wasm memory threshold in bytes. Triggers a callback when exceeded. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_threshold: Option, /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] pub log_memory_limit: Option, /// Environment variables for the canister as key-value pairs. /// These variables are accessible within the canister and can be used to configure /// behavior without hardcoding values in the WASM module. + #[serde(skip_serializing_if = "Option::is_none")] pub environment_variables: Option>, } diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 6fafaf721..b033346a2 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -61,7 +61,7 @@ pub enum ManifestInitArgs { /// Represents the manifest describing a single canister. /// This struct is typically loaded from a `canister.yaml` file and defines /// the canister's name and how it should be built into WebAssembly. -#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] pub struct CanisterManifest { /// The unique name of the canister as defined in this manifest. pub name: String, @@ -72,6 +72,7 @@ pub struct CanisterManifest { pub settings: Settings, /// Initialization arguments passed to the canister during installation. + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option, #[serde(flatten)] @@ -236,7 +237,7 @@ impl<'de> Deserialize<'de> for CanisterManifest { } } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(untagged)] pub enum Instructions { Recipe { @@ -249,6 +250,7 @@ pub enum Instructions { build: BuildSteps, /// The configuration specifying how to sync the canister + #[serde(skip_serializing_if = "Option::is_none")] sync: Option, }, } diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp/src/manifest/environment.rs index abe73cc4d..3b9d67d29 100644 --- a/crates/icp/src/manifest/environment.rs +++ b/crates/icp/src/manifest/environment.rs @@ -1,18 +1,22 @@ use std::collections::HashMap; use schemars::JsonSchema; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize}; use crate::{canister::Settings, prelude::LOCAL}; use super::canister::ManifestInitArgs; -#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct EnvironmentInner { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub canisters: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub settings: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option>, } @@ -100,6 +104,36 @@ impl<'de> Deserialize<'de> for EnvironmentManifest { } } +impl From<&EnvironmentManifest> for EnvironmentInner { + fn from(env: &EnvironmentManifest) -> Self { + let network = if env.network == LOCAL { + None + } else { + Some(env.network.clone()) + }; + + let canisters = match &env.canisters { + CanisterSelection::Everything => None, + CanisterSelection::Named(names) => Some(names.clone()), + CanisterSelection::None => Some(vec![]), + }; + + EnvironmentInner { + name: env.name.clone(), + network, + canisters, + settings: env.settings.clone(), + init_args: env.init_args.clone(), + } + } +} + +impl Serialize for EnvironmentManifest { + fn serialize(&self, serializer: S) -> Result { + EnvironmentInner::from(self).serialize(serializer) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index fe11aac7e..d743a61e3 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use snafu::prelude::*; use crate::fs; @@ -16,7 +16,12 @@ pub(crate) mod recipe; pub(crate) mod serde_helpers; pub use { - canister::{ArgsFormat, CanisterManifest, ManifestInitArgs}, + adapter::assets, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, environment::EnvironmentManifest, network::NetworkManifest, project::ProjectManifest, @@ -41,6 +46,15 @@ pub enum Item { Manifest(T), } +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + impl<'de, T> Deserialize<'de> for Item where T: Deserialize<'de>, diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index 16ddc05d6..d892f9aa2 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -1,11 +1,11 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use url::Url; use crate::network::SubnetKind; /// A network definition for the project -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] pub struct NetworkManifest { pub name: String, @@ -13,20 +13,20 @@ pub struct NetworkManifest { pub configuration: Mode, } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(rename_all = "lowercase", tag = "mode")] pub enum Mode { Managed(Managed), Connected(Connected), } -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Managed { #[serde(flatten)] pub mode: Box, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] #[serde(untagged, rename_all_fields = "kebab-case")] #[allow(clippy::large_enum_variant)] pub enum ManagedMode { @@ -36,46 +36,68 @@ pub enum ManagedMode { /// Port mappings in the format "host_port:container_port" port_mapping: Vec, /// Whether to delete the container when the network stops + #[serde(skip_serializing_if = "Option::is_none")] rm_on_exit: Option, /// Command line arguments to pass to the container's entrypoint - #[serde(alias = "cmd", alias = "command")] + #[serde( + alias = "cmd", + alias = "command", + skip_serializing_if = "Option::is_none" + )] args: Option>, /// Entrypoint to use for the container + #[serde(skip_serializing_if = "Option::is_none")] entrypoint: Option>, /// Environment variables to set in the container in VAR=VALUE format (or VAR to inherit from host) + #[serde(skip_serializing_if = "Option::is_none")] environment: Option>, /// Volumes to mount into the container in the format name:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] volumes: Option>, /// The platform to use for the container (e.g. linux/amd64) + #[serde(skip_serializing_if = "Option::is_none")] platform: Option, /// The user to run the container as in the format user[:group] + #[serde(skip_serializing_if = "Option::is_none")] user: Option, /// The size of /dev/shm in bytes + #[serde(skip_serializing_if = "Option::is_none")] shm_size: Option, /// The status directory inside the container. Defaults to /app/status + #[serde(skip_serializing_if = "Option::is_none")] status_dir: Option, /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] mounts: Option>, /// Extra hosts entries for Docker networking (e.g. "host.docker.internal:host-gateway") + #[serde(skip_serializing_if = "Option::is_none")] extra_hosts: Option>, }, Launcher { /// HTTP gateway configuration + #[serde(skip_serializing_if = "Option::is_none")] gateway: Option, /// Artificial delay to add to every update call + #[serde(skip_serializing_if = "Option::is_none")] artificial_delay_ms: Option, /// Set up the Internet Identity canister. Makes internet identity available at /// id.ai.localhost: + #[serde(skip_serializing_if = "Option::is_none")] ii: Option, /// Set up the NNS + #[serde(skip_serializing_if = "Option::is_none")] nns: Option, /// Configure the list of subnets (one application subnet by default) + #[serde(skip_serializing_if = "Option::is_none")] subnets: Option>, /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") + #[serde(skip_serializing_if = "Option::is_none")] bitcoind_addr: Option>, /// Dogecoin P2P node addresses to connect to + #[serde(skip_serializing_if = "Option::is_none")] dogecoind_addr: Option>, /// The version of icp-cli-network-launcher to use. Defaults to the latest released version. Launcher versions correspond to published PocketIC or IC-OS releases. + #[serde(skip_serializing_if = "Option::is_none")] version: Option, }, } @@ -95,18 +117,19 @@ impl Default for ManagedMode { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct Connected { #[serde(flatten)] pub endpoints: Endpoints, /// The root key of this network + #[serde(skip_serializing_if = "Option::is_none")] #[schemars(with = "Option", regex(pattern = "^[0-9a-f]{266}$"))] pub root_key: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(untagged, rename_all_fields = "kebab-case")] pub enum Endpoints { Explicit { @@ -114,6 +137,7 @@ pub enum Endpoints { /// otherwise icp-cli will fall back to ?canisterId= query parameters which are frequently brittle in frontend code. /// /// If no HTTP gateway endpoint is provided, canister URLs will not be printed in deploy operations. + #[serde(skip_serializing_if = "Option::is_none")] http_gateway_url: Option, /// The URL of the API endpoint. Should support the standard API routes (e.g. /api/v3). api_url: Url, @@ -126,7 +150,7 @@ pub enum Endpoints { }, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(try_from = "String", into = "String")] pub struct RootKey(pub Vec); @@ -145,13 +169,16 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Gateway { /// Network interface for the gateway. Defaults to 127.0.0.1 + #[serde(skip_serializing_if = "Option::is_none")] pub bind: Option, /// Domains the gateway should respond to. Automatically includes localhost if applicable. + #[serde(skip_serializing_if = "Option::is_none")] pub domains: Option>, /// Port for the gateway to listen on. Defaults to 8000 + #[serde(skip_serializing_if = "Option::is_none")] pub port: Option, } diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index b158cbbd0..928da56b3 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -1,11 +1,11 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::manifest::{ Item, canister::CanisterManifest, environment::EnvironmentManifest, network::NetworkManifest, }; -#[derive(Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct ProjectManifest { #[serde(default)] diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp/src/manifest/recipe.rs index 107ba0fbe..31db6cb3b 100644 --- a/crates/icp/src/manifest/recipe.rs +++ b/crates/icp/src/manifest/recipe.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize, de::Error as _}; /// Represents the accepted values for a recipe type in /// the canister manifest -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema)] #[schemars(from = "String")] pub enum RecipeType { /// path to a locally defined recipe @@ -91,7 +91,13 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +impl Serialize for RecipeType { + fn serialize(&self, serializer: S) -> Result { + String::from(self.clone()).serialize(serializer) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Recipe { /// An identifier for a recipe, it can have one of the following formats: /// diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 9ab50d494..b15066166 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -66,6 +66,7 @@ This document contains the help content for the `icp` command-line program. * [`icp new`↴](#icp-new) * [`icp project`↴](#icp-project) * [`icp project show`↴](#icp-project-show) +* [`icp project bundle`↴](#icp-project-bundle) * [`icp settings`↴](#icp-settings) * [`icp settings autocontainerize`↴](#icp-settings-autocontainerize) * [`icp settings telemetry`↴](#icp-settings-telemetry) @@ -89,7 +90,7 @@ This document contains the help content for the `icp` command-line program. * `identity` — Manage your identities * `network` — Launch and manage local test networks * `new` — Create a new ICP project from a template -* `project` — Display information about the current project +* `project` — Manage the current project * `settings` — Configure user settings * `sync` — Synchronize canisters * `token` — Perform token transactions @@ -1448,13 +1449,14 @@ Under the hood templates are generated with `cargo-generate`. See the cargo-gene ## `icp project` -Display information about the current project +Manage the current project **Usage:** `icp project ` ###### **Subcommands:** * `show` — Outputs the project's effective yaml configuration +* `bundle` — Bundle a project into a self-contained deployable archive @@ -1474,6 +1476,22 @@ The effective yaml configuration includes: +## `icp project bundle` + +Bundle a project into a self-contained deployable archive. + +Builds all project canisters and packages them with a rewritten manifest into a `.tar.gz` file. The rewritten manifest replaces all build steps with pre-built steps referencing the bundled WASM files. Asset sync directories are included in the archive. + +Projects with script sync steps cannot be bundled. + +**Usage:** `icp project bundle --output ` + +###### **Options:** + +* `-o`, `--output ` — Output path for the bundle archive (e.g. bundle.tar.gz) + + + ## `icp settings` Configure user settings diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index ba1ac9637..1dcf345f0 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -394,7 +394,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -535,17 +534,7 @@ "type": "null" } ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, diff --git a/docs/schemas/environment-yaml-schema.json b/docs/schemas/environment-yaml-schema.json index 825958121..0d111b82f 100644 --- a/docs/schemas/environment-yaml-schema.json +++ b/docs/schemas/environment-yaml-schema.json @@ -206,7 +206,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index efd5d1bc7..ae6f6c98f 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -220,17 +220,7 @@ "type": "null" } ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, @@ -878,7 +868,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -973,6 +962,7 @@ "description": "Schema for ProjectManifest", "properties": { "canisters": { + "default": [], "items": { "$ref": "#/$defs/Item" }, @@ -982,6 +972,7 @@ ] }, "environments": { + "default": [], "items": { "$ref": "#/$defs/Item3" }, @@ -991,6 +982,7 @@ ] }, "networks": { + "default": [], "items": { "$ref": "#/$defs/Item2" }, From b27af6f4a550b9fa4b1229dbebeba4c77f79c1aa Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 27 Apr 2026 13:13:23 -0700 Subject: [PATCH 2/4] fix schema --- crates/icp/src/manifest/canister.rs | 1 - docs/schemas/canister-yaml-schema.json | 9 +-------- docs/schemas/icp-yaml-schema.json | 9 +-------- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index b033346a2..98d1aa0a4 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -68,7 +68,6 @@ pub struct CanisterManifest { /// The configuration specifying the various settings when creating the canister. #[serde(default)] - #[schemars(with = "Option")] pub settings: Settings, /// Initialization arguments passed to the canister during installation. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 1dcf345f0..3c69350b5 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -526,14 +526,7 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], + "$ref": "#/$defs/Settings", "default": {}, "description": "The configuration specifying the various settings when creating the canister." } diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index ae6f6c98f..0c9ff8fe1 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -212,14 +212,7 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], + "$ref": "#/$defs/Settings", "default": {}, "description": "The configuration specifying the various settings when creating the canister." } From 0fff4d3b2cf764c5960dd3cbb402e7da9d8987f9 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 27 Apr 2026 13:16:35 -0700 Subject: [PATCH 3/4] fix more schema --- crates/icp/src/manifest/project.rs | 3 --- crates/icp/src/manifest/recipe.rs | 2 +- docs/schemas/canister-yaml-schema.json | 5 +---- docs/schemas/icp-yaml-schema.json | 20 ++++---------------- 4 files changed, 6 insertions(+), 24 deletions(-) diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index 928da56b3..310b57f9f 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -9,15 +9,12 @@ use crate::manifest::{ #[serde(deny_unknown_fields)] pub struct ProjectManifest { #[serde(default)] - #[schemars(with = "Option>>")] pub canisters: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub networks: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub environments: Vec>, } diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp/src/manifest/recipe.rs index 31db6cb3b..b98105016 100644 --- a/crates/icp/src/manifest/recipe.rs +++ b/crates/icp/src/manifest/recipe.rs @@ -115,7 +115,7 @@ pub struct Recipe { pub recipe_type: RecipeType, #[serde(default)] - #[schemars(with = "Option>")] + #[schemars(with = "HashMap")] pub configuration: HashMap, /// Optional sha256 checksum for the recipe template. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 3c69350b5..441a67c4d 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -285,10 +285,7 @@ "configuration": { "additionalProperties": true, "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 0c9ff8fe1..be895ec6e 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -752,10 +752,7 @@ "configuration": { "additionalProperties": true, "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", @@ -959,30 +956,21 @@ "items": { "$ref": "#/$defs/Item" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "environments": { "default": [], "items": { "$ref": "#/$defs/Item3" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "networks": { "default": [], "items": { "$ref": "#/$defs/Item2" }, - "type": [ - "array", - "null" - ] + "type": "array" } }, "title": "ProjectManifest", From a8b1666e730fc766ee77bf8338d5ba64fd04ce89 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 29 Apr 2026 17:41:33 -0700 Subject: [PATCH 4/4] feedback Co-authored-by: Copilot --- Cargo.lock | 1 + crates/icp-cli/Cargo.toml | 1 + crates/icp-cli/src/operations/bundle.rs | 158 +++++++++++++++- crates/icp-cli/tests/bundle_tests.rs | 235 ++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a76ec39fb..df1da0b76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,6 +3490,7 @@ dependencies = [ "bigdecimal", "bip32", "byte-unit", + "camino", "camino-tempfile", "candid", "candid_parser", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 9d3ac7b65..bddc01ea8 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -21,6 +21,7 @@ base64.workspace = true bigdecimal.workspace = true bip32.workspace = true byte-unit.workspace = true +camino.workspace = true camino-tempfile.workspace = true candid_parser = { workspace = true, features = ["assist"] } candid.workspace = true diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 95726e8c0..ea7645acf 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, fs::File, io::{BufWriter, Cursor}, sync::Arc, @@ -6,14 +7,16 @@ use std::{ use sha2::{Digest, Sha256}; +use camino::Utf8Component; use flate2::{Compression, write::GzEncoder}; use icp::{ Canister, InitArgs, canister::build::Build, + fs, manifest::{ - ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, Item, - LoadManifestFromPathError, ManifestInitArgs, PROJECT_MANIFEST, ProjectManifest, SyncStep, - SyncSteps, assets::DirField, prebuilt, + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, EnvironmentManifest, Instructions, + Item, LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, + ProjectManifest, SyncStep, SyncSteps, assets::DirField, load_manifest_from_path, prebuilt, }, package::PackageCache, prelude::*, @@ -43,6 +46,21 @@ pub enum BundleError { #[snafu(display("failed to load project manifest for bundle"))] LoadManifest { source: LoadManifestFromPathError }, + #[snafu(display("failed to load network manifest from '{path}'"))] + LoadNetwork { + path: PathBuf, + source: LoadManifestFromPathError, + }, + + #[snafu(display("failed to load environment manifest from '{path}'"))] + LoadEnvironment { + path: PathBuf, + source: LoadManifestFromPathError, + }, + + #[snafu(display("failed to read init_args file '{path}'"))] + ReadInitArgs { path: PathBuf, source: fs::IoError }, + #[snafu(display("failed to serialize bundle manifest"))] SerializeManifest { source: serde_yaml::Error }, @@ -93,7 +111,7 @@ pub(crate) async fn create_bundle( // Re-read the raw manifest to preserve networks and environments verbatim. let raw_manifest: ProjectManifest = - icp::manifest::load_manifest_from_path(&project_dir.join(PROJECT_MANIFEST)) + load_manifest_from_path(&project_dir.join(PROJECT_MANIFEST)) .await .context(LoadManifestSnafu)?; @@ -111,7 +129,8 @@ pub(crate) async fn create_bundle( })?; let sha256 = hex::encode(Sha256::digest(&wasm)); - let wasm_filename = format!("{}.wasm", canister.name); + let path_name = path_segment(&canister.name); + let wasm_filename = format!("{path_name}.wasm"); // Collect asset dirs and rewrite their paths for the bundle. let mut bundle_sync_steps = Vec::new(); @@ -126,7 +145,7 @@ pub(crate) async fn create_bundle( let prefixed_dirs: Vec = dirs .iter() - .map(|d| format!("{}/{d}", canister.name)) + .map(|d| format!("{path_name}/{}", normalize_archive_dir(d))) .collect(); let new_dir = if prefixed_dirs.len() == 1 { @@ -143,7 +162,7 @@ pub(crate) async fn create_bundle( } if !raw_asset_dirs.is_empty() { - asset_dirs.push((canister_path.clone(), canister.name.clone(), raw_asset_dirs)); + asset_dirs.push((canister_path.clone(), path_name.clone(), raw_asset_dirs)); } let init_args = canister.init_args.as_ref().map(convert_init_args); @@ -177,10 +196,79 @@ pub(crate) async fn create_bundle( canister_wasms.push((wasm_filename, wasm)); } + let mut networks = Vec::new(); + for item in raw_manifest.networks { + let inlined = match item { + Item::Manifest(_) => item, + Item::Path(ref path) => { + let full = project_dir.join(path); + let m = load_manifest_from_path::(&full) + .await + .context(LoadNetworkSnafu { path: full })?; + Item::Manifest(m) + } + }; + networks.push(inlined); + } + + // canister name → its directory, for resolving init_args file references. + // Inline canisters use the project dir (matching project.rs Item::Manifest handling). + let canister_path_map: HashMap<&str, &Path> = canisters + .iter() + .map(|(path, canister)| (canister.name.as_str(), path.as_path())) + .collect(); + + // init_args files to copy into the archive: (source_path, archive_path). + let mut init_args_files: Vec<(PathBuf, String)> = Vec::new(); + + let mut environments = Vec::new(); + for item in raw_manifest.environments { + let mut inlined = match item { + Item::Manifest(_) => item, + Item::Path(ref path) => { + let full = project_dir.join(path); + let m = load_manifest_from_path::(&full) + .await + .context(LoadEnvironmentSnafu { path: full })?; + Item::Manifest(m) + } + }; + + if let Item::Manifest(ref mut env) = inlined + && let Some(ref mut overrides) = env.init_args + { + for (canister_name, mia) in overrides.iter_mut() { + if let ManifestInitArgs::Path { + path: orig_path, + format: fmt, + } = &*mia + { + let base = canister_path_map + .get(canister_name.as_str()) + .copied() + .unwrap_or(project_dir); + let src = base.join(orig_path); + let archive_path = format!( + "init-args/{}/{}", + path_segment(canister_name), + normalize_archive_dir(orig_path) + ); + init_args_files.push((src, archive_path.clone())); + *mia = ManifestInitArgs::Path { + path: archive_path, + format: fmt.clone(), + }; + } + } + } + + environments.push(inlined); + } + let bundle_manifest = ProjectManifest { canisters: bundle_canisters, - networks: raw_manifest.networks, - environments: raw_manifest.environments, + networks, + environments, }; let manifest_yaml = serde_yaml::to_string(&bundle_manifest).context(SerializeManifestSnafu)?; @@ -216,11 +304,27 @@ pub(crate) async fn create_bundle( })?; } + // Init args files + for (src_path, archive_path) in &init_args_files { + let data = fs::read(src_path).context(ReadInitArgsSnafu { + path: src_path.clone(), + })?; + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, archive_path, Cursor::new(data)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(archive_path), + })?; + } + // Asset directories for (canister_path, canister_name, dirs) in &asset_dirs { for dir in dirs { let src_path = canister_path.join(dir); - let archive_prefix = format!("{canister_name}/{dir}"); + let archive_prefix = format!("{canister_name}/{}", normalize_archive_dir(dir)); archive .append_dir_all(&archive_prefix, src_path.as_std_path()) .context(WriteArchiveEntrySnafu { @@ -235,6 +339,40 @@ pub(crate) async fn create_bundle( Ok(()) } +/// Normalizes a relative directory path for use as a tar archive prefix. +/// +/// Resolves `.` and `..` lexically, strips leading `..` that would escape the +/// canister root, and discards any absolute prefix. The result is a clean +/// forward-slash-separated relative path safe to embed in a tar entry name. +fn normalize_archive_dir(dir: &str) -> String { + let mut parts: Vec = Vec::new(); + for component in PathBuf::from(dir).components() { + match component { + Utf8Component::Normal(s) => parts.push(s.to_owned()), + Utf8Component::CurDir => {} + Utf8Component::ParentDir => { + parts.pop(); + } + Utf8Component::RootDir | Utf8Component::Prefix(_) => parts.clear(), + } + } + parts.join("/") +} + +/// Converts a canister name into a cross-platform-safe path segment. +/// +/// Replaces any character that is not alphanumeric, `-`, `_`, or `.` with `_`. +/// This covers all characters prohibited on Windows (`< > : " / \ | ? *`), +/// path separators on Unix, and control characters. +fn path_segment(name: &str) -> String { + name.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' => c, + _ => '_', + }) + .collect() +} + fn convert_init_args(args: &InitArgs) -> ManifestInitArgs { match args { InitArgs::Text { content, format } => ManifestInitArgs::Value { diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 80f879cd2..4fa7aa02b 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -207,6 +207,241 @@ async fn bundle_and_deploy() { ); } +/// Bundle a canister whose environment override uses an external init_args file. +/// The file must be copied into the archive at a normalized path and the manifest +/// reference rewritten to match. +#[test] +fn bundle_inlines_external_init_args_file() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + write_string(&project_dir.join("args.idl"), "(\"world\")").expect("failed to write args file"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + + networks: + - name: random-network + mode: managed + gateway: + port: 0 + + environments: + - name: random-environment + network: random-network + init_args: + my-canister: + path: ./args.idl + format: candid + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_args_file = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "init-args/my-canister/args.idl" => { + found_args_file = true; + } + _ => {} + } + } + + assert!( + found_args_file, + "init-args/my-canister/args.idl not found in bundle" + ); + assert!( + manifest_yaml.contains("init-args/my-canister/args.idl"), + "bundle manifest should reference the relocated init_args file" + ); + assert!( + !manifest_yaml.contains("./args.idl"), + "bundle manifest should not contain the original relative path" + ); +} + +/// Canister names with characters invalid in file paths (spaces, `!`, `/`, etc.) +/// must be sanitized for archive entry names while the manifest preserves the +/// original name. +#[test] +fn bundle_sanitizes_canister_name_for_paths() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let pm = formatdoc! {r#" + canisters: + - name: "my canister!" + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_wasm = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "my_canister_.wasm" => { + found_wasm = true; + } + _ => {} + } + } + + assert!(found_wasm, "my_canister_.wasm not found in bundle"); + assert!( + manifest_yaml.contains("my_canister_.wasm"), + "bundle manifest should reference sanitized wasm filename" + ); + assert!( + manifest_yaml.contains("my canister!"), + "bundle manifest should preserve original canister name" + ); +} + +/// An asset sync step whose `dir` starts with `..` must be normalized when +/// written into the archive — no literal `..` components in entry names. +#[test] +fn bundle_normalizes_dotdot_asset_path() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Assets live outside the project dir, one level up in a sibling directory. + let assets_dir = project_dir + .parent() + .expect("project dir has no parent") + .join("shared-assets"); + create_dir_all(&assets_dir).expect("failed to create sibling assets dir"); + write_string(&assets_dir.join("index.html"), "hello").expect("failed to write asset"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: assets + dir: ../shared-assets + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_asset = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + if path.contains("..") { + panic!("archive entry '{path}' contains '..' component"); + } + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + p if p.starts_with("my-canister/shared-assets/") => { + found_asset = true; + } + _ => {} + } + } + + assert!( + found_asset, + "asset not found under my-canister/shared-assets/ in bundle" + ); + assert!( + manifest_yaml.contains("my-canister/shared-assets"), + "bundle manifest sync dir should use normalized path" + ); + assert!( + !manifest_yaml.contains(".."), + "bundle manifest should not contain '..' in any path" + ); +} + /// Projects with script sync steps must be rejected with a clear error. #[test] fn bundle_rejects_script_sync_step() {