diff --git a/Cargo.lock b/Cargo.lock index 04d300437..a76ec39fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3541,6 +3541,7 @@ dependencies = [ "shellwords", "snafu", "sysinfo", + "tar", "test-tag", "time", "tiny-bip39", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index f940f7034..9d3ac7b65 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -32,6 +32,7 @@ dunce.workspace = true elliptic-curve.workspace = true flate2.workspace = true futures.workspace = true +tar.workspace = true hex.workspace = true httptest.workspace = true ic-agent.workspace = true diff --git a/crates/icp-cli/src/commands/project/bundle.rs b/crates/icp-cli/src/commands/project/bundle.rs new file mode 100644 index 000000000..0b758cbca --- /dev/null +++ b/crates/icp-cli/src/commands/project/bundle.rs @@ -0,0 +1,41 @@ +use anyhow::Context as _; +use clap::Args; +use icp::context::Context; +use icp::prelude::*; + +use crate::operations::bundle::create_bundle; + +/// Bundle a project into a self-contained deployable archive. +/// +/// Builds all project canisters and packages them with a rewritten manifest +/// into a `.tar.gz` file. The rewritten manifest replaces all build steps +/// with pre-built steps referencing the bundled WASM files. Asset sync +/// directories are included in the archive. +/// +/// Projects with script sync steps cannot be bundled. +#[derive(Args, Debug)] +pub(crate) struct BundleArgs { + /// Output path for the bundle archive (e.g. bundle.tar.gz) + #[arg(long, short)] + pub(crate) output: PathBuf, +} + +pub(crate) async fn exec(ctx: &Context, args: &BundleArgs) -> Result<(), anyhow::Error> { + let project = ctx.project.load().await.context("failed to load project")?; + + let canisters: Vec<_> = project.canisters.into_values().collect(); + + create_bundle( + &project.dir, + canisters, + ctx.builder.clone(), + ctx.artifacts.clone(), + &ctx.dirs.package_cache()?, + ctx.debug, + &args.output, + ) + .await + .context("failed to create bundle")?; + + Ok(()) +} diff --git a/crates/icp-cli/src/commands/project/mod.rs b/crates/icp-cli/src/commands/project/mod.rs index 48ae1348a..99a4e84e6 100644 --- a/crates/icp-cli/src/commands/project/mod.rs +++ b/crates/icp-cli/src/commands/project/mod.rs @@ -1,9 +1,11 @@ use clap::Subcommand; +pub(crate) mod bundle; pub(crate) mod show; -/// Display information about the current project +/// Manage the current project #[derive(Debug, Subcommand)] pub(crate) enum Command { Show(show::ShowArgs), + Bundle(bundle::BundleArgs), } diff --git a/crates/icp-cli/src/main.rs b/crates/icp-cli/src/main.rs index 9d580708a..b883c6a91 100644 --- a/crates/icp-cli/src/main.rs +++ b/crates/icp-cli/src/main.rs @@ -406,6 +406,9 @@ async fn dispatch(ctx: &icp::context::Context, command: Command) -> Result<(), E commands::project::Command::Show(args) => { commands::project::show::exec(ctx, &args).await? } + commands::project::Command::Bundle(args) => { + commands::project::bundle::exec(ctx, &args).await? + } }, // Settings diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs new file mode 100644 index 000000000..95726e8c0 --- /dev/null +++ b/crates/icp-cli/src/operations/bundle.rs @@ -0,0 +1,249 @@ +use std::{ + fs::File, + io::{BufWriter, Cursor}, + sync::Arc, +}; + +use sha2::{Digest, Sha256}; + +use flate2::{Compression, write::GzEncoder}; +use icp::{ + Canister, InitArgs, + canister::build::Build, + manifest::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, Item, + LoadManifestFromPathError, ManifestInitArgs, PROJECT_MANIFEST, ProjectManifest, SyncStep, + SyncSteps, assets::DirField, prebuilt, + }, + package::PackageCache, + prelude::*, + store_artifact, +}; +use snafu::{ResultExt, Snafu}; +use tar::Builder; + +use crate::operations::build::{BuildManyError, build_many_with_progress_bar}; + +#[derive(Debug, Snafu)] +pub enum BundleError { + #[snafu(display( + "canister '{canister}' has a script sync step, which is not supported in bundles" + ))] + ScriptSyncStep { canister: String }, + + #[snafu(transparent)] + Build { source: BuildManyError }, + + #[snafu(display("failed to look up built artifact for canister '{canister}'"))] + LookupArtifact { + canister: String, + source: store_artifact::LookupArtifactError, + }, + + #[snafu(display("failed to load project manifest for bundle"))] + LoadManifest { source: LoadManifestFromPathError }, + + #[snafu(display("failed to serialize bundle manifest"))] + SerializeManifest { source: serde_yaml::Error }, + + #[snafu(display("failed to add '{path}' to bundle archive"))] + WriteArchiveEntry { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to create bundle output file at '{path}'"))] + CreateOutput { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to finalize bundle archive"))] + FlushArchive { source: std::io::Error }, +} + +pub(crate) async fn create_bundle( + project_dir: &Path, + canisters: Vec<(PathBuf, Canister)>, + builder: Arc, + artifacts: Arc, + pkg_cache: &PackageCache, + debug: bool, + output: &Path, +) -> Result<(), BundleError> { + for (_, canister) in &canisters { + for step in &canister.sync.steps { + if matches!(step, SyncStep::Script(_)) { + return ScriptSyncStepSnafu { + canister: canister.name.clone(), + } + .fail(); + } + } + } + + build_many_with_progress_bar( + canisters.clone(), + builder, + artifacts.clone(), + pkg_cache, + debug, + ) + .await?; + + // Re-read the raw manifest to preserve networks and environments verbatim. + let raw_manifest: ProjectManifest = + icp::manifest::load_manifest_from_path(&project_dir.join(PROJECT_MANIFEST)) + .await + .context(LoadManifestSnafu)?; + + let mut bundle_canisters = Vec::new(); + let mut canister_wasms: Vec<(String, Vec)> = Vec::new(); + // (canister_path, canister_name, asset_dirs) + let mut asset_dirs: Vec<(PathBuf, String, Vec)> = Vec::new(); + + for (canister_path, canister) in &canisters { + let wasm = artifacts + .lookup(&canister.name) + .await + .context(LookupArtifactSnafu { + canister: canister.name.clone(), + })?; + + let sha256 = hex::encode(Sha256::digest(&wasm)); + let wasm_filename = format!("{}.wasm", canister.name); + + // Collect asset dirs and rewrite their paths for the bundle. + let mut bundle_sync_steps = Vec::new(); + let mut raw_asset_dirs = Vec::new(); + + for step in &canister.sync.steps { + match step { + SyncStep::Script(_) => unreachable!("validated above"), + SyncStep::Assets(adapter) => { + let dirs = adapter.dir.as_vec(); + raw_asset_dirs.extend(dirs.clone()); + + let prefixed_dirs: Vec = dirs + .iter() + .map(|d| format!("{}/{d}", canister.name)) + .collect(); + + let new_dir = if prefixed_dirs.len() == 1 { + DirField::Dir(prefixed_dirs.into_iter().next().unwrap()) + } else { + DirField::Dirs(prefixed_dirs) + }; + + bundle_sync_steps.push(SyncStep::Assets(icp::manifest::assets::Adapter { + dir: new_dir, + })); + } + } + } + + if !raw_asset_dirs.is_empty() { + asset_dirs.push((canister_path.clone(), canister.name.clone(), raw_asset_dirs)); + } + + let init_args = canister.init_args.as_ref().map(convert_init_args); + + let sync = if bundle_sync_steps.is_empty() { + None + } else { + Some(SyncSteps { + steps: bundle_sync_steps, + }) + }; + + let canister_manifest = CanisterManifest { + name: canister.name.clone(), + settings: canister.settings.clone(), + init_args, + instructions: Instructions::BuildSync { + build: BuildSteps { + steps: vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(prebuilt::LocalSource { + path: wasm_filename.as_str().into(), + }), + sha256: Some(sha256), + })], + }, + sync, + }, + }; + + bundle_canisters.push(Item::Manifest(canister_manifest)); + canister_wasms.push((wasm_filename, wasm)); + } + + let bundle_manifest = ProjectManifest { + canisters: bundle_canisters, + networks: raw_manifest.networks, + environments: raw_manifest.environments, + }; + + let manifest_yaml = serde_yaml::to_string(&bundle_manifest).context(SerializeManifestSnafu)?; + + let file = File::create(output.as_std_path()).context(CreateOutputSnafu { + path: output.to_path_buf(), + })?; + let gz = GzEncoder::new(BufWriter::new(file), Compression::default()); + let mut archive = Builder::new(gz); + + // icp.yaml + let manifest_bytes = manifest_yaml.as_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "icp.yaml", Cursor::new(manifest_bytes)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from("icp.yaml"), + })?; + + // WASM files + for (filename, wasm) in &canister_wasms { + let mut header = tar::Header::new_gnu(); + header.set_size(wasm.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, filename, Cursor::new(wasm)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(filename), + })?; + } + + // Asset directories + for (canister_path, canister_name, dirs) in &asset_dirs { + for dir in dirs { + let src_path = canister_path.join(dir); + let archive_prefix = format!("{canister_name}/{dir}"); + archive + .append_dir_all(&archive_prefix, src_path.as_std_path()) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(&archive_prefix), + })?; + } + } + + let gz = archive.into_inner().context(FlushArchiveSnafu)?; + gz.finish().context(FlushArchiveSnafu)?; + + Ok(()) +} + +fn convert_init_args(args: &InitArgs) -> ManifestInitArgs { + match args { + InitArgs::Text { content, format } => ManifestInitArgs::Value { + value: content.clone(), + format: format.clone(), + }, + InitArgs::Binary(bytes) => ManifestInitArgs::Value { + value: hex::encode(bytes), + format: ArgsFormat::Hex, + }, + } +} diff --git a/crates/icp-cli/src/operations/mod.rs b/crates/icp-cli/src/operations/mod.rs index f1d8ba2b7..a53b619a0 100644 --- a/crates/icp-cli/src/operations/mod.rs +++ b/crates/icp-cli/src/operations/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod binding_env_vars; pub(crate) mod build; +pub(crate) mod bundle; pub(crate) mod candid_compat; pub(crate) mod canister_migration; pub(crate) mod create; diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs new file mode 100644 index 000000000..80f879cd2 --- /dev/null +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -0,0 +1,238 @@ +use std::{ + fs, + io::{BufReader, Read as _}, +}; + +use flate2::bufread::GzDecoder; +use icp::{ + fs::{create_dir_all, json, write_string}, + prelude::*, + store_id::IdMapping, +}; +use indoc::formatdoc; +use predicates::{ + ord::eq, + prelude::PredicateBooleanExt, + str::{PredicateStrExt, contains}, +}; +use tar::Archive; + +use crate::common::{ENVIRONMENT_RANDOM_PORT, NETWORK_RANDOM_PORT, TestContext, clients}; + +mod common; + +/// Bundle a standard frontend-backend project: a script-built backend canister and an +/// asset-canister-recipe frontend canister with an assets sync step. +/// Verify archive structure, manifest content, and that the bundle deploys successfully. +#[tokio::test] +async fn bundle_and_deploy() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + // A small WASM to serve as the pre-built artifact for the backend. + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Create asset directory for the frontend canister. + let asset_dir = project_dir.join("www"); + create_dir_all(&asset_dir).expect("failed to create asset dir"); + write_string(&asset_dir.join("index.html"), "hello").expect("failed to write asset file"); + + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.1.0" + configuration: + dir: www + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + assert!(bundle_path.exists(), "bundle file was not created"); + + // Extract and inspect archive contents. + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_manifest = false; + let mut found_backend_wasm = false; + let mut found_frontend_wasm = false; + let mut found_asset = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + found_manifest = true; + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "backend.wasm" => { + found_backend_wasm = true; + } + "frontend.wasm" => { + found_frontend_wasm = true; + } + p if p.starts_with("frontend/www/") => { + found_asset = true; + } + _ => {} + } + } + + assert!(found_manifest, "icp.yaml not found in bundle"); + assert!(found_backend_wasm, "backend.wasm not found in bundle"); + assert!(found_frontend_wasm, "frontend.wasm not found in bundle"); + assert!( + found_asset, + "asset file not found under frontend/www/ in bundle" + ); + + // Manifest must contain pre-built steps and no script or recipe steps. + assert!( + manifest_yaml.contains("pre-built"), + "bundle manifest should have pre-built build steps" + ); + assert!( + !manifest_yaml.contains("type: script"), + "bundle manifest should not contain script steps" + ); + assert!( + !manifest_yaml.contains("recipe:"), + "bundle manifest should not contain recipe sections" + ); + assert!( + manifest_yaml.contains("sha256:"), + "bundle manifest should include sha256 for pre-built wasms" + ); + + // Extract bundle to a fresh directory and deploy from it. + let bundle_dir = project_dir.join("bundle-extracted"); + create_dir_all(&bundle_dir).expect("failed to create bundle-extracted dir"); + + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + archive + .unpack(bundle_dir.as_std_path()) + .expect("failed to extract bundle"); + + let _g = ctx.start_network_in(&bundle_dir, "random-network").await; + ctx.ping_until_healthy(&bundle_dir, "random-network"); + + let network_port = ctx + .wait_for_network_descriptor(&bundle_dir, "random-network") + .gateway_port; + + clients::icp(&ctx, &bundle_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "deploy", + "--subnet", + common::SUBNET_ID, + "--environment", + "random-environment", + ]) + .assert() + .success(); + + // Verify the backend canister responds to a query call. + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "canister", + "call", + "--environment", + "random-environment", + "backend", + "greet", + "(\"world\")", + ]) + .assert() + .success() + .stdout(eq("(\"Hello, world!\")").trim()); + + // Verify the frontend canister serves the bundled asset. + let id_mapping: IdMapping = json::load( + &bundle_dir + .join(".icp") + .join("cache") + .join("mappings") + .join("random-environment.ids.json"), + ) + .expect("failed to read ID mapping"); + + let frontend_cid = id_mapping + .get("frontend") + .expect("frontend canister ID not found"); + + let resp = reqwest::get(format!( + "http://localhost:{network_port}/?canisterId={frontend_cid}" + )) + .await + .expect("request to frontend canister failed"); + + assert_eq!( + resp.text().await.expect("failed to read response body"), + "hello" + ); +} + +/// Projects with script sync steps must be rejected with a clear error. +#[test] +fn bundle_rejects_script_sync_step() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + let pm = r#" +canisters: + - name: my-canister + build: + steps: + - type: script + command: echo build + sync: + steps: + - type: script + command: echo sync +"#; + + write_string(&project_dir.join("icp.yaml"), pm).expect("failed to write project manifest"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", "bundle.tar.gz"]) + .assert() + .failure() + .stderr(contains("my-canister").and(contains("script sync step"))); +} diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index 791ca7888..e1f895854 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -144,40 +144,48 @@ impl From for LogVisibility { #[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Settings { /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] pub log_visibility: Option, /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] pub compute_allocation: Option, /// Memory allocation in bytes. If unset, memory is allocated dynamically. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub memory_allocation: Option, /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] pub freezing_threshold: Option, /// Upper limit on cycles reserved for future resource payments. /// Memory allocations that would push the reserved balance above this limit will fail. /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub reserved_cycles_limit: Option, /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_limit: Option, /// Wasm memory threshold in bytes. Triggers a callback when exceeded. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_threshold: Option, /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] pub log_memory_limit: Option, /// Environment variables for the canister as key-value pairs. /// These variables are accessible within the canister and can be used to configure /// behavior without hardcoding values in the WASM module. + #[serde(skip_serializing_if = "Option::is_none")] pub environment_variables: Option>, } diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 6fafaf721..98d1aa0a4 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -61,17 +61,17 @@ pub enum ManifestInitArgs { /// Represents the manifest describing a single canister. /// This struct is typically loaded from a `canister.yaml` file and defines /// the canister's name and how it should be built into WebAssembly. -#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] pub struct CanisterManifest { /// The unique name of the canister as defined in this manifest. pub name: String, /// The configuration specifying the various settings when creating the canister. #[serde(default)] - #[schemars(with = "Option")] pub settings: Settings, /// Initialization arguments passed to the canister during installation. + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option, #[serde(flatten)] @@ -236,7 +236,7 @@ impl<'de> Deserialize<'de> for CanisterManifest { } } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(untagged)] pub enum Instructions { Recipe { @@ -249,6 +249,7 @@ pub enum Instructions { build: BuildSteps, /// The configuration specifying how to sync the canister + #[serde(skip_serializing_if = "Option::is_none")] sync: Option, }, } diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp/src/manifest/environment.rs index abe73cc4d..3b9d67d29 100644 --- a/crates/icp/src/manifest/environment.rs +++ b/crates/icp/src/manifest/environment.rs @@ -1,18 +1,22 @@ use std::collections::HashMap; use schemars::JsonSchema; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize}; use crate::{canister::Settings, prelude::LOCAL}; use super::canister::ManifestInitArgs; -#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct EnvironmentInner { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub canisters: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub settings: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option>, } @@ -100,6 +104,36 @@ impl<'de> Deserialize<'de> for EnvironmentManifest { } } +impl From<&EnvironmentManifest> for EnvironmentInner { + fn from(env: &EnvironmentManifest) -> Self { + let network = if env.network == LOCAL { + None + } else { + Some(env.network.clone()) + }; + + let canisters = match &env.canisters { + CanisterSelection::Everything => None, + CanisterSelection::Named(names) => Some(names.clone()), + CanisterSelection::None => Some(vec![]), + }; + + EnvironmentInner { + name: env.name.clone(), + network, + canisters, + settings: env.settings.clone(), + init_args: env.init_args.clone(), + } + } +} + +impl Serialize for EnvironmentManifest { + fn serialize(&self, serializer: S) -> Result { + EnvironmentInner::from(self).serialize(serializer) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index fe11aac7e..d743a61e3 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use snafu::prelude::*; use crate::fs; @@ -16,7 +16,12 @@ pub(crate) mod recipe; pub(crate) mod serde_helpers; pub use { - canister::{ArgsFormat, CanisterManifest, ManifestInitArgs}, + adapter::assets, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, environment::EnvironmentManifest, network::NetworkManifest, project::ProjectManifest, @@ -41,6 +46,15 @@ pub enum Item { Manifest(T), } +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + impl<'de, T> Deserialize<'de> for Item where T: Deserialize<'de>, diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index 16ddc05d6..d892f9aa2 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -1,11 +1,11 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use url::Url; use crate::network::SubnetKind; /// A network definition for the project -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] pub struct NetworkManifest { pub name: String, @@ -13,20 +13,20 @@ pub struct NetworkManifest { pub configuration: Mode, } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(rename_all = "lowercase", tag = "mode")] pub enum Mode { Managed(Managed), Connected(Connected), } -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Managed { #[serde(flatten)] pub mode: Box, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] #[serde(untagged, rename_all_fields = "kebab-case")] #[allow(clippy::large_enum_variant)] pub enum ManagedMode { @@ -36,46 +36,68 @@ pub enum ManagedMode { /// Port mappings in the format "host_port:container_port" port_mapping: Vec, /// Whether to delete the container when the network stops + #[serde(skip_serializing_if = "Option::is_none")] rm_on_exit: Option, /// Command line arguments to pass to the container's entrypoint - #[serde(alias = "cmd", alias = "command")] + #[serde( + alias = "cmd", + alias = "command", + skip_serializing_if = "Option::is_none" + )] args: Option>, /// Entrypoint to use for the container + #[serde(skip_serializing_if = "Option::is_none")] entrypoint: Option>, /// Environment variables to set in the container in VAR=VALUE format (or VAR to inherit from host) + #[serde(skip_serializing_if = "Option::is_none")] environment: Option>, /// Volumes to mount into the container in the format name:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] volumes: Option>, /// The platform to use for the container (e.g. linux/amd64) + #[serde(skip_serializing_if = "Option::is_none")] platform: Option, /// The user to run the container as in the format user[:group] + #[serde(skip_serializing_if = "Option::is_none")] user: Option, /// The size of /dev/shm in bytes + #[serde(skip_serializing_if = "Option::is_none")] shm_size: Option, /// The status directory inside the container. Defaults to /app/status + #[serde(skip_serializing_if = "Option::is_none")] status_dir: Option, /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] mounts: Option>, /// Extra hosts entries for Docker networking (e.g. "host.docker.internal:host-gateway") + #[serde(skip_serializing_if = "Option::is_none")] extra_hosts: Option>, }, Launcher { /// HTTP gateway configuration + #[serde(skip_serializing_if = "Option::is_none")] gateway: Option, /// Artificial delay to add to every update call + #[serde(skip_serializing_if = "Option::is_none")] artificial_delay_ms: Option, /// Set up the Internet Identity canister. Makes internet identity available at /// id.ai.localhost: + #[serde(skip_serializing_if = "Option::is_none")] ii: Option, /// Set up the NNS + #[serde(skip_serializing_if = "Option::is_none")] nns: Option, /// Configure the list of subnets (one application subnet by default) + #[serde(skip_serializing_if = "Option::is_none")] subnets: Option>, /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") + #[serde(skip_serializing_if = "Option::is_none")] bitcoind_addr: Option>, /// Dogecoin P2P node addresses to connect to + #[serde(skip_serializing_if = "Option::is_none")] dogecoind_addr: Option>, /// The version of icp-cli-network-launcher to use. Defaults to the latest released version. Launcher versions correspond to published PocketIC or IC-OS releases. + #[serde(skip_serializing_if = "Option::is_none")] version: Option, }, } @@ -95,18 +117,19 @@ impl Default for ManagedMode { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct Connected { #[serde(flatten)] pub endpoints: Endpoints, /// The root key of this network + #[serde(skip_serializing_if = "Option::is_none")] #[schemars(with = "Option", regex(pattern = "^[0-9a-f]{266}$"))] pub root_key: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(untagged, rename_all_fields = "kebab-case")] pub enum Endpoints { Explicit { @@ -114,6 +137,7 @@ pub enum Endpoints { /// otherwise icp-cli will fall back to ?canisterId= query parameters which are frequently brittle in frontend code. /// /// If no HTTP gateway endpoint is provided, canister URLs will not be printed in deploy operations. + #[serde(skip_serializing_if = "Option::is_none")] http_gateway_url: Option, /// The URL of the API endpoint. Should support the standard API routes (e.g. /api/v3). api_url: Url, @@ -126,7 +150,7 @@ pub enum Endpoints { }, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(try_from = "String", into = "String")] pub struct RootKey(pub Vec); @@ -145,13 +169,16 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Gateway { /// Network interface for the gateway. Defaults to 127.0.0.1 + #[serde(skip_serializing_if = "Option::is_none")] pub bind: Option, /// Domains the gateway should respond to. Automatically includes localhost if applicable. + #[serde(skip_serializing_if = "Option::is_none")] pub domains: Option>, /// Port for the gateway to listen on. Defaults to 8000 + #[serde(skip_serializing_if = "Option::is_none")] pub port: Option, } diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index b158cbbd0..310b57f9f 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -1,23 +1,20 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::manifest::{ Item, canister::CanisterManifest, environment::EnvironmentManifest, network::NetworkManifest, }; -#[derive(Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct ProjectManifest { #[serde(default)] - #[schemars(with = "Option>>")] pub canisters: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub networks: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub environments: Vec>, } diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp/src/manifest/recipe.rs index 107ba0fbe..b98105016 100644 --- a/crates/icp/src/manifest/recipe.rs +++ b/crates/icp/src/manifest/recipe.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize, de::Error as _}; /// Represents the accepted values for a recipe type in /// the canister manifest -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema)] #[schemars(from = "String")] pub enum RecipeType { /// path to a locally defined recipe @@ -91,7 +91,13 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +impl Serialize for RecipeType { + fn serialize(&self, serializer: S) -> Result { + String::from(self.clone()).serialize(serializer) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Recipe { /// An identifier for a recipe, it can have one of the following formats: /// @@ -109,7 +115,7 @@ pub struct Recipe { pub recipe_type: RecipeType, #[serde(default)] - #[schemars(with = "Option>")] + #[schemars(with = "HashMap")] pub configuration: HashMap, /// Optional sha256 checksum for the recipe template. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 9ab50d494..b15066166 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -66,6 +66,7 @@ This document contains the help content for the `icp` command-line program. * [`icp new`↴](#icp-new) * [`icp project`↴](#icp-project) * [`icp project show`↴](#icp-project-show) +* [`icp project bundle`↴](#icp-project-bundle) * [`icp settings`↴](#icp-settings) * [`icp settings autocontainerize`↴](#icp-settings-autocontainerize) * [`icp settings telemetry`↴](#icp-settings-telemetry) @@ -89,7 +90,7 @@ This document contains the help content for the `icp` command-line program. * `identity` — Manage your identities * `network` — Launch and manage local test networks * `new` — Create a new ICP project from a template -* `project` — Display information about the current project +* `project` — Manage the current project * `settings` — Configure user settings * `sync` — Synchronize canisters * `token` — Perform token transactions @@ -1448,13 +1449,14 @@ Under the hood templates are generated with `cargo-generate`. See the cargo-gene ## `icp project` -Display information about the current project +Manage the current project **Usage:** `icp project ` ###### **Subcommands:** * `show` — Outputs the project's effective yaml configuration +* `bundle` — Bundle a project into a self-contained deployable archive @@ -1474,6 +1476,22 @@ The effective yaml configuration includes: +## `icp project bundle` + +Bundle a project into a self-contained deployable archive. + +Builds all project canisters and packages them with a rewritten manifest into a `.tar.gz` file. The rewritten manifest replaces all build steps with pre-built steps referencing the bundled WASM files. Asset sync directories are included in the archive. + +Projects with script sync steps cannot be bundled. + +**Usage:** `icp project bundle --output ` + +###### **Options:** + +* `-o`, `--output ` — Output path for the bundle archive (e.g. bundle.tar.gz) + + + ## `icp settings` Configure user settings diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index ba1ac9637..441a67c4d 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -285,10 +285,7 @@ "configuration": { "additionalProperties": true, "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", @@ -394,7 +391,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -527,25 +523,8 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "$ref": "#/$defs/Settings", + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, diff --git a/docs/schemas/environment-yaml-schema.json b/docs/schemas/environment-yaml-schema.json index 825958121..0d111b82f 100644 --- a/docs/schemas/environment-yaml-schema.json +++ b/docs/schemas/environment-yaml-schema.json @@ -206,7 +206,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index efd5d1bc7..be895ec6e 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -212,25 +212,8 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "$ref": "#/$defs/Settings", + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, @@ -769,10 +752,7 @@ "configuration": { "additionalProperties": true, "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", @@ -878,7 +858,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -973,31 +952,25 @@ "description": "Schema for ProjectManifest", "properties": { "canisters": { + "default": [], "items": { "$ref": "#/$defs/Item" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "environments": { + "default": [], "items": { "$ref": "#/$defs/Item3" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "networks": { + "default": [], "items": { "$ref": "#/$defs/Item2" }, - "type": [ - "array", - "null" - ] + "type": "array" } }, "title": "ProjectManifest",