Feature gate for embedding data

.github/workflows/rust.yml

@@ -4,9 +4,9 @@ on:
   push:
     branches:
       - main
-    tags:
-      - '*'
   pull_request:
+  release:
+    types: [published]
   workflow_dispatch:
 
 env:
@@ -42,11 +42,11 @@ jobs:
       run: cargo build --verbose
     - name: Run tests
       id: run-tests
-      run: cargo test --verbose
+      run: cargo test --verbose --no-default-features
       continue-on-error: true
-    - name: Dump test fixtures on failure
+    - name: Dump test fixtures on failure  --no-default-features
       if: steps.run-tests.outcome == 'failure'
-      run: TRYCMD=dump cargo test --verbose
+      run: TRYCMD=dump cargo test --verbose  --no-default-features
       continue-on-error: true
     - name: Upload updated fixtures
       if: steps.run-tests.outcome == 'failure'
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       id-token: write  # OIDC trusted-publishing token for crates.io
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:

core/Cargo.toml

@@ -8,6 +8,10 @@ description.workspace = true
 edition.workspace = true
 rust-version.workspace = true
 
+[features]
+default = ["embedded-data"]
+embedded-data = ["dep:flate2"]
+
 [build-dependencies]
 serde_yaml_ng = "0.10"
 serde = { version = "1.0.228", features = ["derive"] }
@@ -21,5 +25,6 @@ path = "src/main.rs"
 dyn-clone = "1.0.20"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_yaml_ng = "0.10"
-flate2 = "1.1"
+serde_json = "1"
+flate2 = { version = "1.1", optional = true }
 indicium = "0.6.5"

core/build.rs

@@ -124,8 +124,10 @@ fn write_knowledge() {
 }
 
 fn main() {
-    write_cf_standards_from_yaml();
-    write_knowledge();
+    if std::env::var("CARGO_FEATURE_EMBEDDED_DATA").is_ok() {
+        write_cf_standards_from_yaml();
+        write_knowledge();
+    }
 
     println!("cargo::rerun-if-changed=build.rs");
     println!("cargo::rerun-if-changed=standards/")

core/src/cf.rs

@@ -1,7 +1,5 @@
 use std::collections::HashMap;
-use std::io::Read;
 
-use flate2::read::GzDecoder;
 use serde::{Deserialize, Serialize};
 
 use crate::standard::Standard;
@@ -18,18 +16,6 @@ struct CfStandard {
     unit: String,
 }
 
-fn load_cf_yaml() -> CfYaml {
-    let compressed_data = include_bytes!(concat!(env!("OUT_DIR"), "/cf_standards.yaml.gz"));
-
-    // Decompress the data
-    let mut decoder = GzDecoder::new(&compressed_data[..]);
-    let mut yaml_data = String::new();
-    decoder.read_to_string(&mut yaml_data).unwrap();
-
-    // Deserialize from YAML
-    serde_yaml_ng::from_str(&yaml_data).unwrap()
-}
-
 /// Returns a HashMap of standard names: vector of aliases
 fn aliases_by_standard_name(cf_yaml: &CfYaml) -> HashMap<String, Vec<String>> {
     let aliases = &cf_yaml.aliases;
@@ -46,17 +32,12 @@ fn aliases_by_standard_name(cf_yaml: &CfYaml) -> HashMap<String, Vec<String>> {
     standards
 }
 
-/// Returns a HashMap of standard names to Standard
-pub fn cf_standards() -> HashMap<String, Standard> {
-    let cf_yaml = load_cf_yaml();
+fn cf_yaml_to_standards(cf_yaml: CfYaml) -> HashMap<String, Standard> {
     let alias_map = aliases_by_standard_name(&cf_yaml);
-
     let mut standards = HashMap::new();
-
     for (name, cf_standard) in &cf_yaml.standard_names {
         let empty_vec = Vec::new();
         let aliases = alias_map.get(name).unwrap_or(&empty_vec);
-
         standards.insert(
             name.to_string(),
             Standard {
@@ -68,14 +49,51 @@ pub fn cf_standards() -> HashMap<String, Standard> {
             },
         );
     }
-
     standards
 }
 
+#[cfg(feature = "embedded-data")]
+fn load_cf_yaml() -> CfYaml {
+    use flate2::read::GzDecoder;
+    use std::io::Read;
+
+    let compressed_data = include_bytes!(concat!(env!("OUT_DIR"), "/cf_standards.yaml.gz"));
+
+    // Decompress the data
+    let mut decoder = GzDecoder::new(&compressed_data[..]);
+    let mut yaml_data = String::new();
+    decoder.read_to_string(&mut yaml_data).unwrap();
+
+    // Deserialize from YAML
+    serde_yaml_ng::from_str(&yaml_data).unwrap()
+}
+
+/// Returns a HashMap of standard names to Standard from the embedded compressed data.
+#[cfg(feature = "embedded-data")]
+pub fn cf_standards() -> HashMap<String, Standard> {
+    let cf_yaml = load_cf_yaml();
+    cf_yaml_to_standards(cf_yaml)
+}
+
+/// Returns CF standards by deserializing a YAML string.
+pub fn cf_standards_from_yaml(yaml: &str) -> Result<HashMap<String, Standard>, String> {
+    serde_yaml_ng::from_str::<CfYaml>(yaml)
+        .map(cf_yaml_to_standards)
+        .map_err(|e| e.to_string())
+}
+
+/// Returns CF standards by deserializing a JSON string.
+pub fn cf_standards_from_json(json: &str) -> Result<HashMap<String, Standard>, String> {
+    serde_json::from_str::<CfYaml>(json)
+        .map(cf_yaml_to_standards)
+        .map_err(|e| e.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[cfg(feature = "embedded-data")]
     #[test]
     fn load_cf_standards() {
         let standards = cf_standards();
@@ -92,8 +110,12 @@ mod tests {
         )
     }
 
+    #[cfg(feature = "embedded-data")]
     #[test]
     fn test_compressed_loading() {
+        use flate2::read::GzDecoder;
+        use std::io::Read;
+
         // Load the compressed data directly to ensure compression is working
         let compressed_data = include_bytes!(concat!(env!("OUT_DIR"), "/cf_standards.yaml.gz"));
 
@@ -120,4 +142,34 @@ mod tests {
             "Compressed data should be less than 1MB"
         );
     }
+
+    #[test]
+    fn cf_standards_from_yaml_roundtrip() {
+        let yaml = r#"
+aliases:
+  air_pressure_at_sea_level: air_pressure_at_mean_sea_level
+standard_names:
+  air_pressure_at_mean_sea_level:
+    description: Air pressure at mean sea level
+    unit: Pa
+"#;
+        let standards = cf_standards_from_yaml(yaml).unwrap();
+        let p = &standards["air_pressure_at_mean_sea_level"];
+        assert_eq!(p.unit, "Pa");
+        assert!(p.aliases.contains(&"air_pressure_at_sea_level".to_string()));
+    }
+
+    #[test]
+    fn cf_standards_from_json_roundtrip() {
+        let json = r#"{
+  "aliases": {"air_pressure_at_sea_level": "air_pressure_at_mean_sea_level"},
+  "standard_names": {
+    "air_pressure_at_mean_sea_level": {"description": "Air pressure at mean sea level", "unit": "Pa"}
+  }
+}"#;
+        let standards = cf_standards_from_json(json).unwrap();
+        let p = &standards["air_pressure_at_mean_sea_level"];
+        assert_eq!(p.unit, "Pa");
+        assert!(p.aliases.contains(&"air_pressure_at_sea_level".to_string()));
+    }
 }

core/src/knowledge_include.rs

@@ -11,18 +11,23 @@ pub struct Knowledge {
     pub ioos_category: Option<String>,
 
     /// Common variable names in a dataset
+    #[serde(default)]
     pub common_variable_names: Vec<String>,
 
     /// Other standards to consider
+    #[serde(default)]
     pub related_standards: Vec<String>,
 
     /// Standards that are usually used together
+    #[serde(default)]
     pub sibling_standards: Vec<String>,
 
     /// Extra attributes that are usually included in Xarray or NetCDF metadata
+    #[serde(default)]
     pub extra_attrs: BTreeMap<String, String>,
 
     /// Other units that may be seen
+    #[serde(default)]
     pub other_units: Vec<String>,
 
     /// Community comments on standard usage

core/src/library_knowledge.rs

@@ -1,9 +1,10 @@
-use flate2::read::GzDecoder;
-use std::io::Read;
-
 use crate::knowledge::Knowledge;
 
+#[cfg(feature = "embedded-data")]
 pub fn load_knowledge() -> Vec<Knowledge> {
+    use flate2::read::GzDecoder;
+    use std::io::Read;
+
     let compressed_data = include_bytes!(concat!(env!("OUT_DIR"), "/knowledge.yaml.gz"));
 
     // Decompress the data
@@ -16,3 +17,13 @@ pub fn load_knowledge() -> Vec<Knowledge> {
 
     knowledge
 }
+
+/// Deserializes knowledge from a YAML string.
+pub fn load_knowledge_from_yaml(yaml: &str) -> Result<Vec<Knowledge>, String> {
+    serde_yaml_ng::from_str(yaml).map_err(|e| e.to_string())
+}
+
+/// Deserializes knowledge from a JSON string.
+pub fn load_knowledge_from_json(json: &str) -> Result<Vec<Knowledge>, String> {
+    serde_json::from_str(json).map_err(|e| e.to_string())
+}

core/src/main.rs

@@ -2,6 +2,7 @@ use standard_knowledge::standards_library::StandardsLibrary;
 
 fn main() {
     let mut library = StandardsLibrary::default();
+    #[cfg(feature = "embedded-data")]
     library.load_cf_standards();
     println!(
         "By name: {:?}",