Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion crates/storage/opendal/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,15 @@ impl StorageFactory for OpenDalStorageFactory {
OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs {
config: gcs_config_parse(config.props().clone())?.into(),
})),
#[cfg(feature = "opendal-oss")]
// OSS is S3-API-compatible; route through S3 so `s3.*` props
// work for OSS-backed tables (mirrors pyiceberg/Java S3FileIO).
#[cfg(all(feature = "opendal-oss", feature = "opendal-s3"))]
OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::S3 {
configured_scheme: "oss".to_string(),
config: s3_config_parse(config.props().clone())?.into(),
customized_credential_load: None,
})),
#[cfg(all(feature = "opendal-oss", not(feature = "opendal-s3")))]
OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss {
config: oss_config_parse(config.props().clone())?.into(),
})),
Expand Down
89 changes: 88 additions & 1 deletion crates/storage/opendal/src/resolving.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,20 @@ fn build_storage_for_scheme(
config: Arc::new(config),
})
}
#[cfg(feature = "opendal-oss")]
// OSS is S3-API-compatible; route through S3 so `s3.*` props
// work for OSS-backed tables (mirrors pyiceberg/Java S3FileIO).
#[cfg(feature = "opendal-s3")]
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
#[cfg(feature = "opendal-s3")]
#[cfg(all(feature = "opendal-oss", feature = "opendal-s3"))]

Only having s3 feature flag will introduce oss related code to non-oss users

Scheme::Oss => {
let config = crate::s3::s3_config_parse(props.clone())?;
Ok(OpenDalStorage::S3 {
configured_scheme: scheme.to_string(),
config: Arc::new(config),
customized_credential_load: customized_credential_load.clone(),
})
}
// Fallback: builds without `opendal-s3` but with `opendal-oss`
// still use the native OSS service (which consumes `oss.*` keys).
#[cfg(all(feature = "opendal-oss", not(feature = "opendal-s3")))]
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not very familiar with OSS storage so not sure about it: Is the fallback actually necessary? I'm thinking if using OSS exactly like S3 is the common case, maybe we can just use OpenDalS3Storage directly?

Scheme::Oss => {
let config = crate::oss::oss_config_parse(props.clone())?;
Ok(OpenDalStorage::Oss {
Expand Down Expand Up @@ -317,3 +330,77 @@ impl Storage for OpenDalResolvingStorage {
))
}
}

#[cfg(test)]
mod tests {
use super::*;

#[cfg(feature = "opendal-s3")]
#[test]
fn oss_scheme_routes_through_s3_backend_when_available() {
use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT};

let mut props = HashMap::new();
props.insert(S3_ACCESS_KEY_ID.to_string(), "AKIA_TEST".to_string());
props.insert(
S3_ENDPOINT.to_string(),
"https://oss-cn-shanghai.aliyuncs.com".to_string(),
);

let storage = build_storage_for_scheme(SCHEME_OSS, &props, &None).unwrap();

match storage {
OpenDalStorage::S3 {
configured_scheme, ..
} => {
assert_eq!(
configured_scheme, SCHEME_OSS,
"S3 backend should advertise the `oss` scheme so `oss://` \
URLs are accepted by OpenDalStorage::S3::create_operator"
);
}
other => panic!(
"expected OpenDalStorage::S3 for oss://, got {other:?}; OSS is \
S3-API-compatible and should share the S3 code path"
),
}
}

#[cfg(feature = "opendal-s3")]
#[test]
fn oss_resolve_uses_s3_backend_and_caches() {
use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT};

let mut props = HashMap::new();
props.insert(S3_ACCESS_KEY_ID.to_string(), "AKIA_TEST".to_string());
props.insert(
S3_ENDPOINT.to_string(),
"https://oss-cn-shanghai.aliyuncs.com".to_string(),
);

let resolving = OpenDalResolvingStorage {
props,
storages: RwLock::new(HashMap::new()),
#[cfg(feature = "opendal-s3")]
customized_credential_load: None,
};

let storage = resolving.resolve("oss://my-bucket/path/to/file").unwrap();
assert!(matches!(storage.as_ref(), OpenDalStorage::S3 { .. }));

// Second call should hit the cache.
let cached = resolving.resolve("oss://my-bucket/other").unwrap();
assert!(Arc::ptr_eq(&storage, &cached));
}

#[cfg(all(feature = "opendal-oss", not(feature = "opendal-s3")))]
#[test]
fn oss_scheme_falls_back_to_native_oss_backend() {
let props = HashMap::new();
let storage = build_storage_for_scheme(SCHEME_OSS, &props).unwrap();
assert!(
matches!(storage, OpenDalStorage::Oss { .. }),
"Without opendal-s3, OSS should use the native OSS backend"
);
}
}
Loading