diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fdf61e..e88fb04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,8 +184,9 @@ target_compile_features(trx PUBLIC cxx_std_17) target_link_libraries(trx PUBLIC - ${TRX_LIBZIP_TARGET} ${TRX_EIGEN3_TARGET} + PRIVATE + ${TRX_LIBZIP_TARGET} ) target_include_directories(trx PUBLIC @@ -202,8 +203,8 @@ get_target_property(_trx_libzip_includes ${TRX_LIBZIP_TARGET} INTERFACE_INCLUDE_ if(NOT _trx_libzip_includes) find_path(TRX_LIBZIP_INCLUDE_DIR zip.h) if(TRX_LIBZIP_INCLUDE_DIR) - target_include_directories(trx PUBLIC - $ + target_include_directories(trx PRIVATE + ${TRX_LIBZIP_INCLUDE_DIR} ) else() message(FATAL_ERROR "libzip headers not found. Set TRX_LIBZIP_INCLUDE_DIR or fix libzip CMake targets.") @@ -300,20 +301,19 @@ if(TRX_BUILD_DOCS) endif() # ── Installation and package config ───────────────────────────────────────── -# Installation requires that link dependencies are IMPORTED targets (so they -# don't need to be in the export set). This is the case when libzip and Eigen -# were found via find_package (libzip::zip, Eigen3::Eigen). When they were -# built via FetchContent (bare "zip" target), installation is not supported — -# the FetchContent path is a developer/CI convenience, not an installable -# configuration. +# CMake's install(EXPORT) requires that all link dependencies (even PRIVATE) +# are either IMPORTED targets or in the same export set. When libzip was built +# via FetchContent the bare "zip" target is neither, so install must be skipped. if(TRX_ENABLE_INSTALL AND TARGET zip) get_target_property(_zip_imported zip IMPORTED) - if(NOT _zip_imported) + get_target_property(_zip_aliased zip ALIASED_TARGET) + if(NOT _zip_imported AND NOT _zip_aliased) message(STATUS "trx-cpp: skipping install — libzip was built via FetchContent. " "Install libzip separately and reconfigure to enable installation.") set(TRX_ENABLE_INSTALL OFF) endif() unset(_zip_imported) + unset(_zip_aliased) endif() if(TRX_ENABLE_INSTALL) diff --git a/cmake/trx-cppConfig.cmake.in b/cmake/trx-cppConfig.cmake.in index 0d707cb..85e15bb 100644 --- a/cmake/trx-cppConfig.cmake.in +++ b/cmake/trx-cppConfig.cmake.in @@ -1,9 +1,9 @@ @PACKAGE_INIT@ include(CMakeFindDependencyMacro) -find_dependency(libzip) find_dependency(Eigen3) -# mio is header-only and vendored; no package required +# mio and json11 are header-only and vendored; no package required +# libzip is a PRIVATE dependency — consumers do not need it include("${CMAKE_CURRENT_LIST_DIR}/trx-cppTargets.cmake") diff --git a/include/trx/trx.h b/include/trx/trx.h index 3127b17..97babf6 100644 --- a/include/trx/trx.h +++ b/include/trx/trx.h @@ -30,13 +30,13 @@ #include #include #include -#include + #include #include #include -#include + namespace trx { namespace fs = std::filesystem; @@ -47,6 +47,8 @@ using json = json11::Json; namespace trx { enum class TrxSaveMode { Auto, Archive, Directory }; +enum class TrxCompression { None, Deflate }; + enum class TrxScalarType { Float16, Float32, Float64 }; enum class ConnectivityMeasure { @@ -61,7 +63,7 @@ struct ConnectivityMatrixResult { }; struct TrxSaveOptions { - zip_uint32_t compression_standard = ZIP_CM_STORE; + TrxCompression compression = TrxCompression::None; TrxSaveMode mode = TrxSaveMode::Auto; size_t memory_limit_bytes = 0; // Reserved for future save-path tuning. bool overwrite_existing = true; @@ -121,30 +123,7 @@ inline std::string to_utf8_string(const trx::fs::path &path) { #endif } -inline zip_t *open_zip_for_read(const std::string &path, int &errorp) { - zip_t *zf = zip_open(path.c_str(), 0, &errorp); - if (zf != nullptr) { - return zf; - } - - const trx::fs::path fs_path(path); - const std::string generic = fs_path.generic_string(); - if (generic != path) { - zf = zip_open(generic.c_str(), 0, &errorp); - if (zf != nullptr) { - return zf; - } - } - -#if defined(_WIN32) || defined(_WIN64) - const std::string u8 = to_utf8_string(fs_path); - if (!u8.empty() && u8 != path && u8 != generic) { - zf = zip_open(u8.c_str(), 0, &errorp); - } -#endif - return zf; -} template struct DTypeName { static constexpr bool supported = false; @@ -330,9 +309,9 @@ template class TrxFile { * @brief Save a TrxFile * * @param filename The path to save the TrxFile to - * @param compression_standard The compression standard to use, as defined by libzip (default: no compression) + * @param compression Compression method (default: no compression). */ - void save(const std::string &filename, zip_uint32_t compression_standard = ZIP_CM_STORE); + void save(const std::string &filename, TrxCompression compression = TrxCompression::None); void save(const std::string &filename, const TrxSaveOptions &options); /** * @brief Normalize in-memory arrays for deterministic save semantics. @@ -723,7 +702,7 @@ class AnyTrxFile { size_t num_vertices() const; size_t num_streamlines() const; void close(); - void save(const std::string &filename, zip_uint32_t compression_standard = ZIP_CM_STORE); + void save(const std::string &filename, TrxCompression compression = TrxCompression::None); void save(const std::string &filename, const TrxSaveOptions &options); const TypedArray *get_dps(const std::string &name) const; @@ -852,10 +831,10 @@ class TrxStream { /** * @brief Finalize and write a TRX file. */ - template void finalize(const std::string &filename, zip_uint32_t compression_standard = ZIP_CM_STORE); + template void finalize(const std::string &filename, TrxCompression compression = TrxCompression::None); void finalize(const std::string &filename, TrxScalarType output_dtype, - zip_uint32_t compression_standard = ZIP_CM_STORE); + TrxCompression compression = TrxCompression::None); void finalize(const std::string &filename, const TrxSaveOptions &options); /** @@ -955,15 +934,6 @@ class TrxStream { */ json assignHeader(const json &root); -/** - * This function loads the header json file - * stored within a Zip archive - * - * @param[in] zfolder a pointer to an opened zip archive - * @param[out] header the JSONCpp root of the header. nullptr on error. - * - * */ -json load_header(zip_t *zfolder); /** * Load the TRX file stored within a Zip archive. @@ -1052,7 +1022,7 @@ PositionsOutputInfo prepare_positions_output(const AnyTrxFile &input, struct MergeTrxShardsOptions { std::vector shard_directories; std::string output_path; - zip_uint32_t compression_standard = ZIP_CM_STORE; + TrxCompression compression = TrxCompression::None; bool output_directory = false; bool overwrite_existing = true; }; @@ -1325,22 +1295,28 @@ void ediff1d(Eigen::Matrix &lengths, * @tparam DT * @param trx The TrxFile to save * @param filename The path to save the TrxFile to - * @param compression_standard The compression standard to use, as defined by libzip (default: no - * compression) + * @param compression Compression method (default: no compression). + */ + +/** + * @brief Extract a TRX zip archive to a temporary directory. + * + * Returns the path to the temporary directory containing the extracted files. */ +std::string extract_trx_archive(const std::string &zip_path); /** - * @brief Utils function to zip on-disk memmaps + * @brief Create a TRX zip archive from an on-disk directory. * - * @param directory The path to the on-disk memmap - * @param filename The path where the zip file should be created - * @param compression_standard The compression standard to use, as defined by the ZipFile library + * Creates the archive at @p filename, compresses the contents of @p source_dir + * into it, optionally adding a converted-positions entry. */ -void zip_from_folder(zip_t *zf, - const std::string &root, - const std::string &directory, - zip_uint32_t compression_standard = ZIP_CM_STORE, - const std::unordered_set *skip = nullptr); +void write_trx_archive(const std::string &filename, + const std::string &source_dir, + TrxCompression compression, + const std::string &converted_positions_path = "", + const std::string &converted_positions_entry = "", + const std::unordered_set *skip = nullptr); std::string get_base(const std::string &delimiter, const std::string &str); std::string get_ext(const std::string &str); @@ -1350,7 +1326,6 @@ void copy_dir(const std::string &src, const std::string &dst); void copy_file(const std::string &src, const std::string &dst); int rm_dir(const std::string &d); std::string make_temp_dir(const std::string &prefix); -std::string extract_zip_to_directory(zip_t *zfolder); std::string rm_root(const std::string &root, const std::string &path); @@ -1363,13 +1338,13 @@ std::string rm_root(const std::string &root, const std::string &path); * * @param path Path to an existing TRX zip file (.trx). * @param groups Map from group name to sorted vector of uint32 streamline indices. - * @param compression Compression method for new entries (default: ZIP_CM_STORE). + * @param compression Compression method for new entries (default: TrxCompression::None). * @param overwrite If false, existing entries with the same name are skipped (default: true). * @throws TrxIOError on any zip failure. */ void append_groups_to_zip(const std::string &path, const std::map> &groups, - zip_uint32_t compression = ZIP_CM_STORE, bool overwrite = true); + TrxCompression compression = TrxCompression::None, bool overwrite = true); /** * @brief Append named groups to an existing TRX directory. @@ -1396,12 +1371,12 @@ void append_groups_to_directory(const std::string &directory, * * @param path Path to an existing TRX zip file (.trx). * @param dps Map from array name to TypedArray (e.g. AnyTrxFile::data_per_streamline). - * @param compression Compression method for new entries (default: ZIP_CM_STORE). + * @param compression Compression method for new entries (default: TrxCompression::None). * @param overwrite If false, existing entries with the same name are skipped (default: true). * @throws TrxIOError on any zip failure. */ void append_dps_to_zip(const std::string &path, const std::map &dps, - zip_uint32_t compression = ZIP_CM_STORE, bool overwrite = true); + TrxCompression compression = TrxCompression::None, bool overwrite = true); /** * @brief Append per-streamline data arrays to an existing TRX directory. @@ -1427,12 +1402,12 @@ void append_dps_to_directory(const std::string &directory, * * @param path Path to an existing TRX zip file (.trx). * @param dpv Map from array name to TypedArray (e.g. AnyTrxFile::data_per_vertex). - * @param compression Compression method for new entries (default: ZIP_CM_STORE). + * @param compression Compression method for new entries (default: TrxCompression::None). * @param overwrite If false, existing entries with the same name are skipped (default: true). * @throws TrxIOError on any zip failure. */ void append_dpv_to_zip(const std::string &path, const std::map &dpv, - zip_uint32_t compression = ZIP_CM_STORE, bool overwrite = true); + TrxCompression compression = TrxCompression::None, bool overwrite = true); /** * @brief Append per-vertex data arrays to an existing TRX directory. diff --git a/include/trx/trx.tpp b/include/trx/trx.tpp index 4351f87..ac92ef7 100644 --- a/include/trx/trx.tpp +++ b/include/trx/trx.tpp @@ -969,13 +969,7 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { } template std::unique_ptr> TrxFile
::load_from_zip(const std::string &filename) { - int errorp = 0; - detail::ZipArchive zf(open_zip_for_read(filename, errorp)); - if (!zf) { - throw TrxIOError("Could not open zip file: " + filename); - } - - std::string temp_dir = extract_zip_to_directory(zf.get()); + std::string temp_dir = extract_trx_archive(filename); auto trx = TrxFile
::load_from_directory(temp_dir); trx->_uncompressed_folder_handle = temp_dir; @@ -1134,9 +1128,9 @@ auto with_trx_reader(const std::string &path, Fn &&fn) } } -template void TrxFile
::save(const std::string &filename, zip_uint32_t compression_standard) { +template void TrxFile
::save(const std::string &filename, TrxCompression compression) { TrxSaveOptions options; - options.compression_standard = compression_standard; + options.compression = compression; save(filename, options); } @@ -1280,13 +1274,9 @@ template void TrxFile
::save(const std::string &filename, const } } - int errorp; - detail::ZipArchive zf(zip_open(filename.c_str(), ZIP_CREATE + ZIP_TRUNCATE, &errorp)); - if (!zf) { - throw TrxIOError("Could not open archive " + filename + ": " + strerror(errorp)); - } - std::unordered_set skip; + std::string converted_pos_path; + std::string converted_pos_entry; detail::TempFileGuard tmp_pos_guard; if (options.positions_dtype.has_value() && save_trx->streamlines) { const TrxScalarType target = *options.positions_dtype; @@ -1299,24 +1289,14 @@ template void TrxFile
::save(const std::string &filename, const auto src_any = load_any(tmp_dir_name); write_positions_as_dtype(src_any, target, tmp_pos_guard.path); } - const std::string new_pos_name = "positions.3." + new_dtype_str; - zip_source_t *pos_src = zip_source_file(zf.get(), tmp_pos_guard.path.c_str(), 0, -1); - if (!pos_src) - throw TrxIOError("Failed to create zip source for converted positions"); - const zip_int64_t pos_idx = zip_file_add( - zf.get(), new_pos_name.c_str(), pos_src, ZIP_FL_ENC_UTF_8 | ZIP_FL_OVERWRITE); - if (pos_idx < 0) - throw TrxIOError("Failed to add converted positions to archive"); - if (zip_set_file_compression( - zf.get(), pos_idx, - static_cast(options.compression_standard), 0) < 0) - throw TrxIOError("Failed to set compression for converted positions"); - } - } - zip_from_folder(zf.get(), tmp_dir_name, tmp_dir_name, options.compression_standard, - skip.empty() ? nullptr : &skip); - zf.commit(filename); - // tmp_pos_guard destructor removes the temp file after commit. + converted_pos_path = tmp_pos_guard.path; + converted_pos_entry = "positions.3." + new_dtype_str; + } + } + write_trx_archive(filename, tmp_dir_name, options.compression, + converted_pos_path, converted_pos_entry, + skip.empty() ? nullptr : &skip); + // tmp_pos_guard destructor removes the temp file after archive is written. } else { std::error_code ec; if (!trx::fs::exists(tmp_dir_name, ec) || !trx::fs::is_directory(tmp_dir_name, ec)) { @@ -2008,7 +1988,7 @@ inline void TrxStream::push_group_from_indices(const std::string &name, const st } } -template void TrxStream::finalize(const std::string &filename, zip_uint32_t compression_standard) { +template void TrxStream::finalize(const std::string &filename, TrxCompression compression) { if (finalized_) { throw TrxArgumentError("TrxStream already finalized"); } @@ -2092,7 +2072,7 @@ template void TrxStream::finalize(const std::string &filename, zip } } - trx.save(filename, compression_standard); + trx.save(filename, compression); trx.close(); cleanup_tmp(); @@ -2100,17 +2080,17 @@ template void TrxStream::finalize(const std::string &filename, zip inline void TrxStream::finalize(const std::string &filename, TrxScalarType output_dtype, - zip_uint32_t compression_standard) { + TrxCompression compression) { switch (output_dtype) { case TrxScalarType::Float16: - finalize(filename, compression_standard); + finalize(filename, compression); break; case TrxScalarType::Float64: - finalize(filename, compression_standard); + finalize(filename, compression); break; case TrxScalarType::Float32: default: - finalize(filename, compression_standard); + finalize(filename, compression); break; } } @@ -2134,7 +2114,7 @@ inline void TrxStream::finalize(const std::string &filename, const TrxSaveOption } else if (positions_dtype_ == "float64") { out_type = TrxScalarType::Float64; } - finalize(filename, out_type, options.compression_standard); + finalize(filename, out_type, options.compression); } inline void TrxStream::finalize_directory_impl(const std::string &directory, bool remove_existing) { diff --git a/src/trx.cpp b/src/trx.cpp index baa3b61..869a734 100644 --- a/src/trx.cpp +++ b/src/trx.cpp @@ -48,6 +48,49 @@ using std::uniform_int_distribution; using std::vector; namespace trx { + +// Forward declarations for functions defined later in this file. +// These were previously declared in trx.h but are now internal. +std::string extract_zip_to_directory(zip_t *zfolder); +void zip_from_folder(zip_t *zf, const std::string &root, const std::string &directory, + zip_uint32_t compression_standard, const std::unordered_set *skip); +json load_header(zip_t *zfolder); + +zip_uint32_t to_zip_compression(TrxCompression c) { + switch (c) { + case TrxCompression::Deflate: + return ZIP_CM_DEFLATE; + case TrxCompression::None: + default: + return ZIP_CM_STORE; + } +} + +zip_t *open_zip_for_read(const std::string &path, int &errorp) { + zip_t *zf = zip_open(path.c_str(), 0, &errorp); + if (zf != nullptr) { + return zf; + } + + const trx::fs::path fs_path(path); + const std::string generic = fs_path.generic_string(); + if (generic != path) { + zf = zip_open(generic.c_str(), 0, &errorp); + if (zf != nullptr) { + return zf; + } + } + +#if defined(_WIN32) || defined(_WIN64) + const std::string u8 = to_utf8_string(fs_path); + if (!u8.empty() && u8 != path && u8 != generic) { + zf = zip_open(u8.c_str(), 0, &errorp); + } +#endif + + return zf; +} + namespace { inline int sys_error() { return errno; } @@ -577,9 +620,9 @@ void write_positions_as_dtype(const AnyTrxFile &source, throw TrxIOError("I/O error writing converted positions to: " + out_path); } -void AnyTrxFile::save(const std::string &filename, zip_uint32_t compression_standard) { +void AnyTrxFile::save(const std::string &filename, TrxCompression compression) { TrxSaveOptions options; - options.compression_standard = compression_standard; + options.compression = compression; save(filename, options); } @@ -649,7 +692,7 @@ void AnyTrxFile::save(const std::string &filename, const TrxSaveOptions &options if (header_idx < 0) { throw TrxIOError("Failed to add header.json to archive: " + std::string(zip_strerror(zf.get()))); } - const zip_int32_t compression = static_cast(options.compression_standard); + const zip_int32_t compression = static_cast(to_zip_compression(options.compression)); if (zip_set_file_compression(zf.get(), header_idx, compression, 0) < 0) { throw TrxIOError("Failed to set compression for header.json: " + std::string(zip_strerror(zf.get()))); @@ -679,7 +722,7 @@ void AnyTrxFile::save(const std::string &filename, const TrxSaveOptions &options throw TrxIOError("Failed to set compression for converted positions"); } } - zip_from_folder(zf.get(), source_dir, source_dir, options.compression_standard, &skip); + zip_from_folder(zf.get(), source_dir, source_dir, to_zip_compression(options.compression), &skip); zf.commit(filename); } else { std::error_code ec; @@ -1141,6 +1184,44 @@ void zip_from_folder(zip_t *zf, } } +std::string extract_trx_archive(const std::string &zip_path) { + int errorp = 0; + detail::ZipArchive zf(open_zip_for_read(zip_path, errorp)); + if (!zf) { + throw TrxIOError("Could not open zip file: " + zip_path); + } + return extract_zip_to_directory(zf.get()); +} + +void write_trx_archive(const std::string &filename, + const std::string &source_dir, + TrxCompression compression, + const std::string &converted_positions_path, + const std::string &converted_positions_entry, + const std::unordered_set *skip) { + const zip_uint32_t zip_comp = to_zip_compression(compression); + int errorp; + detail::ZipArchive zf(zip_open(filename.c_str(), ZIP_CREATE + ZIP_TRUNCATE, &errorp)); + if (!zf) { + throw TrxIOError("Could not open archive " + filename + ": " + strerror(errorp)); + } + + if (!converted_positions_path.empty() && !converted_positions_entry.empty()) { + zip_source_t *pos_src = zip_source_file(zf.get(), converted_positions_path.c_str(), 0, -1); + if (!pos_src) + throw TrxIOError("Failed to create zip source for converted positions"); + const zip_int64_t pos_idx = zip_file_add( + zf.get(), converted_positions_entry.c_str(), pos_src, ZIP_FL_ENC_UTF_8 | ZIP_FL_OVERWRITE); + if (pos_idx < 0) + throw TrxIOError("Failed to add converted positions to archive"); + if (zip_set_file_compression(zf.get(), pos_idx, static_cast(zip_comp), 0) < 0) + throw TrxIOError("Failed to set compression for converted positions"); + } + + zip_from_folder(zf.get(), source_dir, source_dir, zip_comp, skip); + zf.commit(filename); +} + std::string rm_root(const std::string &root, const std::string &path) { std::string stripped; @@ -1630,7 +1711,7 @@ void merge_trx_shards(const MergeTrxShardsOptions &options) { if (!zf) { throw TrxIOError("Could not open archive " + options.output_path + ": " + strerror(errorp)); } - zip_from_folder(zf.get(), output_dir, output_dir, options.compression_standard, nullptr); + zip_from_folder(zf.get(), output_dir, output_dir, to_zip_compression(options.compression), nullptr); zf.commit(options.output_path); rm_dir(output_dir); } @@ -1736,13 +1817,13 @@ void append_raw_entries_to_directory(const std::string &directory, const std::st void append_groups_to_zip(const std::string &path, const std::map> &groups, - zip_uint32_t compression, bool overwrite) { + TrxCompression compression, bool overwrite) { std::vector entries; entries.reserve(groups.size()); for (const auto &kv : groups) { entries.push_back({kv.first + ".uint32", kv.second.data(), kv.second.size() * sizeof(uint32_t)}); } - append_raw_entries_to_zip(path, "groups", entries, compression, overwrite); + append_raw_entries_to_zip(path, "groups", entries, to_zip_compression(compression), overwrite); } void append_groups_to_directory(const std::string &directory, @@ -1757,14 +1838,14 @@ void append_groups_to_directory(const std::string &directory, } void append_dps_to_zip(const std::string &path, const std::map &dps, - zip_uint32_t compression, bool overwrite) { + TrxCompression compression, bool overwrite) { std::vector entries; entries.reserve(dps.size()); for (const auto &kv : dps) { const auto bytes = kv.second.to_bytes(); entries.push_back({typed_array_filename(kv.first, kv.second), bytes.data, bytes.size}); } - append_raw_entries_to_zip(path, "dps", entries, compression, overwrite); + append_raw_entries_to_zip(path, "dps", entries, to_zip_compression(compression), overwrite); } void append_dps_to_directory(const std::string &directory, @@ -1779,14 +1860,14 @@ void append_dps_to_directory(const std::string &directory, } void append_dpv_to_zip(const std::string &path, const std::map &dpv, - zip_uint32_t compression, bool overwrite) { + TrxCompression compression, bool overwrite) { std::vector entries; entries.reserve(dpv.size()); for (const auto &kv : dpv) { const auto bytes = kv.second.to_bytes(); entries.push_back({typed_array_filename(kv.first, kv.second), bytes.data, bytes.size}); } - append_raw_entries_to_zip(path, "dpv", entries, compression, overwrite); + append_raw_entries_to_zip(path, "dpv", entries, to_zip_compression(compression), overwrite); } void append_dpv_to_directory(const std::string &directory, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4f97569..f37f3db 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,11 +69,11 @@ else() endif() add_executable(test_mmap test_trx_mmap.cpp) -target_link_libraries(test_mmap PRIVATE trx GTest::gtest_main) +target_link_libraries(test_mmap PRIVATE trx ${TRX_LIBZIP_TARGET} GTest::gtest_main) target_compile_features(test_mmap PRIVATE cxx_std_17) add_executable(test_io test_trx_io.cpp) -target_link_libraries(test_io PRIVATE trx GTest::gtest_main) +target_link_libraries(test_io PRIVATE trx ${TRX_LIBZIP_TARGET} GTest::gtest_main) target_compile_features(test_io PRIVATE cxx_std_17) if(TRX_ENABLE_NIFTI) target_link_libraries(test_io PRIVATE trx-nifti) diff --git a/tests/test_trx_anytrxfile.cpp b/tests/test_trx_anytrxfile.cpp index e6671c5..7dfe8e1 100644 --- a/tests/test_trx_anytrxfile.cpp +++ b/tests/test_trx_anytrxfile.cpp @@ -410,7 +410,7 @@ TEST(AnyTrxFile, SaveUpdatesHeader) { const auto temp_dir = make_temp_test_dir("trx_any_save"); const fs::path out_path = temp_dir / "saved_copy.trx"; - trx.save(out_path.string(), ZIP_CM_STORE); + trx.save(out_path.string(), trx::TrxCompression::None); trx.close(); auto reloaded = load_any(out_path.string()); @@ -465,14 +465,14 @@ TEST(AnyTrxFile, AppendGroupsToZipAddsAndOverwrites) { const fs::path archive = temp_root / "base.trx"; auto source = load_any(shard.string()); - source.save(archive.string(), ZIP_CM_STORE); + source.save(archive.string(), trx::TrxCompression::None); source.close(); const std::map> appended_groups = { {"Bundle", {1}}, {"NewBundle", {0, 1}}, }; - append_groups_to_zip(archive.string(), appended_groups, ZIP_CM_STORE); + append_groups_to_zip(archive.string(), appended_groups, trx::TrxCompression::None); auto loaded = load_any(archive.string()); auto bundle_it = loaded.groups.find("Bundle"); @@ -543,14 +543,14 @@ TEST(AnyTrxFile, AppendDpsToZipAddsAndOverwrites) { const fs::path archive = temp_root / "base.trx"; auto source = load_any(shard.string()); - source.save(archive.string(), ZIP_CM_STORE); + source.save(archive.string(), trx::TrxCompression::None); source.close(); const std::map appended_dps = { {"weight", make_float32_array({99.0F, 88.0F})}, // overwrite existing {"confidence", make_float32_array({0.9F, 0.8F})}, // new entry }; - append_dps_to_zip(archive.string(), appended_dps, ZIP_CM_STORE); + append_dps_to_zip(archive.string(), appended_dps, trx::TrxCompression::None); auto loaded = load_any(archive.string()); @@ -625,14 +625,14 @@ TEST(AnyTrxFile, AppendDpvToZipAddsAndOverwrites) { const fs::path archive = temp_root / "base.trx"; auto source = load_any(shard.string()); - source.save(archive.string(), ZIP_CM_STORE); + source.save(archive.string(), trx::TrxCompression::None); source.close(); const std::map appended_dpv = { {"signal", make_float32_array({1.1F, 2.2F, 3.3F})}, // overwrite existing {"curvature", make_float32_array({0.5F, 0.6F, 0.7F})}, // new entry }; - append_dpv_to_zip(archive.string(), appended_dpv, ZIP_CM_STORE); + append_dpv_to_zip(archive.string(), appended_dpv, trx::TrxCompression::None); auto loaded = load_any(archive.string()); @@ -731,18 +731,18 @@ TEST(AnyTrxFile, AppendNoOverwriteSkipsExistingZip) { const fs::path archive = temp_root / "base.trx"; auto source = load_any(shard.string()); - source.save(archive.string(), ZIP_CM_STORE); + source.save(archive.string(), trx::TrxCompression::None); source.close(); append_dps_to_zip(archive.string(), {{"weight", make_float32_array({99.0F, 88.0F})}, {"confidence", make_float32_array({0.9F, 0.8F})}}, - ZIP_CM_STORE, /*overwrite=*/false); + trx::TrxCompression::None, /*overwrite=*/false); append_dpv_to_zip(archive.string(), {{"signal", make_float32_array({9.9F, 8.8F, 7.7F})}, {"curvature", make_float32_array({0.5F, 0.6F, 0.7F})}}, - ZIP_CM_STORE, /*overwrite=*/false); - append_groups_to_zip(archive.string(), {{"Bundle", {0, 1}}, {"NewBundle", {1}}}, ZIP_CM_STORE, + trx::TrxCompression::None, /*overwrite=*/false); + append_groups_to_zip(archive.string(), {{"Bundle", {0, 1}}, {"NewBundle", {1}}}, trx::TrxCompression::None, /*overwrite=*/false); auto loaded = load_any(archive.string()); @@ -1020,7 +1020,7 @@ TEST(AnyTrxFile, SaveRejectsUnsupportedExtension) { const auto temp_dir = make_temp_test_dir("trx_any_save_badext"); const fs::path out_path = temp_dir / "bad.txt"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxDTypeError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxDTypeError); trx.close(); std::error_code ec; @@ -1035,7 +1035,7 @@ TEST(AnyTrxFile, SaveRejectsMissingOffsets) { trx.offsets = TypedArray(); const auto temp_dir = make_temp_test_dir("trx_any_save_no_offsets"); const fs::path out_path = temp_dir / "missing_offsets.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1050,7 +1050,7 @@ TEST(AnyTrxFile, SaveRejectsMissingDecodedOffsets) { trx.offsets_u64.clear(); const auto temp_dir = make_temp_test_dir("trx_any_save_no_offsets_u64"); const fs::path out_path = temp_dir / "missing_offsets_u64.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1068,7 +1068,7 @@ TEST(AnyTrxFile, SaveRejectsStreamlineCountMismatch) { const auto temp_dir = make_temp_test_dir("trx_any_save_bad_streamlines"); const fs::path out_path = temp_dir / "bad_streamlines.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1086,7 +1086,7 @@ TEST(AnyTrxFile, SaveRejectsVertexCountMismatch) { const auto temp_dir = make_temp_test_dir("trx_any_save_bad_vertices"); const fs::path out_path = temp_dir / "bad_vertices.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1104,7 +1104,7 @@ TEST(AnyTrxFile, SaveRejectsNonMonotonicOffsets) { const auto temp_dir = make_temp_test_dir("trx_any_save_non_mono"); const fs::path out_path = temp_dir / "non_mono.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1122,7 +1122,7 @@ TEST(AnyTrxFile, SaveRejectsPositionsRowMismatch) { const auto temp_dir = make_temp_test_dir("trx_any_save_bad_positions"); const fs::path out_path = temp_dir / "bad_positions.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxFormatError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxFormatError); trx.close(); std::error_code ec; @@ -1139,7 +1139,7 @@ TEST(AnyTrxFile, SaveRejectsMissingBackingDirectory) { const auto temp_dir = make_temp_test_dir("trx_any_save_no_backing"); const fs::path out_path = temp_dir / "no_backing.trx"; - EXPECT_THROW(trx.save(out_path.string(), ZIP_CM_STORE), trx::TrxIOError); + EXPECT_THROW(trx.save(out_path.string(), trx::TrxCompression::None), trx::TrxIOError); trx.close(); std::error_code ec; @@ -1424,7 +1424,7 @@ TEST(AnyTrxFile, SaveRespectsExplicitMode) { const fs::path archive_out = temp_root / "save_archive_mode.trx"; TrxSaveOptions archive_opts; archive_opts.mode = TrxSaveMode::Archive; - archive_opts.compression_standard = ZIP_CM_STORE; + archive_opts.compression = trx::TrxCompression::None; trx.save(archive_out.string(), archive_opts); EXPECT_TRUE(fs::is_regular_file(archive_out)); diff --git a/tests/test_trx_mmap.cpp b/tests/test_trx_mmap.cpp index 270158e..376dc8e 100644 --- a/tests/test_trx_mmap.cpp +++ b/tests/test_trx_mmap.cpp @@ -7,9 +7,19 @@ #include #include #include +#include #include #include +// Internal functions not exposed in the public API — declared here for testing. +namespace trx { +zip_t *open_zip_for_read(const std::string &path, int &errorp); +std::string extract_zip_to_directory(zip_t *zfolder); +void zip_from_folder(zip_t *zf, const std::string &root, const std::string &directory, + zip_uint32_t compression_standard, const std::unordered_set *skip); +json load_header(zip_t *zfolder); +} // namespace trx + using namespace Eigen; using ::json; using trx::TrxFile; @@ -712,7 +722,7 @@ TEST(TrxFileMemmap, load_mixed_metadata_dtype_into_half_reader) { stream.push_streamline(std::vector{2.0f, 2.0f, 2.0f, 3.0f, 3.0f, 3.0f}); stream.push_dps_from_vector("dps1", "float32", std::vector{1.0f, 4.0f}); stream.push_dpv_from_vector("dpv1", "float32", std::vector{100.0f, 101.0f, 106.0f, 107.0f}); - stream.finalize(out_path.string(), ZIP_CM_STORE); + stream.finalize(out_path.string(), trx::TrxCompression::None); trx::TrxReader reader(out_path.string()); auto *trx = reader.get(); diff --git a/tests/test_trx_trxfile.cpp b/tests/test_trx_trxfile.cpp index df78331..a4c5d93 100644 --- a/tests/test_trx_trxfile.cpp +++ b/tests/test_trx_trxfile.cpp @@ -196,7 +196,7 @@ fs::path create_connectivity_fixture_trx() { stream.push_group_from_indices("B", {1, 3, 4}); stream.push_group_from_indices("C", {2, 4}); stream.push_dps_from_vector("weight", "float32", std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f}); - stream.finalize(out_path.string(), ZIP_CM_STORE); + stream.finalize(out_path.string(), trx::TrxCompression::None); return out_path; } @@ -215,7 +215,7 @@ fs::path create_many_groups_fixture_trx(size_t group_count, size_t streamline_co const uint32_t sid = static_cast(g % streamline_count); stream.push_group_from_indices(name, {sid}); } - stream.finalize(out_path.string(), ZIP_CM_STORE); + stream.finalize(out_path.string(), trx::TrxCompression::None); return out_path; } @@ -275,7 +275,7 @@ fs::path create_connectivity_no_groups_fixture_trx(size_t streamline_count = 3) for (size_t i = 0; i < streamline_count; ++i) { stream.push_streamline(std::vector{0.0f, 0.0f, 0.0f}); } - stream.finalize(out_path.string(), ZIP_CM_STORE); + stream.finalize(out_path.string(), trx::TrxCompression::None); return out_path; } @@ -635,7 +635,7 @@ TEST(TrxFileTpp, TrxStreamFinalize) { proto.push_group_from_indices("GroupA", {0, 1}); proto.push_dps_from_vector("weight", "float32", std::vector{0.5f, 1.5f}); proto.push_dpv_from_vector("score", "float32", std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f}); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); auto trx = load_any(out_path.string()); EXPECT_EQ(trx.num_streamlines(), 2u); @@ -690,7 +690,7 @@ TEST(TrxFileTpp, TrxStreamOnDiskMetadataAllDtypes) { proto.push_dpv_from_vector("s_f32", "float32", std::vector{1.0f, 2.0f, 3.0f}); proto.push_dpv_from_vector("s_f64", "float64", std::vector{1.0, 2.0, 3.0}); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); auto trx = load_any(out_path.string()); EXPECT_EQ(trx.num_streamlines(), 2u); @@ -1059,7 +1059,7 @@ TEST(TrxFileTpp, TrxStreamFloat16Unbuffered) { std::vector sl2 = {2.0f, 0.0f, 0.0f}; proto.push_streamline(sl1); proto.push_streamline(sl2); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); auto trx = load_any(out_path.string()); EXPECT_EQ(trx.num_streamlines(), 2u); @@ -1086,7 +1086,7 @@ TEST(TrxFileTpp, TrxStreamFloat16Buffered) { proto.push_streamline(pt); // buffer: 3 halves proto.push_streamline(pt); // buffer: 6 halves >= 6 → flush proto.push_streamline(pt); // buffer: 3 halves (after flush) - proto.finalize(out_path.string(), ZIP_CM_STORE); // flush remainder, then early-return + proto.finalize(out_path.string(), trx::TrxCompression::None); // flush remainder, then early-return auto trx = load_any(out_path.string()); EXPECT_EQ(trx.num_streamlines(), 3u); @@ -1104,7 +1104,7 @@ TEST(TrxFileTpp, LoadFloat32PositionsFromFloat16Chunked) { TrxStream proto("float16"); proto.push_streamline(std::vector{0.1f, 1.2f, 2.3f, 3.4f, 4.5f, 5.6f}); proto.push_streamline(std::vector{6.7f, 7.8f, 8.9f}); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); LoadFloat32Options options; options.chunk_rows = 1; // force chunked loop execution @@ -1130,7 +1130,7 @@ TEST(TrxFileTpp, LoadFloat32PositionsPassthroughFromFloat32) { TrxStream proto("float32"); proto.push_streamline(std::vector{0.25f, 1.25f, 2.25f, 3.25f, 4.25f, 5.25f}); proto.push_streamline(std::vector{6.25f, 7.25f, 8.25f}); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); auto via_api = load_float32_positions(out_path.string()); auto direct = load(out_path.string()); @@ -1286,7 +1286,7 @@ TEST(TrxFileTpp, TrxStreamFloat16InMemoryFloat32DpsRoundtrip) { const std::vector dpv_in = {1.5f, 26880.0f, 0.03125f}; proto.push_dpv_from_vector("score", "float32", dpv_in); - proto.finalize(out_path.string(), ZIP_CM_STORE); + proto.finalize(out_path.string(), trx::TrxCompression::None); // Load via AnyTrxFile so as_matrix() reads the on-disk dtype directly. auto trx = load_any(out_path.string());