diff --git a/src/GlobalStates/CacheStore.ts b/src/GlobalStates/CacheStore.ts index ac7c592e..2d433f3e 100644 --- a/src/GlobalStates/CacheStore.ts +++ b/src/GlobalStates/CacheStore.ts @@ -4,23 +4,30 @@ import MemoryLRU from "../utils/MemoryLRU"; type CacheState = { cache: MemoryLRU; maxSize: number; + cacheVersion: number; clearCache: () => void; setMaxSize: (maxSize: number) => void; } -const memoryModule = new MemoryLRU({ maxSize: 200 * 1024 * 1024 }) // 200 MB // Maybe moving it outside will allow Garbe Collector to correctly remove the cached data +export const useCacheStore = create((set, get) => { + const memoryModule = new MemoryLRU({ + maxSize: 200 * 1024 * 1024, + onChange: () => set(state => ({ cacheVersion: state.cacheVersion + 1 })) + }) -export const useCacheStore = create((set, get) => ({ - cache: memoryModule as MemoryLRU, // 200 MB - maxSize: 200 * 1024 * 1024, - // Cache operations - clearCache: () => { - const { cache } = get() - cache.clear() - }, - setMaxSize: (maxSize) => { - const { cache } = get() - cache.resize(maxSize) - set({ maxSize }) + return { + cache: memoryModule as MemoryLRU, // 200 MB + maxSize: 200 * 1024 * 1024, + cacheVersion: 0, + // Cache operations + clearCache: () => { + const { cache } = get() + cache.clear() + }, + setMaxSize: (maxSize) => { + const { cache } = get() + cache.resize(maxSize) + set({ maxSize }) + } } -})) \ No newline at end of file +}) \ No newline at end of file diff --git a/src/components/ui/MainPanel/MetaDataInfo.tsx b/src/components/ui/MainPanel/MetaDataInfo.tsx index 53abdfef..42a085fe 100644 --- a/src/components/ui/MainPanel/MetaDataInfo.tsx +++ b/src/components/ui/MainPanel/MetaDataInfo.tsx @@ -50,8 +50,11 @@ function ChunkIDs(slices:{xSlice: Slice, ySlice: Slice, zSlice: Slice}, chunkSha return ids } -function HandleCustomSteps(e: string, chunkSize: number){ +function HandleCustomSteps(e: string, chunkSize: number, unrestricted: boolean = false){ const newVal = parseInt(e); + if (unrestricted) { + return newVal; + } const chunkStep = Math.floor(newVal/chunkSize) * chunkSize return chunkStep } @@ -64,7 +67,7 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi setIs4D: state.setIs4D, setIdx4D: state.setIdx4D, setVariable: state.setVariable, setTextureArrayDepths: state.setTextureArrayDepths, }))) - const {maxSize, cache, setMaxSize} = useCacheStore(useShallow(state => ({maxSize: state.maxSize, cache:state.cache, setMaxSize:state.setMaxSize}))) + const {maxSize, cache, setMaxSize, cacheVersion} = useCacheStore(useShallow(state => ({maxSize: state.maxSize, cache:state.cache, setMaxSize:state.setMaxSize, cacheVersion: state.cacheVersion}))) const [cacheSize, setCacheSize] = useState(maxSize) const { zSlice, ySlice, xSlice, compress, coarsen, kernelSize, kernelDepth, setZSlice, setYSlice, setXSlice, ReFetch, setCompress, setCoarsen, setKernelSize, setKernelDepth } = useZarrStore(useShallow(state => ({ zSlice: state.zSlice, ySlice: state.ySlice, xSlice: state.xSlice, @@ -83,6 +86,7 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi const [texCount, setTexCount] = useState(0) const [displaySpat, setDisplaySpat] = useState(String(kernelSize)) const [displayDepth, setDisplayDepth] = useState(String(kernelDepth)) + const [unrestrictedSliders, setUnrestrictedSliders] = useState(false) // ---- Meta Info ---- // const {dimArrays, dimNames, dimUnits} = meta.dimInfo @@ -166,16 +170,25 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi const sizeRatio = totalSteps / (meta.shape[0] * meta.shape[1]); return totalSize * sizeRatio; } else { - const chunkIndices = is4D ? [3, 2, 1] : [2, 1, 0]; - const xChunksNeeded = Math.ceil(x.steps / meta.chunks[chunkIndices[0]]); - const yChunksNeeded = Math.ceil(y.steps / meta.chunks[chunkIndices[1]]); - const zChunksNeeded = Math.ceil(z.steps / meta.chunks[chunkIndices[2]]); + if (unrestrictedSliders) { + // For unrestricted mode, calculate based on actual steps selected, not chunks + const elementsPerChunk = meta.chunkSize / (is2D ? 4 : 4); // Assuming 4 bytes per element + const totalElements = z.steps * y.steps * x.steps; + const size = totalElements * 4; // 4 bytes per float32 + return size / (coarsen ? kernelDepth * Math.pow(kernelSize, 2) : 1); + } else { + // Original chunk-based calculation + const chunkIndices = is4D ? [3, 2, 1] : [2, 1, 0]; + const xChunksNeeded = Math.ceil(x.steps / meta.chunks[chunkIndices[0]]); + const yChunksNeeded = Math.ceil(y.steps / meta.chunks[chunkIndices[1]]); + const zChunksNeeded = Math.ceil(z.steps / meta.chunks[chunkIndices[2]]); - const size = xChunksNeeded * yChunksNeeded * zChunksNeeded * meta.chunkSize - - return size / (coarsen ? kernelDepth * Math.pow(kernelSize, 2) : 1) + const size = xChunksNeeded * yChunksNeeded * zChunksNeeded * meta.chunkSize + + return size / (coarsen ? kernelDepth * Math.pow(kernelSize, 2) : 1) + } } - }, [meta, zSlice, xSlice, ySlice, zLength, is3D, is4D, coarsen, kernelSize, kernelDepth]); + }, [meta, zSlice, xSlice, ySlice, zLength, is3D, is4D, coarsen, kernelSize, kernelDepth, unrestrictedSliders]); const cachedSize = useMemo(()=>{ const thisDtype = meta.dtype as string @@ -223,7 +236,7 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi } else { setCached(false) } - },[meta, chunkIDs]) + },[meta, chunkIDs, cacheVersion]) return ( <> @@ -310,6 +323,18 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi {(hasTimeChunks || hasXChunks || hasYChunks ) && ( <> Trim Data +
+ + setUnrestrictedSliders(e)}/> + + + + + + Enable to select any value instead of being restricted to chunk boundaries + + +
{hasTimeChunks &&
@@ -319,20 +344,20 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi min={0} max={zLength} value={[zSlice[0] ? zSlice[0] : 0, zSlice[1] ? zSlice[1] : zLength]} - step={chunkShape[0]} + step={unrestrictedSliders ? 1 : chunkShape[0]} onValueChange={(values: number[]) => setZSlice([values[0], values[1]] as [number, number | null])} />
Min: {parseLoc(dimArrays[is4D ? 1 : 0]?.[zSlice[0]]?? null, dimUnits[is4D ? 1 : 0]?? null)}
Index: setZSlice([parseInt(e.target.value), zSlice[1]])} - onBlur={e=>setZSlice([HandleCustomSteps(e.target.value,chunkShape[0]), zSlice[1]])} + onBlur={e=>setZSlice([HandleCustomSteps(e.target.value,chunkShape[0], unrestrictedSliders), zSlice[1]])} /> Max: {parseLoc(dimArrays[is4D ? 1 : 0]?.[zSlice[1] ? zSlice[1]-1 : zLength-1]?? null, dimUnits[is4D ? 1 : 0]?? null)}
Index: setZSlice([zSlice[0], parseInt(e.target.value)])} - onBlur={e=>setZSlice([zSlice[0], HandleCustomSteps(e.target.value,chunkShape[0])])} + onBlur={e=>setZSlice([zSlice[0], HandleCustomSteps(e.target.value,chunkShape[0], unrestrictedSliders)])} />
@@ -345,20 +370,20 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi min={0} max={yLength} value={[ySlice[0] ? ySlice[0] : 0, ySlice[1] ? ySlice[1] : yLength]} - step={isFlat ? chunkShape[0] : chunkShape[1]} + step={unrestrictedSliders ? 1 : (isFlat ? chunkShape[0] : chunkShape[1])} onValueChange={(values: number[]) => setYSlice([values[0], values[1]] as [number, number | null])} />
Min: {parseLoc(dimArrays[isFlat ? 0 : shapeLength-2]?.[ySlice[0]]?? null, dimUnits[isFlat ? 0 : shapeLength-2]?? null)}
Index: setYSlice([parseInt(e.target.value), ySlice[1]])} - onBlur={e=>setYSlice([HandleCustomSteps(e.target.value,chunkShape[1]), ySlice[1]])} + onBlur={e=>setYSlice([HandleCustomSteps(e.target.value, isFlat ? chunkShape[0] : chunkShape[1], unrestrictedSliders), ySlice[1]])} /> Max: {parseLoc(dimArrays[isFlat ? 0 : shapeLength-2]?.[ySlice[1] ? ySlice[1]-1 : yLength-1]?? null, dimUnits[isFlat ? 0 : shapeLength-2]?? null)}
Index: setYSlice([ySlice[0] , parseInt(e.target.value)])} - onBlur={e=>setYSlice([ySlice[0], HandleCustomSteps(e.target.value,chunkShape[1])])} + onBlur={e=>setYSlice([ySlice[0], HandleCustomSteps(e.target.value, isFlat ? chunkShape[0] : chunkShape[1], unrestrictedSliders)])} />
@@ -371,20 +396,20 @@ const MetaDataInfo = ({ meta, metadata, setShowMeta, setOpenVariables, popoverSi min={0} max={xLength} value={[xSlice[0] ? xSlice[0] : 0, xSlice[1] ? xSlice[1] : xLength]} - step={isFlat ? chunkShape[1] : chunkShape[2]} + step={unrestrictedSliders ? 1 : (isFlat ? chunkShape[1] : chunkShape[2])} onValueChange={(values: number[]) => setXSlice([values[0], values[1]] as [number, number | null])} />
Min: {parseLoc(dimArrays[isFlat ? 1 : shapeLength-1]?.[xSlice[0]]?? null, dimUnits[isFlat ? 1 : shapeLength-1]?? null)}
Index: setXSlice([parseInt(e.target.value), xSlice[1]])} - onBlur={e=>setXSlice([HandleCustomSteps(e.target.value,chunkShape[2]), xSlice[1]])} + onBlur={e=>setXSlice([HandleCustomSteps(e.target.value, isFlat ? chunkShape[1] : chunkShape[2], unrestrictedSliders), xSlice[1]])} /> Max: {parseLoc(dimArrays[isFlat ? 1 : shapeLength-1]?.[xSlice[1] ? xSlice[1]-1 : xLength-1]?? null, dimUnits[isFlat ? 1 : shapeLength-1]?? null)}
Index: setXSlice([xSlice[0] , parseInt(e.target.value)])} - onBlur={e=>setXSlice([xSlice[0], HandleCustomSteps(e.target.value,chunkShape[2])])} + onBlur={e=>setXSlice([xSlice[0], HandleCustomSteps(e.target.value, isFlat ? chunkShape[1] : chunkShape[2], unrestrictedSliders)])} />
diff --git a/src/components/zarr/GetArray.ts b/src/components/zarr/GetArray.ts index c1ded8fa..58d03c36 100644 --- a/src/components/zarr/GetArray.ts +++ b/src/components/zarr/GetArray.ts @@ -25,15 +25,55 @@ export async function GetArray(varOveride?: string) { if (dimIdx < 0) return { start: 0, end: 1, size: 0, chunkDim: 1 }; const dimSize = shape[dimIdx]; const chunkDim = chunkShape[dimIdx]; - const start = Math.floor(slice[0] / chunkDim); + const sliceStart = slice[0]; const sliceEnd = slice[1] ?? dimSize; - return { start, end: Math.ceil(sliceEnd / chunkDim), size: sliceEnd - slice[0], chunkDim }; + + // Calculate chunk range that intersects with slice + const startChunk = Math.floor(sliceStart / chunkDim); + const endChunk = Math.ceil(sliceEnd / chunkDim); + + return { + start: startChunk, + end: endChunk, + size: sliceEnd - sliceStart, + chunkDim, + sliceStart, + sliceEnd + }; }; const xDim = calcDim(xSlice, xDimIndex); const yDim = calcDim(ySlice, yDimIndex); const zDim = calcDim(zSlice, zDimIndex); + // Calculate actual data overlap for each chunk + const calculateChunkOverlap = (chunkCoord: number, dimIdx: number, sliceStart: number, sliceEnd: number, chunkDim: number) => { + const chunkStart = chunkCoord * chunkDim; + const chunkEnd = Math.min((chunkCoord + 1) * chunkDim, shape[dimIdx]); + + const overlapStart = Math.max(chunkStart, sliceStart); + const overlapEnd = Math.min(chunkEnd, sliceEnd); + + return Math.max(0, overlapEnd - overlapStart); + }; + + // Calculate priority score for chunk ordering (higher = more important) + const calculateChunkPriority = (z: number, y: number, x: number) => { + const xOverlap = calculateChunkOverlap(x, xDimIndex, xDim.sliceStart ?? 0, xDim.sliceEnd ?? xDim.chunkDim, xDim.chunkDim); + const yOverlap = calculateChunkOverlap(y, yDimIndex, yDim.sliceStart ?? 0, yDim.sliceEnd ?? yDim.chunkDim, yDim.chunkDim); + const zOverlap = hasZ ? calculateChunkOverlap(z, zDimIndex, zDim.sliceStart ?? 0, zDim.sliceEnd ?? zDim.chunkDim, zDim.chunkDim) : 1; + + // Priority based on total useful data volume (not density) + // This avoids penalizing chunks in sparse dimensions + const totalUsefulData = xOverlap * yOverlap * zOverlap; + + // Also consider chunk size (smaller chunks may download faster) + const chunkSize = xDim.chunkDim * yDim.chunkDim * (hasZ ? zDim.chunkDim : 1); + + // Combine factors: prioritize high data volume, then smaller chunks for faster downloads + return totalUsefulData * 1000 + (1 / chunkSize) * 100; + }; + let outputShape = hasZ ? [zDim.size, yDim.size, xDim.size] : [yDim.size, xDim.size]; if (coarsen) { outputShape = outputShape.map((dim, idx) => Math.floor(dim / (hasZ && idx === 0 ? kernelDepth : kernelSize))); @@ -47,95 +87,146 @@ export async function GetArray(varOveride?: string) { setArraySize(totalElements); setCurrentChunks({ x: [xDim.start, xDim.end], y: [yDim.start, yDim.end], z: [zDim.start, zDim.end] }); // These are used in GetCurrentArray() function + setStatus("Downloading..."); + setProgress(0); + const typedArray = new Float16Array(totalElements); let scalingFactor: number | null = null; const totalChunks = (zDim.end - zDim.start) * (yDim.end - yDim.start) * (xDim.end - xDim.start); - let iter = 1; + let processedChunks = 0; const rescaleIDs: string[] = []; - - setStatus("Downloading..."); - setProgress(0); - + const cacheBase = rank > 3 ? `${initStore}_${targetVariable}_${idx4D}` : `${initStore}_${targetVariable}`; + + // Collect all chunks that need fetching, sorted by priority + const chunksToFetch: Array<{ + chunkID: string; + cacheName: string; + coords: { z: number; y: number; x: number }; + priority: number; + dataDensity: number; + }> = []; + + // First pass: collect chunks and check cache for (let z = zDim.start; z < zDim.end; z++) { for (let y = yDim.start; y < yDim.end; y++) { for (let x = xDim.start; x < xDim.end; x++) { + // Skip chunks with zero data overlap + const xOverlap = calculateChunkOverlap(x, xDimIndex, xDim.sliceStart ?? 0, xDim.sliceEnd ?? xDim.chunkDim, xDim.chunkDim); + const yOverlap = calculateChunkOverlap(y, yDimIndex, yDim.sliceStart ?? 0, yDim.sliceEnd ?? yDim.chunkDim, yDim.chunkDim); + const zOverlap = hasZ ? calculateChunkOverlap(z, zDimIndex, zDim.sliceStart ?? 0, zDim.sliceEnd ?? zDim.chunkDim, zDim.chunkDim) : xDim.chunkDim; + + if (xOverlap === 0 || yOverlap === 0 || zOverlap === 0) { + continue; // Skip chunks that don't contain any requested data + } + const chunkID = `z${z}_y${y}_x${x}`; - const cacheBase = rank > 3 ? `${initStore}_${targetVariable}_${idx4D}` : `${initStore}_${targetVariable}`; const cacheName = `${cacheBase}_chunk_${chunkID}`; const cachedChunk = cache.get(cacheName); const isCacheValid = cachedChunk && cachedChunk.kernel.kernelSize === (coarsen ? kernelSize : undefined) && cachedChunk.kernel.kernelDepth === (coarsen ? kernelDepth : undefined); - if (isCacheValid) { - const chunkData = cachedChunk.compressed ? DecompressArray(cachedChunk.data) : cachedChunk.data.slice(); - if (hasZ) { - copyChunkToArray( - chunkData, - cachedChunk.shape, - cachedChunk.stride, - typedArray, - outputShape, - destStride as any, [z, y, x], - [zDim.start, yDim.start, xDim.start] - ) - } else { - copyChunkToArray2D( - chunkData, - cachedChunk.shape, - cachedChunk.stride, - typedArray, - outputShape, - destStride as any, [y, x], - [yDim.start, xDim.start]) - } - } else { - const raw = await fetcher.fetchChunk({ variable:targetVariable, rank, shape, chunkShape, x, y, z, xDimIndex, yDimIndex, zDimIndex, idx4D }); - - const rawData = Number.isFinite(fillValue) ? raw.data.map((v: number) => v === fillValue ? NaN : v) : raw.data; // Don't map if no fillvalue - - let [chunkF16, newScalingFactor] = ToFloat16(rawData, scalingFactor); - let thisShape = raw.shape; - let chunkStride = raw.stride; - - if (coarsen) { - chunkF16 = await Convolve(chunkF16, { shape: chunkShape, strides: chunkStride }, "Mean3D", { kernelSize, kernelDepth }) as Float16Array; - thisShape = thisShape.map((dim, idx) => Math.floor(dim / (idx === 0 ? kernelDepth : kernelSize))); - chunkF16 = coarsen3DArray(chunkF16, chunkShape, chunkStride as any, kernelSize, kernelDepth, thisShape.reduce((a, b) => a * b, 1)); - chunkStride = calculateStrides(thisShape); - } - - if (newScalingFactor != null && newScalingFactor !== scalingFactor) { - const delta = scalingFactor ? newScalingFactor - scalingFactor : newScalingFactor; - RescaleArray(typedArray, delta); - scalingFactor = newScalingFactor; - for (const id of rescaleIDs) { - const tempChunk = cache.get(`${cacheBase}_chunk_${id}`); - tempChunk.scaling = scalingFactor; - RescaleArray(tempChunk.data, delta); - cache.set(`${cacheBase}_chunk_${id}`, tempChunk); - } - } - - if (hasZ) { - copyChunkToArray(chunkF16, thisShape.slice(-3), chunkStride.slice(-3) as any, typedArray, outputShape, destStride as any, [z, y, x], [zDim.start, yDim.start, xDim.start]); - } else { - copyChunkToArray2D(chunkF16, thisShape, chunkStride as any, typedArray, outputShape, destStride as any, [y, x], [yDim.start, xDim.start]); - } - - cache.set(cacheName, { - data: compress ? CompressArray(chunkF16, 7) : chunkF16, - shape: chunkShape, stride: chunkStride, - scaling: scalingFactor, compressed: compress, coarsened: coarsen, - kernel: { kernelDepth: coarsen ? kernelDepth : undefined, kernelSize: coarsen ? kernelSize : undefined } + if (!isCacheValid) { + const priority = calculateChunkPriority(z, y, x); + const dataDensity = (xOverlap / xDim.chunkDim) * (yOverlap / yDim.chunkDim) * (hasZ ? zOverlap / zDim.chunkDim : 1); + + chunksToFetch.push({ + chunkID, + cacheName, + coords: { z, y, x }, + priority, + dataDensity }); - rescaleIDs.push(chunkID); } - setProgress(Math.round(iter++ / totalChunks * 100)); - } } + } } + + // Sort chunks by priority, but preserve natural disk-local chunk order when priorities are equal + chunksToFetch.sort((a, b) => { + const priorityDiff = b.priority - a.priority; + if (priorityDiff !== 0) return priorityDiff; + if (a.coords.z !== b.coords.z) return a.coords.z - b.coords.z; + if (a.coords.y !== b.coords.y) return a.coords.y - b.coords.y; + return a.coords.x - b.coords.x; + }); + + const totalFetchChunks = chunksToFetch.length; + setProgress(0); + + // Use a bounded batch size to balance parallelism and disk-local chunk reads + const baseBatchSize = Math.min(10, totalFetchChunks || 10); + let currentBatchSize = Math.min(baseBatchSize, totalFetchChunks); + + for (let i = 0; i < totalFetchChunks; i += currentBatchSize) { + const remainingChunks = totalFetchChunks - i; + currentBatchSize = Math.min(baseBatchSize, remainingChunks); + + const batch = chunksToFetch.slice(i, i + currentBatchSize); + + // Start all fetches in this batch + const fetchPromises = batch.map(chunk => { + const { z, y, x } = chunk.coords; + return fetcher.fetchChunk({ variable: targetVariable, rank, shape, chunkShape, x, y, z, xDimIndex, yDimIndex, zDimIndex, idx4D }); + }); + + // Wait for all in batch to complete + const results = await Promise.all(fetchPromises); + + // Process results in order + for (let j = 0; j < batch.length; j++) { + const chunk = batch[j]; + const raw = results[j]; + const { z, y, x } = chunk.coords; + + const rawData = Number.isFinite(fillValue) ? raw.data.map((v: number) => v === fillValue ? NaN : v) : raw.data; + + const [tempChunkF16, newScalingFactor] = ToFloat16(rawData, scalingFactor); + let chunkF16 = tempChunkF16; + let thisShape = raw.shape; + let chunkStride = raw.stride; + + if (coarsen) { + chunkF16 = await Convolve(chunkF16, { shape: chunkShape, strides: chunkStride }, "Mean3D", { kernelSize, kernelDepth }) as Float16Array; + thisShape = thisShape.map((dim, idx) => Math.floor(dim / (idx === 0 ? kernelDepth : kernelSize))); + chunkF16 = coarsen3DArray(chunkF16, chunkShape as [number, number, number], chunkStride.slice(-3) as [number, number, number], kernelSize, kernelDepth, thisShape.reduce((a, b) => a * b, 1)); + chunkStride = calculateStrides(thisShape); + } + + if (newScalingFactor != null && newScalingFactor !== scalingFactor) { + const delta = scalingFactor ? newScalingFactor - scalingFactor : newScalingFactor; + RescaleArray(typedArray, delta); + scalingFactor = newScalingFactor; + for (const id of rescaleIDs) { + const tempChunk = cache.get(`${cacheBase}_chunk_${id}`); + tempChunk.scaling = scalingFactor; + RescaleArray(tempChunk.data, delta); + cache.set(`${cacheBase}_chunk_${id}`, tempChunk); + } + } + + if (hasZ) { + copyChunkToArray(chunkF16, thisShape.slice(-3), chunkStride.slice(-3), typedArray, outputShape, destStride, [z, y, x], [zDim.start, yDim.start, xDim.start]); + } else { + copyChunkToArray2D(chunkF16, thisShape, chunkStride, typedArray, outputShape, destStride, [y, x], [yDim.start, xDim.start]); + } + + cache.set(chunk.cacheName, { + data: compress ? CompressArray(chunkF16, 7) : chunkF16, + shape: chunkShape, stride: chunkStride, + scaling: scalingFactor, compressed: compress, coarsened: coarsen, + kernel: { kernelDepth: coarsen ? kernelDepth : undefined, kernelSize: coarsen ? kernelSize : undefined } + }); + rescaleIDs.push(chunk.chunkID); + + processedChunks += 1; + setProgress(Math.round(processedChunks / totalChunks * 100)); + } + } + + setProgress(100); + setStatus(null); return { data: typedArray, shape: outputShape, dtype, scalingFactor }; } \ No newline at end of file diff --git a/src/components/zarr/NCGetters.ts b/src/components/zarr/NCGetters.ts index 1e0f9e2b..84c5d697 100644 --- a/src/components/zarr/NCGetters.ts +++ b/src/components/zarr/NCGetters.ts @@ -47,7 +47,7 @@ export async function GetNCMetadata(thisVariable? : string){ } export async function GetNCArray(variable: string){ - const {idx4D, initStore, setProgress, setStrides, setStatus} = useGlobalStore.getState(); + const {idx4D, initStore, setProgress, setStrides} = useGlobalStore.getState(); const {compress, xSlice, ySlice, zSlice, ncModule, coarsen, kernelDepth, kernelSize, setCurrentChunks, setArraySize} = useZarrStore.getState() const {cache} = useCacheStore.getState(); const varInfo = await ncModule.getVariableInfo(variable) @@ -86,21 +86,50 @@ export async function GetNCArray(variable: string){ //---- Dimension Indices to Grab ----// const calcDim = (slice: [number, number | null], dimIdx: number) => { // Return an empty array if no zIdx - if (dimIdx < 0) return { start: 0, end: 1, size: 0, chunkDim: 1 }; + if (dimIdx < 0) return { start: 0, end: 1, size: 0, chunkDim: 1, sliceStart: 0, sliceEnd: 1 }; const dimSize = shape[dimIdx]; const chunkDim = chunkShape[dimIdx]; - const start = Math.floor(slice[0] / chunkDim); - const sliceEnd = slice[1] ?? dimSize; + const sliceStart = slice[0]; + const sliceEnd = slice[1] ?? dimSize; + const start = Math.floor(sliceStart / chunkDim); const end = Math.ceil(sliceEnd / chunkDim); - const size = sliceEnd - slice[0]; + const size = sliceEnd - sliceStart; //Chunkdim is the shape of the chunk at that index - return { start, end, size, chunkDim }; + return { start, end, size, chunkDim, sliceStart, sliceEnd }; }; const xDim = calcDim(xSlice, xDimIndex); const yDim = calcDim(ySlice, yDimIndex); const zDim = calcDim(zSlice, zDimIndex); + // Calculate actual data overlap for each chunk + const calculateChunkOverlap = (chunkCoord: number, dimIdx: number, sliceStart: number, sliceEnd: number, chunkDim: number) => { + const chunkStart = chunkCoord * chunkDim; + const chunkEnd = Math.min((chunkCoord + 1) * chunkDim, shape[dimIdx]); + + const overlapStart = Math.max(chunkStart, sliceStart); + const overlapEnd = Math.min(chunkEnd, sliceEnd); + + return Math.max(0, overlapEnd - overlapStart); + }; + + // Calculate priority score for chunk ordering (higher = more important) + const calculateChunkPriority = (z: number, y: number, x: number) => { + const xOverlap = calculateChunkOverlap(x, xDimIndex, xDim.sliceStart ?? 0, xDim.sliceEnd ?? xDim.chunkDim, xDim.chunkDim); + const yOverlap = calculateChunkOverlap(y, yDimIndex, yDim.sliceStart ?? 0, yDim.sliceEnd ?? yDim.chunkDim, yDim.chunkDim); + const zOverlap = hasZ ? calculateChunkOverlap(z, zDimIndex, zDim.sliceStart ?? 0, zDim.sliceEnd ?? zDim.chunkDim, zDim.chunkDim) : 1; + + // Priority based on total useful data volume (not density) + // This avoids penalizing chunks in sparse dimensions + const totalUsefulData = xOverlap * yOverlap * zOverlap; + + // Also consider chunk size (smaller chunks may download faster) + const chunkSize = xDim.chunkDim * yDim.chunkDim * (hasZ ? zDim.chunkDim : 1); + + // Combine factors: prioritize high data volume, then smaller chunks for faster downloads + return totalUsefulData * 1000 + (1 / chunkSize) * 100; + }; + // Setup Output Array let outputShape = hasZ ? [zDim.size, yDim.size, xDim.size] @@ -130,26 +159,43 @@ export async function GetNCArray(variable: string){ // State for the loop let scalingFactor: number | null = null; const totalChunksToLoad = (zDim.end - zDim.start) * (yDim.end - yDim.start) * (xDim.end - xDim.start); - let iter = 1; // For progress bar + let processedChunks = 0; const rescaleIDs: string[] = [] // These are the downloaded chunks that need to be rescaled + const cacheBase = rank > 3 + ? `${initStore}_${variable}_${idx4D}` + : `${initStore}_${variable}` + + // Collect all chunks that need fetching, sorted by priority + const chunksToFetch: Array<{ + chunkID: string; + cacheName: string; + coords: { z: number; y: number; x: number }; + priority: number; + dataDensity: number; + }> = []; + + // First pass: collect chunks and check cache + for (let z = zDim.start; z < zDim.end; z++) { + for (let y = yDim.start; y < yDim.end; y++) { + for (let x = xDim.start; x < xDim.end; x++) { + // Skip chunks with zero data overlap + const xOverlap = calculateChunkOverlap(x, xDimIndex, xDim.sliceStart, xDim.sliceEnd, xDim.chunkDim); + const yOverlap = calculateChunkOverlap(y, yDimIndex, yDim.sliceStart, yDim.sliceEnd, yDim.chunkDim); + const zOverlap = hasZ ? calculateChunkOverlap(z, zDimIndex, zDim.sliceStart, zDim.sliceEnd, zDim.chunkDim) : xDim.chunkDim; + + if (xOverlap === 0 || yOverlap === 0 || zOverlap === 0) { + continue; // Skip chunks that don't contain any requested data + } - setStatus("Downloading..."); - setProgress(0); - for (let z= zDim.start ; z < zDim.end ; z++){ // Iterate through chunks we need - for (let y= yDim.start ; y < yDim.end ; y++){ - for (let x= xDim.start ; x < xDim.end ; x++){ const chunkID = `z${z}_y${y}_x${x}` // Unique ID for each chunk - const cacheBase = rank > 3 - ? `${initStore}_${variable}_${idx4D}` - : `${initStore}_${variable}` const cacheName = `${cacheBase}_chunk_${chunkID}` const cachedChunk = cache.get(cacheName); - const isCacheValid = cachedChunk && + const isCacheValid = cachedChunk && cachedChunk.kernel.kernelSize === (coarsen ? kernelSize : undefined) && // If the data is coarsened. Make sure it's the same as current coarsen. Otherwise refetch - cachedChunk.kernel.kernelDepth === (coarsen ? kernelSize : undefined) ; - if (isCacheValid){ - const chunkData = cachedChunk.compressed ? DecompressArray(cachedChunk.data) : cachedChunk.data.slice() // Decompress if needed. Gemini thinks the .slice() helps with garbage collector as it doesn't maintain a reference to the original array + cachedChunk.kernel.kernelDepth === (coarsen ? kernelSize : undefined); + if (isCacheValid) { + const chunkData = cachedChunk.compressed ? DecompressArray(cachedChunk.data) : cachedChunk.data.slice(); copyChunkToArray( chunkData, cachedChunk.shape, @@ -157,122 +203,191 @@ export async function GetNCArray(variable: string){ typedArray, outputShape, destStride as [number, number, number], - [z,y,x], - [zDim.start,yDim.start,xDim.start], - ) - setProgress(Math.round(iter/totalChunksToLoad*100)) // Progress Bar - iter ++; + [z, y, x], + [zDim.start, yDim.start, xDim.start], + ); + processedChunks += 1; + setProgress(Math.round(processedChunks / totalChunksToLoad * 100)); + } else { + const priority = calculateChunkPriority(z, y, x); + const dataDensity = (xOverlap / xDim.chunkDim) * (yOverlap / yDim.chunkDim) * (hasZ ? zOverlap / zDim.chunkDim : 1); + + chunksToFetch.push({ + chunkID, + cacheName, + coords: { z, y, x }, + priority, + dataDensity + }); } - else{ - const getStartsAndCounts = () => { - const starts = new Array(rank).fill(0); - const counts = new Array(rank).fill(1); - if (rank > 3) { //When rank is 4 or 5. The first will always be depth. In the case of 5 that will only be if last dimension is vector variable. This case not handled yet - starts[0] = idx4D; - counts[0] = 1; - } - starts[xDimIndex] = x * chunkShape[xDimIndex]; - counts[xDimIndex] = Math.min(chunkShape[xDimIndex], shape[xDimIndex] - starts[xDimIndex]); + } + } + } - starts[yDimIndex] = y * chunkShape[yDimIndex]; - counts[yDimIndex] = Math.min(chunkShape[yDimIndex], shape[yDimIndex] - starts[yDimIndex]); + // Sort chunks by priority, then preserve contiguous disk order when priorities tie + chunksToFetch.sort((a, b) => { + const priorityDiff = b.priority - a.priority; + if (priorityDiff !== 0) return priorityDiff; + if (a.coords.z !== b.coords.z) return a.coords.z - b.coords.z; + if (a.coords.y !== b.coords.y) return a.coords.y - b.coords.y; + return a.coords.x - b.coords.x; + }); - if (zDimIndex >= 0) { - starts[zDimIndex] = z * chunkShape[zDimIndex]; - counts[zDimIndex] = Math.min(chunkShape[zDimIndex], shape[zDimIndex] - starts[zDimIndex]); - } - return {starts, counts} - } - const { starts, counts } = getStartsAndCounts(); - let chunkArray = await ncModule.getSlicedVariableArray(variable, starts, counts) - const chunkType = chunkArray.constructor.name - const isInt = chunkType.includes("int") - let thisShape = counts - let chunkStride = calculateStrides(thisShape) - const filterValues = (array: TypedArray) =>{ - for (let i = 0; i < array.length; i++){ - if (array[i] === fillValue && !isInt) array[i] = NaN - if (validRange){ - if (isInt){ - if (array[i] < validRange.min) array[i] = validRange.min - if (array[i] > validRange.max) array[i] = validRange.max - } else{ - if (array[i] < validRange.min || array[i] > validRange.max) array[i] = NaN - } - - } - } - } - const preScale = (array: TypedArray, scaler: number) =>{ - const tempArray = new Float32Array(array.length); - for (let i = 0; i < array.length; i++){ - tempArray[i] = array[i] * scaler; - } - return tempArray; - } - filterValues(chunkArray) - if (preScaling) chunkArray = preScale(chunkArray, preScaling) - let [chunkF16, newScalingFactor] = ToFloat16(chunkArray, scalingFactor) - if (coarsen){ - chunkF16 = await Convolve(chunkF16, {shape:chunkShape, strides:chunkStride}, "Mean3D", {kernelSize, kernelDepth}) as Float16Array - thisShape = thisShape.map((dim: number, idx: number) => Math.floor(dim / (idx === 0 ? kernelDepth : kernelSize))) - const newSize = thisShape.reduce((a: number, b: number) => a*b, 1) - chunkF16 = coarsen3DArray(chunkF16, chunkShape, chunkStride as [number, number, number], kernelSize, kernelDepth, newSize) - chunkStride = calculateStrides(thisShape) - } - if (newScalingFactor != null - && newScalingFactor != scalingFactor){ // If the scalingFactor has changed, need to rescale main array. Not worried about shrinking values at the moment. - const thisScaling = scalingFactor ? newScalingFactor - scalingFactor : newScalingFactor - RescaleArray(typedArray, thisScaling) - scalingFactor = newScalingFactor - for (const id of rescaleIDs){ // Set new scalingFactor on the chunks - const tempName = `${cacheBase}_chunk_${id}` - const tempChunk = cache.get(tempName) - tempChunk.scaling = scalingFactor - RescaleArray(tempChunk.data, thisScaling) - cache.set(tempName, tempChunk) - } - } - if (hasZ)copyChunkToArray( - chunkF16, - chunkShape.slice(-3), - chunkStride.slice(-3) as [number, number, number], - typedArray, - outputShape, - destStride as [number, number, number], - [z,y,x], - [zDim.start,yDim.start,xDim.start], - ) - else copyChunkToArray2D( - chunkF16, - chunkShape, - chunkStride as [number, number], - typedArray, - outputShape, - destStride as [number, number], - [y,x], - [yDim.start,xDim.start], - ) - const cacheChunk = { - data: compress ? CompressArray(chunkF16, 7) : chunkF16, - shape: chunkShape, - stride: chunkStride, - scaling: scalingFactor, - compressed: compress, - coarsened: coarsen, - kernel: { - kernelDepth: coarsen ? kernelDepth : undefined, - kernelSize: coarsen ? kernelSize : undefined + const totalFetchChunks = chunksToFetch.length; + setProgress(0); + + // Use a bounded batch size to balance parallelism and contiguous chunk reads + const baseBatchSize = Math.min(10, totalFetchChunks || 10); + let currentBatchSize = Math.min(baseBatchSize, totalFetchChunks); + + for (let i = 0; i < totalFetchChunks; i += currentBatchSize) { + // Adjust batch size based on remaining chunks + const remainingChunks = totalFetchChunks - i; + currentBatchSize = Math.min(baseBatchSize, remainingChunks); + + // For high-priority chunks (first 20%), use smaller batches for better responsiveness + if (i < chunksToFetch.length * 0.2) { + currentBatchSize = Math.min(5, currentBatchSize); + } + + const batch = chunksToFetch.slice(i, i + currentBatchSize); + + // Start all fetches in this batch + const fetchPromises = batch.map(chunk => { + const { z, y, x } = chunk.coords; + const starts = new Array(rank).fill(0); + const counts = new Array(rank).fill(1); + if (rank > 3) { //When rank is 4 or 5. The first will always be depth. In the case of 5 that will only be if last dimension is vector variable. This case not handled yet + starts[0] = idx4D; + counts[0] = 1; + } + starts[xDimIndex] = x * chunkShape[xDimIndex]; + counts[xDimIndex] = Math.min(chunkShape[xDimIndex], shape[xDimIndex] - starts[xDimIndex]); + + starts[yDimIndex] = y * chunkShape[yDimIndex]; + counts[yDimIndex] = Math.min(chunkShape[yDimIndex], shape[yDimIndex] - starts[yDimIndex]); + + if (zDimIndex >= 0) { + starts[zDimIndex] = z * chunkShape[zDimIndex]; + counts[zDimIndex] = Math.min(chunkShape[zDimIndex], shape[zDimIndex] - starts[zDimIndex]); + } + return ncModule.getSlicedVariableArray(variable, starts, counts); + }); + + // Wait for all in batch to complete + const results = await Promise.all(fetchPromises); + + // Process results in order + for (let j = 0; j < batch.length; j++) { + const chunk = batch[j]; + const { z, y, x } = chunk.coords; + let chunkArray = results[j]; + + const chunkType = chunkArray.constructor.name + const isInt = chunkType.includes("int") + let thisShape = [1, 1, 1]; // Will be set properly below + let chunkStride = calculateStrides(thisShape) + + // Recalculate shape for this specific chunk + const starts = new Array(rank).fill(0); + const counts = new Array(rank).fill(1); + if (rank > 3) { + starts[0] = idx4D; + counts[0] = 1; + } + starts[xDimIndex] = x * chunkShape[xDimIndex]; + counts[xDimIndex] = Math.min(chunkShape[xDimIndex], shape[xDimIndex] - starts[xDimIndex]); + starts[yDimIndex] = y * chunkShape[yDimIndex]; + counts[yDimIndex] = Math.min(chunkShape[yDimIndex], shape[yDimIndex] - starts[yDimIndex]); + if (zDimIndex >= 0) { + starts[zDimIndex] = z * chunkShape[zDimIndex]; + counts[zDimIndex] = Math.min(chunkShape[zDimIndex], shape[zDimIndex] - starts[zDimIndex]); + } + thisShape = counts; + chunkStride = calculateStrides(thisShape); + + const filterValues = (array: TypedArray) =>{ + for (let i = 0; i < array.length; i++){ + if (array[i] === fillValue && !isInt) array[i] = NaN + if (validRange){ + if (isInt){ + if (array[i] < validRange.min) array[i] = validRange.min + if (array[i] > validRange.max) array[i] = validRange.max + } else{ + if (array[i] < validRange.min || array[i] > validRange.max) array[i] = NaN } + } - cache.set(cacheName,cacheChunk) - setProgress(Math.round(iter/totalChunksToLoad*100)) // Progress Bar - iter ++; - rescaleIDs.push(chunkID) } } + const preScale = (array: TypedArray, scaler: number) =>{ + const tempArray = new Float32Array(array.length); + for (let i = 0; i < array.length; i++){ + tempArray[i] = array[i] * scaler; + } + return tempArray; + } + filterValues(chunkArray) + if (preScaling) chunkArray = preScale(chunkArray, preScaling) + let [chunkF16, newScalingFactor] = ToFloat16(chunkArray, scalingFactor) + if (coarsen){ + chunkF16 = await Convolve(chunkF16, {shape:chunkShape, strides:chunkStride}, "Mean3D", {kernelSize, kernelDepth}) as Float16Array + thisShape = thisShape.map((dim: number, idx: number) => Math.floor(dim / (idx === 0 ? kernelDepth : kernelSize))) + const newSize = thisShape.reduce((a: number, b: number) => a*b, 1) + chunkF16 = coarsen3DArray(chunkF16, chunkShape, chunkStride as [number, number, number], kernelSize, kernelDepth, newSize) + chunkStride = calculateStrides(thisShape) + } + if (newScalingFactor != null + && newScalingFactor != scalingFactor){ // If the scalingFactor has changed, need to rescale main array. Not worried about shrinking values at the moment. + const thisScaling = scalingFactor ? newScalingFactor - scalingFactor : newScalingFactor + RescaleArray(typedArray, thisScaling) + scalingFactor = newScalingFactor + for (const id of rescaleIDs){ // Set new scalingFactor on the chunks + const tempName = `${cacheBase}_chunk_${id}` + const tempChunk = cache.get(tempName) + tempChunk.scaling = scalingFactor + RescaleArray(tempChunk.data, thisScaling) + cache.set(tempName, tempChunk) + } + } + if (hasZ)copyChunkToArray( + chunkF16, + chunkShape.slice(-3), + chunkStride.slice(-3) as [number, number, number], + typedArray, + outputShape, + destStride as [number, number, number], + [z,y,x], + [zDim.start,yDim.start,xDim.start], + ) + else copyChunkToArray2D( + chunkF16, chunkShape, + chunkStride as [number, number], + typedArray, + outputShape, + destStride as [number, number], + [y,x], + [yDim.start,xDim.start], + ) + const cacheChunk = { + data: compress ? CompressArray(chunkF16, 7) : chunkF16, + shape: chunkShape, + stride: chunkStride, + scaling: scalingFactor, + compressed: compress, + coarsened: coarsen, + kernel: { + kernelDepth: coarsen ? kernelDepth : undefined, + kernelSize: coarsen ? kernelSize : undefined + } + } + cache.set(chunk.cacheName, cacheChunk) + processedChunks += 1; + setProgress(Math.round(processedChunks / totalChunksToLoad * 100)); + rescaleIDs.push(chunk.chunkID) } } + return { data: typedArray, shape: outputShape, diff --git a/src/utils/MemoryLRU.ts b/src/utils/MemoryLRU.ts index e318a991..63144d59 100644 --- a/src/utils/MemoryLRU.ts +++ b/src/utils/MemoryLRU.ts @@ -26,6 +26,7 @@ function defaultSizeCalculator(value: T): number { interface MemoryLRUOptions { maxSize: number; sizeCalculator?: SizeCalculator; + onChange?: () => void; } export class MemoryLRU { @@ -34,11 +35,13 @@ export class MemoryLRU { public totalSize = 0; private maxSize: number; private readonly sizeCalculator: SizeCalculator; + private readonly onChange?: () => void; private sizes = new Map(); constructor(options: MemoryLRUOptions) { this.maxSize = options.maxSize; this.sizeCalculator = options.sizeCalculator ?? defaultSizeCalculator; + this.onChange = options.onChange; } get(key: K): V | undefined { @@ -74,6 +77,8 @@ export class MemoryLRU { this.sizes.delete(oldestKey); this.totalSize -= oldestSize; } + + this.onChange?.(); } resize(newSize: number): void { @@ -103,6 +108,7 @@ export class MemoryLRU { this.cache.delete(key); this.sizes.delete(key); this.order = this.order.filter(k => k !== key); + this.onChange?.(); return true; } @@ -111,6 +117,7 @@ export class MemoryLRU { this.sizes.clear(); this.order = []; this.totalSize = 0; + this.onChange?.(); } get size(): number {