diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index 5d839d8..839abfc 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -1,24 +1,20 @@ import { NoSuchKey } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { StorageProvider, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; +import { StorageProvider, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; import { inject, injectable } from 'tsyringe'; import type { ConfigType } from '@common/config'; import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; import { mergeFailures, summarizeDeleteFailures, type DeleteFailure, type StorageProviders } from '@src/cleaner/storageProviders'; import { RecoverableError, UnrecoverableError, describeError } from '../errors'; import { ResolvedStorageProvider } from '../storageProviders/iStorageProvider'; -import { validateSchema } from '../utils'; +import { resolveTileKeyGenerator, validateSchema } from '../utils'; import type { TaskContext } from './strategyFactory'; import type { ITaskStrategy } from './taskStrategy'; const NOT_FOUND_REASONS = new Set([NoSuchKey.name, 'ENOENT']); -/** - * The params shapes this strategy can act on today. Redis tiles deletion is not implemented - * yet (MAPCO-11261): its params carry a key prefix instead of a tiles path, so there are no - * tile paths to generate. - */ +/** Redis tiles deletion is not implemented yet (MAPCO-11263). */ type SupportedTilesDeletionParams = Exclude; @injectable() @@ -63,7 +59,7 @@ export class TilesDeletionStrategy implements ITaskStrategy totalTiles, }); - const failures = await this.deleteAllTiles(provider, storageTarget, params, totalTiles); + const failures = await this.deleteTiles(provider, storageTarget, params, totalTiles); this.reportOutcome(failures, totalTiles); } @@ -125,7 +121,7 @@ export class TilesDeletionStrategy implements ITaskStrategy return { provider: storageProvider, storageTarget }; } - private async deleteAllTiles( + private async deleteTiles( provider: ResolvedStorageProvider, storageTarget: string, params: SupportedTilesDeletionParams, @@ -137,8 +133,8 @@ export class TilesDeletionStrategy implements ITaskStrategy let batch: string[] = []; let processedTiles = 0; - for (const tilePath of this.generateTilePaths(params)) { - batch.push(tilePath); + for (const tileKey of this.generateTileKeys(params)) { + batch.push(tileKey); if (batch.length === this.batchSize) { pendingBatches.push(batch); batch = []; @@ -210,17 +206,11 @@ export class TilesDeletionStrategy implements ITaskStrategy return params.ranges.reduce((sum, r) => sum + (r.maxX - r.minX + 1) * (r.maxY - r.minY + 1), 0); } - private *generateTilePaths(params: SupportedTilesDeletionParams): Generator { - for (const range of params.ranges) { - yield* this.generateRangePaths(range, params.tilesRelativePath, params.fileExtension); - } - } + private *generateTileKeys(params: SupportedTilesDeletionParams): Generator { + const toKeys = resolveTileKeyGenerator(params); - private *generateRangePaths(range: TileRange, tilesRelativePath: string, fileExtension: string): Generator { - for (let x = range.minX; x <= range.maxX; x++) { - for (let y = range.minY; y <= range.maxY; y++) { - yield `${tilesRelativePath}/${range.zoom}/${x}/${y}.${fileExtension}`; - } + for (const range of params.ranges) { + yield* toKeys(range); } } } diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index a7eead6..7a70a30 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,5 +1,13 @@ export { getChunk } from './chunk'; export { assertCanDeleteFromFolder } from './fs'; export { buildPollingPairs } from './pairBuilder'; -export { isPathWithinAllowedSubPaths, normalizeFolderPath, resolveAbsolutePath } from './path'; +export { + generateRangeKeys, + generateRedisTileKeys, + generateTilePaths, + isPathWithinAllowedSubPaths, + normalizeFolderPath, + resolveAbsolutePath, +} from './path'; +export { resolveTileKeyGenerator } from './tileKeys'; export { validateSchema } from './validationHelper'; diff --git a/src/cleaner/utils/path.ts b/src/cleaner/utils/path.ts index b59df75..38801a0 100644 --- a/src/cleaner/utils/path.ts +++ b/src/cleaner/utils/path.ts @@ -1,9 +1,30 @@ import { join, resolve, sep } from 'node:path/posix'; +import type { TileRange } from '@map-colonies/raster-shared'; export const normalizeFolderPath = (path: string): string => { return path.endsWith(sep) ? path : `${path}${sep}`; }; +/** + * Walks every tile in a range and yields whatever `formatKey` makes of it, so the traversal + * lives in one place and each provider only supplies its own key shape. + */ +export function* generateRangeKeys(range: TileRange, formatKey: (zoom: number, x: number, y: number) => string): Generator { + for (let x = range.minX; x <= range.maxX; x++) { + for (let y = range.minY; y <= range.maxY; y++) { + yield formatKey(range.zoom, x, y); + } + } +} + +/** Tile paths for the S3 and FS providers, which address a tile identically. */ +export const generateTilePaths = (range: TileRange, tilesRelativePath: string, fileExtension: string): Generator => + generateRangeKeys(range, (zoom, x, y) => `${tilesRelativePath}/${zoom}/${x}/${y}.${fileExtension}`); + +/** Redis tile keys in mapproxy's format */ +export const generateRedisTileKeys = (range: TileRange, prefix: string): Generator => + generateRangeKeys(range, (zoom, x, y) => `${prefix}-${zoom}-${x}-${y}`); + /** * Resolves a file system path to an absolute path. * Ensures the path is resolved as an absolute path and properly formatted diff --git a/src/cleaner/utils/tileKeys.ts b/src/cleaner/utils/tileKeys.ts new file mode 100644 index 0000000..03671d9 --- /dev/null +++ b/src/cleaner/utils/tileKeys.ts @@ -0,0 +1,16 @@ +import { StorageProvider, type TileRange, type TilesDeletionParams } from '@map-colonies/raster-shared'; +import { generateRedisTileKeys, generateTilePaths } from './path'; + +/** + * Picks the tile-key format for a deletion request. Switching on the discriminant (rather than + * indexing a `Record`) is what narrows `params` per provider without a cast. + */ +export function resolveTileKeyGenerator(params: TilesDeletionParams): (range: TileRange) => Generator { + switch (params.storageProvider) { + case StorageProvider.S3: + case StorageProvider.FS: + return (range) => generateTilePaths(range, params.tilesRelativePath, params.fileExtension); + case StorageProvider.REDIS: + return (range) => generateRedisTileKeys(range, params.prefix); + } +} diff --git a/tests/utils/path.spec.ts b/tests/utils/path.spec.ts index d745914..5f7f847 100644 --- a/tests/utils/path.spec.ts +++ b/tests/utils/path.spec.ts @@ -1,5 +1,12 @@ +import type { TileRange } from '@map-colonies/raster-shared'; import { describe, expect, it } from 'vitest'; -import { isPathWithinAllowedSubPaths, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils/path'; +import { + generateRedisTileKeys, + generateTilePaths, + isPathWithinAllowedSubPaths, + normalizeFolderPath, + resolveAbsolutePath, +} from '@src/cleaner/utils/path'; const BASE_PATH = '/data'; const ALLOWED_SUB_PATHS = ['artifacts/tiles', 'artifacts/gpkgs']; @@ -65,4 +72,57 @@ describe('path', () => { expect(isPathWithinAllowedSubPaths({ relativePath: 'artifacts/tiles/layer', basePath: BASE_PATH, allowedSubPaths: [] })).toBe(false); }); }); + + describe('#generateTilePaths', () => { + it('should yield a path per tile, ordered x-major then y', () => { + const range: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 }; + + expect([...generateTilePaths(range, 'layer/v1', 'png')]).toEqual([ + 'layer/v1/3/1/5.png', + 'layer/v1/3/1/6.png', + 'layer/v1/3/2/5.png', + 'layer/v1/3/2/6.png', + ]); + }); + + it('should yield a single path for a range covering one tile', () => { + const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 }; + + expect([...generateTilePaths(range, 'layer/v1', 'jpeg')]).toEqual(['layer/v1/0/0/0.jpeg']); + }); + }); + + describe('#generateRedisTileKeys', () => { + it('should yield dash-separated keys with zoom first and no file extension', () => { + const range: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 }; + + expect([...generateRedisTileKeys(range, 'myLayer-redis_WorldCRS84')]).toEqual([ + 'myLayer-redis_WorldCRS84-3-1-5', + 'myLayer-redis_WorldCRS84-3-1-6', + 'myLayer-redis_WorldCRS84-3-2-5', + 'myLayer-redis_WorldCRS84-3-2-6', + ]); + }); + + // Captured off a live MapProxy command stream (DISCOVERY.md). A wrong key deletes nothing + // and still reports success, so it is pinned exactly. + it('should reproduce the key format captured from MapProxy', () => { + const range: TileRange = { zoom: 12, minX: 4892, maxX: 4892, minY: 2784, maxY: 2784 }; + + expect([...generateRedisTileKeys(range, 'benchmark')]).toEqual(['benchmark-12-4892-2784']); + }); + + // Production uses mapproxy-api's `{cacheName}_{gridName}` fallback, not the layer id. + it('should treat the prefix as an opaque string', () => { + const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 }; + + expect([...generateRedisTileKeys(range, 'myLayer-redis_WorldCRS84')]).toEqual(['myLayer-redis_WorldCRS84-0-0-0']); + }); + + it('should yield a single key for a range covering one tile', () => { + const range: TileRange = { zoom: 21, minX: 7, maxX: 7, minY: 9, maxY: 9 }; + + expect([...generateRedisTileKeys(range, 'p')]).toEqual(['p-21-7-9']); + }); + }); }); diff --git a/tests/utils/tileKeys.spec.ts b/tests/utils/tileKeys.spec.ts new file mode 100644 index 0000000..80a8001 --- /dev/null +++ b/tests/utils/tileKeys.spec.ts @@ -0,0 +1,98 @@ +import { faker } from '@faker-js/faker'; +import type { FsTilesDeletionParams, RedisTilesDeletionParams, S3TilesDeletionParams, TileRange } from '@map-colonies/raster-shared'; +import { describe, expect, it } from 'vitest'; +import { resolveTileKeyGenerator } from '@src/cleaner/utils/tileKeys'; + +const RANGE: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 }; + +/** + * Pins the key shape per storage provider. S3 and FS must stay byte-identical — the strategy + * relies on them sharing one key family — while Redis uses mapproxy's flat key format. + */ +describe('resolveTileKeyGenerator', () => { + const relativePath = `${faker.string.uuid()}/${faker.string.uuid()}`; + + describe('S3', () => { + it('should yield a path per tile, ordered x-major then y', () => { + const params: S3TilesDeletionParams = { + storageProvider: 'S3', + bucket: 'b', + tilesRelativePath: relativePath, + fileExtension: 'png', + ranges: [RANGE], + }; + + expect([...resolveTileKeyGenerator(params)(RANGE)]).toEqual([ + `${relativePath}/3/1/5.png`, + `${relativePath}/3/1/6.png`, + `${relativePath}/3/2/5.png`, + `${relativePath}/3/2/6.png`, + ]); + }); + + it('should use the requested file extension', () => { + const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 }; + const params: S3TilesDeletionParams = { + storageProvider: 'S3', + bucket: 'b', + tilesRelativePath: relativePath, + fileExtension: 'jpeg', + ranges: [range], + }; + + expect([...resolveTileKeyGenerator(params)(range)]).toEqual([`${relativePath}/0/0/0.jpeg`]); + }); + }); + + describe('FS', () => { + it('should yield the same path shape as S3', () => { + const fsParams: FsTilesDeletionParams = { + storageProvider: 'FS', + subPath: 'artifacts/tiles', + tilesRelativePath: relativePath, + fileExtension: 'jpeg', + ranges: [RANGE], + }; + const s3Params: S3TilesDeletionParams = { + storageProvider: 'S3', + bucket: 'b', + tilesRelativePath: relativePath, + fileExtension: 'jpeg', + ranges: [RANGE], + }; + + expect([...resolveTileKeyGenerator(fsParams)(RANGE)]).toEqual([...resolveTileKeyGenerator(s3Params)(RANGE)]); + }); + + it('should offset x and y when the range does not start at zero', () => { + const range: TileRange = { zoom: 7, minX: 3, maxX: 4, minY: 8, maxY: 9 }; + const params: FsTilesDeletionParams = { + storageProvider: 'FS', + subPath: 'artifacts/tiles', + tilesRelativePath: relativePath, + fileExtension: 'png', + ranges: [range], + }; + + expect([...resolveTileKeyGenerator(params)(range)]).toEqual([ + `${relativePath}/7/3/8.png`, + `${relativePath}/7/3/9.png`, + `${relativePath}/7/4/8.png`, + `${relativePath}/7/4/9.png`, + ]); + }); + }); + + describe('REDIS', () => { + it('should yield flat prefixed keys rather than paths', () => { + const prefix = 'myLayer-redis_WorldCRS84'; + const params: RedisTilesDeletionParams = { + storageProvider: 'REDIS', + prefix, + ranges: [RANGE], + }; + + expect([...resolveTileKeyGenerator(params)(RANGE)]).toEqual([`${prefix}-3-1-5`, `${prefix}-3-1-6`, `${prefix}-3-2-5`, `${prefix}-3-2-6`]); + }); + }); +});