refactor: resolve tile keys in the tiles-deletion strategy (MAPCO-11269)

src/cleaner/strategies/tilesDeletionStrategy.ts

@@ -1,24 +1,20 @@
 import { NoSuchKey } from '@aws-sdk/client-s3';
 import type { Logger } from '@map-colonies/js-logger';
 import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue';
-import { StorageProvider, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared';
+import { StorageProvider, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared';
 import { inject, injectable } from 'tsyringe';
 import type { ConfigType } from '@common/config';
 import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants';
 import { mergeFailures, summarizeDeleteFailures, type DeleteFailure, type StorageProviders } from '@src/cleaner/storageProviders';
 import { RecoverableError, UnrecoverableError, describeError } from '../errors';
 import { ResolvedStorageProvider } from '../storageProviders/iStorageProvider';
-import { validateSchema } from '../utils';
+import { resolveTileKeyGenerator, validateSchema } from '../utils';
 import type { TaskContext } from './strategyFactory';
 import type { ITaskStrategy } from './taskStrategy';
 
 const NOT_FOUND_REASONS = new Set<string>([NoSuchKey.name, 'ENOENT']);
 
-/**
- * The params shapes this strategy can act on today. Redis tiles deletion is not implemented
- * yet (MAPCO-11261): its params carry a key prefix instead of a tiles path, so there are no
- * tile paths to generate.
- */
+/** Redis tiles deletion is not implemented yet (MAPCO-11263). */
 type SupportedTilesDeletionParams = Exclude<TilesDeletionParams, { storageProvider: 'REDIS' }>;
 
 @injectable()
@@ -63,7 +59,7 @@ export class TilesDeletionStrategy implements ITaskStrategy<TilesDeletionParams>
       totalTiles,
     });
 
-    const failures = await this.deleteAllTiles(provider, storageTarget, params, totalTiles);
+    const failures = await this.deleteTiles(provider, storageTarget, params, totalTiles);
     this.reportOutcome(failures, totalTiles);
   }
 
@@ -125,7 +121,7 @@ export class TilesDeletionStrategy implements ITaskStrategy<TilesDeletionParams>
     return { provider: storageProvider, storageTarget };
   }
 
-  private async deleteAllTiles(
+  private async deleteTiles(
     provider: ResolvedStorageProvider,
     storageTarget: string,
     params: SupportedTilesDeletionParams,
@@ -137,8 +133,8 @@ export class TilesDeletionStrategy implements ITaskStrategy<TilesDeletionParams>
     let batch: string[] = [];
     let processedTiles = 0;
 
-    for (const tilePath of this.generateTilePaths(params)) {
-      batch.push(tilePath);
+    for (const tileKey of this.generateTileKeys(params)) {
+      batch.push(tileKey);
       if (batch.length === this.batchSize) {
         pendingBatches.push(batch);
         batch = [];
@@ -210,17 +206,11 @@ export class TilesDeletionStrategy implements ITaskStrategy<TilesDeletionParams>
     return params.ranges.reduce((sum, r) => sum + (r.maxX - r.minX + 1) * (r.maxY - r.minY + 1), 0);
   }
 
-  private *generateTilePaths(params: SupportedTilesDeletionParams): Generator<string> {
-    for (const range of params.ranges) {
-      yield* this.generateRangePaths(range, params.tilesRelativePath, params.fileExtension);
-    }
-  }
+  private *generateTileKeys(params: SupportedTilesDeletionParams): Generator<string> {
+    const toKeys = resolveTileKeyGenerator(params);
 
-  private *generateRangePaths(range: TileRange, tilesRelativePath: string, fileExtension: string): Generator<string> {
-    for (let x = range.minX; x <= range.maxX; x++) {
-      for (let y = range.minY; y <= range.maxY; y++) {
-        yield `${tilesRelativePath}/${range.zoom}/${x}/${y}.${fileExtension}`;
-      }
+    for (const range of params.ranges) {
+      yield* toKeys(range);
     }
   }
 }

src/cleaner/utils/index.ts

@@ -1,5 +1,13 @@
 export { getChunk } from './chunk';
 export { assertCanDeleteFromFolder } from './fs';
 export { buildPollingPairs } from './pairBuilder';
-export { isPathWithinAllowedSubPaths, normalizeFolderPath, resolveAbsolutePath } from './path';
+export {
+  generateRangeKeys,
+  generateRedisTileKeys,
+  generateTilePaths,
+  isPathWithinAllowedSubPaths,
+  normalizeFolderPath,
+  resolveAbsolutePath,
+} from './path';
+export { resolveTileKeyGenerator } from './tileKeys';
 export { validateSchema } from './validationHelper';

src/cleaner/utils/path.ts

@@ -1,9 +1,30 @@
 import { join, resolve, sep } from 'node:path/posix';
+import type { TileRange } from '@map-colonies/raster-shared';
 
 export const normalizeFolderPath = (path: string): string => {
   return path.endsWith(sep) ? path : `${path}${sep}`;
 };
 
+/**
+ * Walks every tile in a range and yields whatever `formatKey` makes of it, so the traversal
+ * lives in one place and each provider only supplies its own key shape.
+ */
+export function* generateRangeKeys(range: TileRange, formatKey: (zoom: number, x: number, y: number) => string): Generator<string> {
+  for (let x = range.minX; x <= range.maxX; x++) {
+    for (let y = range.minY; y <= range.maxY; y++) {
+      yield formatKey(range.zoom, x, y);
+    }
+  }
+}
+
+/** Tile paths for the S3 and FS providers, which address a tile identically. */
+export const generateTilePaths = (range: TileRange, tilesRelativePath: string, fileExtension: string): Generator<string> =>
+  generateRangeKeys(range, (zoom, x, y) => `${tilesRelativePath}/${zoom}/${x}/${y}.${fileExtension}`);
+
+/** Redis tile keys in mapproxy's format */
+export const generateRedisTileKeys = (range: TileRange, prefix: string): Generator<string> =>
+  generateRangeKeys(range, (zoom, x, y) => `${prefix}-${zoom}-${x}-${y}`);
+
 /**
  * Resolves a file system path to an absolute path.
  * Ensures the path is resolved as an absolute path and properly formatted

src/cleaner/utils/tileKeys.ts

@@ -0,0 +1,16 @@
+import { StorageProvider, type TileRange, type TilesDeletionParams } from '@map-colonies/raster-shared';
+import { generateRedisTileKeys, generateTilePaths } from './path';
+
+/**
+ * Picks the tile-key format for a deletion request. Switching on the discriminant (rather than
+ * indexing a `Record<StorageProvider, …>`) is what narrows `params` per provider without a cast.
+ */
+export function resolveTileKeyGenerator(params: TilesDeletionParams): (range: TileRange) => Generator<string> {
+  switch (params.storageProvider) {
+    case StorageProvider.S3:
+    case StorageProvider.FS:
+      return (range) => generateTilePaths(range, params.tilesRelativePath, params.fileExtension);
+    case StorageProvider.REDIS:
+      return (range) => generateRedisTileKeys(range, params.prefix);
+  }
+}

tests/utils/path.spec.ts

@@ -1,5 +1,12 @@
+import type { TileRange } from '@map-colonies/raster-shared';
 import { describe, expect, it } from 'vitest';
-import { isPathWithinAllowedSubPaths, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils/path';
+import {
+  generateRedisTileKeys,
+  generateTilePaths,
+  isPathWithinAllowedSubPaths,
+  normalizeFolderPath,
+  resolveAbsolutePath,
+} from '@src/cleaner/utils/path';
 
 const BASE_PATH = '/data';
 const ALLOWED_SUB_PATHS = ['artifacts/tiles', 'artifacts/gpkgs'];
@@ -65,4 +72,57 @@ describe('path', () => {
       expect(isPathWithinAllowedSubPaths({ relativePath: 'artifacts/tiles/layer', basePath: BASE_PATH, allowedSubPaths: [] })).toBe(false);
     });
   });
+
+  describe('#generateTilePaths', () => {
+    it('should yield a path per tile, ordered x-major then y', () => {
+      const range: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 };
+
+      expect([...generateTilePaths(range, 'layer/v1', 'png')]).toEqual([
+        'layer/v1/3/1/5.png',
+        'layer/v1/3/1/6.png',
+        'layer/v1/3/2/5.png',
+        'layer/v1/3/2/6.png',
+      ]);
+    });
+
+    it('should yield a single path for a range covering one tile', () => {
+      const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 };
+
+      expect([...generateTilePaths(range, 'layer/v1', 'jpeg')]).toEqual(['layer/v1/0/0/0.jpeg']);
+    });
+  });
+
+  describe('#generateRedisTileKeys', () => {
+    it('should yield dash-separated keys with zoom first and no file extension', () => {
+      const range: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 };
+
+      expect([...generateRedisTileKeys(range, 'myLayer-redis_WorldCRS84')]).toEqual([
+        'myLayer-redis_WorldCRS84-3-1-5',
+        'myLayer-redis_WorldCRS84-3-1-6',
+        'myLayer-redis_WorldCRS84-3-2-5',
+        'myLayer-redis_WorldCRS84-3-2-6',
+      ]);
+    });
+
+    // Captured off a live MapProxy command stream (DISCOVERY.md). A wrong key deletes nothing
+    // and still reports success, so it is pinned exactly.
+    it('should reproduce the key format captured from MapProxy', () => {
+      const range: TileRange = { zoom: 12, minX: 4892, maxX: 4892, minY: 2784, maxY: 2784 };
+
+      expect([...generateRedisTileKeys(range, 'benchmark')]).toEqual(['benchmark-12-4892-2784']);
+    });
+
+    // Production uses mapproxy-api's `{cacheName}_{gridName}` fallback, not the layer id.
+    it('should treat the prefix as an opaque string', () => {
+      const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 };
+
+      expect([...generateRedisTileKeys(range, 'myLayer-redis_WorldCRS84')]).toEqual(['myLayer-redis_WorldCRS84-0-0-0']);
+    });
+
+    it('should yield a single key for a range covering one tile', () => {
+      const range: TileRange = { zoom: 21, minX: 7, maxX: 7, minY: 9, maxY: 9 };
+
+      expect([...generateRedisTileKeys(range, 'p')]).toEqual(['p-21-7-9']);
+    });
+  });
 });

tests/utils/tileKeys.spec.ts

@@ -0,0 +1,98 @@
+import { faker } from '@faker-js/faker';
+import type { FsTilesDeletionParams, RedisTilesDeletionParams, S3TilesDeletionParams, TileRange } from '@map-colonies/raster-shared';
+import { describe, expect, it } from 'vitest';
+import { resolveTileKeyGenerator } from '@src/cleaner/utils/tileKeys';
+
+const RANGE: TileRange = { zoom: 3, minX: 1, maxX: 2, minY: 5, maxY: 6 };
+
+/**
+ * Pins the key shape per storage provider. S3 and FS must stay byte-identical — the strategy
+ * relies on them sharing one key family — while Redis uses mapproxy's flat key format.
+ */
+describe('resolveTileKeyGenerator', () => {
+  const relativePath = `${faker.string.uuid()}/${faker.string.uuid()}`;
+
+  describe('S3', () => {
+    it('should yield a path per tile, ordered x-major then y', () => {
+      const params: S3TilesDeletionParams = {
+        storageProvider: 'S3',
+        bucket: 'b',
+        tilesRelativePath: relativePath,
+        fileExtension: 'png',
+        ranges: [RANGE],
+      };
+
+      expect([...resolveTileKeyGenerator(params)(RANGE)]).toEqual([
+        `${relativePath}/3/1/5.png`,
+        `${relativePath}/3/1/6.png`,
+        `${relativePath}/3/2/5.png`,
+        `${relativePath}/3/2/6.png`,
+      ]);
+    });
+
+    it('should use the requested file extension', () => {
+      const range: TileRange = { zoom: 0, minX: 0, maxX: 0, minY: 0, maxY: 0 };
+      const params: S3TilesDeletionParams = {
+        storageProvider: 'S3',
+        bucket: 'b',
+        tilesRelativePath: relativePath,
+        fileExtension: 'jpeg',
+        ranges: [range],
+      };
+
+      expect([...resolveTileKeyGenerator(params)(range)]).toEqual([`${relativePath}/0/0/0.jpeg`]);
+    });
+  });
+
+  describe('FS', () => {
+    it('should yield the same path shape as S3', () => {
+      const fsParams: FsTilesDeletionParams = {
+        storageProvider: 'FS',
+        subPath: 'artifacts/tiles',
+        tilesRelativePath: relativePath,
+        fileExtension: 'jpeg',
+        ranges: [RANGE],
+      };
+      const s3Params: S3TilesDeletionParams = {
+        storageProvider: 'S3',
+        bucket: 'b',
+        tilesRelativePath: relativePath,
+        fileExtension: 'jpeg',
+        ranges: [RANGE],
+      };
+
+      expect([...resolveTileKeyGenerator(fsParams)(RANGE)]).toEqual([...resolveTileKeyGenerator(s3Params)(RANGE)]);
+    });
+
+    it('should offset x and y when the range does not start at zero', () => {
+      const range: TileRange = { zoom: 7, minX: 3, maxX: 4, minY: 8, maxY: 9 };
+      const params: FsTilesDeletionParams = {
+        storageProvider: 'FS',
+        subPath: 'artifacts/tiles',
+        tilesRelativePath: relativePath,
+        fileExtension: 'png',
+        ranges: [range],
+      };
+
+      expect([...resolveTileKeyGenerator(params)(range)]).toEqual([
+        `${relativePath}/7/3/8.png`,
+        `${relativePath}/7/3/9.png`,
+        `${relativePath}/7/4/8.png`,
+        `${relativePath}/7/4/9.png`,
+      ]);
+    });
+  });
+
+  describe('REDIS', () => {
+    it('should yield flat prefixed keys rather than paths', () => {
+      const prefix = 'myLayer-redis_WorldCRS84';
+      const params: RedisTilesDeletionParams = {
+        storageProvider: 'REDIS',
+        prefix,
+        ranges: [RANGE],
+      };
+
+      expect([...resolveTileKeyGenerator(params)(RANGE)]).toEqual([`${prefix}-3-1-5`, `${prefix}-3-1-6`, `${prefix}-3-2-5`, `${prefix}-3-2-6`]);
+    });
+  });
+});