fix: batch Gateway Connector target scans

src/core/gateway.test.ts

@@ -36,7 +36,7 @@ describe("GatewayClient Connector facade", () => {
       expect(command.input).toEqual({
         gatewayIdentifier: "gateway-1",
         nextToken: "page-2",
-        maxResults: 10,
+        maxResults: 1000,
       });
       return { items: [connectorTarget, ordinary("target-1")] };
     });
@@ -87,10 +87,10 @@ describe("GatewayClient Connector facade", () => {
       },
     );
     expect(requests).toEqual([
-      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 3 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 2 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-3", maxResults: 1 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-4", maxResults: 100 },
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-3", maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-4", maxResults: 1000 },
     ]);
   });
 
@@ -114,8 +114,8 @@ describe("GatewayClient Connector facade", () => {
       },
     );
     expect(requests).toEqual([
-      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 100 },
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 1000 },
     ]);
   });
 
@@ -138,8 +138,8 @@ describe("GatewayClient Connector facade", () => {
       },
     );
     expect(requests).toEqual([
-      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 3 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 2 },
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 1000 },
     ]);
   });
 
@@ -162,17 +162,85 @@ describe("GatewayClient Connector facade", () => {
       items: connectors,
     });
     expect(requests).toEqual([
-      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 100 },
-      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 99 },
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: "page-2", maxResults: 1000 },
     ]);
   });
 
-  test("throws when Connector discovery exceeds the Target page cap", async () => {
+  test("scans more than the default Target quota in one request", async () => {
+    const connectorTarget = connector("connector-1");
+    const targets = [
+      ...Array.from({ length: 150 }, (_, index) => ordinary(`target-${index + 1}`)),
+      connectorTarget,
+    ];
+    const requests: unknown[] = [];
+    const client = gatewayClient(async (command) => {
+      if (!(command instanceof ListGatewayTargetsCommand)) {
+        throw new Error("expected ListGatewayTargetsCommand");
+      }
+      requests.push(command.input);
+      return { items: targets };
+    });
+
+    await expect(client.listGatewayConnectors("gateway-1", undefined, 1, options)).resolves.toEqual(
+      {
+        items: [connectorTarget],
+      },
+    );
+    expect(requests).toEqual([
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+    ]);
+  });
+
+  test("replays the exact Target prefix when a scan finds extra Connectors", async () => {
+    const firstConnector = connector("connector-1");
+    const secondConnector = connector("connector-2");
+    const requests: unknown[] = [];
+    const client = gatewayClient(async (command) => {
+      if (!(command instanceof ListGatewayTargetsCommand)) {
+        throw new Error("expected ListGatewayTargetsCommand");
+      }
+      requests.push(command.input);
+
+      if (command.input.nextToken === "after-connector-1") {
+        return { items: [ordinary("target-2"), secondConnector, ordinary("target-3")] };
+      }
+      if (command.input.maxResults === 2) {
+        return {
+          items: [ordinary("target-1"), firstConnector],
+          nextToken: "after-connector-1",
+        };
+      }
+      return {
+        items: [
+          ordinary("target-1"),
+          firstConnector,
+          ordinary("target-2"),
+          secondConnector,
+          ordinary("target-3"),
+        ],
+      };
+    });
+
+    const first = await client.listGatewayConnectors("gateway-1", undefined, 1, options);
+    const second = await client.listGatewayConnectors("gateway-1", first.nextToken, 1, options);
+
+    expect(first).toEqual({ items: [firstConnector], nextToken: "after-connector-1" });
+    expect(second).toEqual({ items: [secondConnector] });
+    expect(requests).toEqual([
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 1000 },
+      { gatewayIdentifier: "gateway-1", nextToken: undefined, maxResults: 2 },
+      { gatewayIdentifier: "gateway-1", nextToken: "after-connector-1", maxResults: 1000 },
+    ]);
+  });
+
+  test("throws when Connector discovery exceeds the Target scan request cap", async () => {
     let calls = 0;
     const client = gatewayClient(async (command) => {
       if (!(command instanceof ListGatewayTargetsCommand)) {
         throw new Error("expected ListGatewayTargetsCommand");
       }
+      expect(command.input.maxResults).toBe(1000);
       calls += 1;
       return { items: [], nextToken: `page-${calls}` };
     });

src/core/gateway.tsx

@@ -32,7 +32,8 @@ import type { AwsClients, CoreOptions } from "./types";
 import { toClientConfig } from "./utils";
 
 const DEFAULT_CONNECTOR_PAGE_SIZE = 100;
-const MAX_CONNECTOR_TARGET_PAGES = 101;
+const CONNECTOR_TARGET_SCAN_PAGE_SIZE = 1000;
+const MAX_CONNECTOR_TARGET_SCAN_REQUESTS = 101;
 
 export class GatewayClient implements CoreGatewayClient {
   constructor(private readonly clients: AwsClients) {}
@@ -123,34 +124,50 @@ export class GatewayClient implements CoreGatewayClient {
     maxResults: number | undefined,
     options: CoreOptions,
   ): Promise<ListGatewayTargetsResponse> {
-    const pageSize = maxResults ?? DEFAULT_CONNECTOR_PAGE_SIZE;
+    const connectorPageSize = maxResults ?? DEFAULT_CONNECTOR_PAGE_SIZE;
     const items: TargetSummary[] = [];
-    let token = nextToken;
-    let filling = true;
-
-    for (let page = 0; page < MAX_CONNECTOR_TARGET_PAGES; page++) {
-      const requestToken = token;
-      const requestSize = filling ? pageSize - items.length : DEFAULT_CONNECTOR_PAGE_SIZE;
-      const response = await this.listGatewayTargets(gatewayId, token, requestSize, options);
-      const connectors = (response.items ?? []).filter(
-        (target) => target.targetType === TargetType.CONNECTOR,
+    let targetToken = nextToken;
+
+    for (let request = 0; request < MAX_CONNECTOR_TARGET_SCAN_REQUESTS; request++) {
+      const requestToken = targetToken;
+      const response = await this.listGatewayTargets(
+        gatewayId,
+        targetToken,
+        CONNECTOR_TARGET_SCAN_PAGE_SIZE,
+        options,
       );
-
-      if (filling) {
+      const targets = response.items ?? [];
+      const connectors = targets.filter((target) => target.targetType === TargetType.CONNECTOR);
+
+      if (items.length < connectorPageSize) {
+        const remaining = connectorPageSize - items.length;
+        if (connectors.length > remaining) {
+          const boundaryTarget = connectors[remaining - 1]!;
+          const boundarySize = targets.indexOf(boundaryTarget) + 1;
+          // Re-read only through the last returned Connector so the AWS token cannot skip matches.
+          const boundaryResponse = await this.listGatewayTargets(
+            gatewayId,
+            requestToken,
+            boundarySize,
+            options,
+          );
+
+          items.push(...connectors.slice(0, remaining));
+          return { ...boundaryResponse, items };
+        }
         items.push(...connectors);
-        filling = items.length < pageSize;
       } else if (connectors.length > 0) {
         return { ...response, items, nextToken: requestToken };
       }
 
       if (response.nextToken === undefined) {
         return { ...response, items, nextToken: undefined };
       }
-      token = response.nextToken;
+      targetToken = response.nextToken;
     }
 
     throw new ResultTruncationError(
-      `Gateway Connector discovery exceeded ${MAX_CONNECTOR_TARGET_PAGES} Target pages; results are incomplete`,
+      `Gateway Connector discovery exceeded ${MAX_CONNECTOR_TARGET_SCAN_REQUESTS} Target scan requests; results are incomplete`,
     );
   }
 

src/handlers/gateway/__fixtures__/ListGatewayTargetsCommand.e04764abb647e975.json → src/handlers/gateway/__fixtures__/ListGatewayTargetsCommand.87116fefba167d6f.json

@@ -17,4 +17,4 @@
       "targetType": "CONNECTOR"
     }
   ]
-}
+}