extractPages.js

import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { getProjects } from './getProjects.js';
import { getBuilds } from './getBuilds.js';
import { getSessions } from './getSessions.js';
import { getPageUrls } from './getNetworkLogs.js';
import { isMainModule, parseArgs } from './lib/cli.js';

const DEFAULT_OUTPUT = 'output/pages.json';

/**
 * Run the full pipeline: projects → builds → sessions → page URLs.
 *
 * Narrow the scope with optional IDs. When an ID is provided, that stage
 * is skipped and the pipeline starts from that level.
 *
 * @param {{
 *   projectId?: string|number,
 *   buildId?: string,
 *   sessionId?: string,
 *   buildLimit?: number,
 *   quiet?: boolean
 * }} [options]
 * @returns {Promise<object[]>}
 */
export async function extractPages({
  projectId,
  buildId,
  sessionId,
  buildLimit = 100,
  quiet = false,
} = {}) {
  const log = quiet ? () => {} : console.log;
  const results = [];

  // Start from a single session
  if (sessionId) {
    const urls = await getPageUrls(sessionId);
    results.push({ sessionId, urls });
    return results;
  }

  // Start from a single build
  if (buildId) {
    const sessions = await getSessions(buildId);
    log(`Build ${buildId}: ${sessions.length} session(s)`);
    for (const session of sessions) {
      const id = session.hashed_id;
      try {
        const urls = await getPageUrls(id);
        results.push({ projectId, buildId, sessionId: id, urls });
        log(`  Session ${id}: ${urls.length} page URL(s)`);
      } catch (error) {
        log(`  Session ${id}: skipped (${error.message})`);
        results.push({ projectId, buildId, sessionId: id, urls: [], error: error.message });
      }
    }
    return results;
  }

  // Resolve project list
  let projects;
  if (projectId) {
    projects = [{ id: projectId }];
  } else {
    projects = await getProjects();
    log(`Found ${projects.length} project(s)`);
  }

  for (const project of projects) {
    const pid = project.id;
    const builds = await getBuilds(pid, { limit: buildLimit });
    log(`Project ${pid}: ${builds.length} build(s)`);

    for (const build of builds) {
      const bid = build.hashed_id;
      const sessions = await getSessions(bid);
      log(`  Build ${bid}: ${sessions.length} session(s)`);

      for (const session of sessions) {
        const sid = session.hashed_id;
        try {
          const urls = await getPageUrls(sid);
          results.push({ projectId: pid, buildId: bid, sessionId: sid, urls });
          log(`    Session ${sid}: ${urls.length} page URL(s)`);
        } catch (error) {
          log(`    Session ${sid}: skipped (${error.message})`);
          results.push({
            projectId: pid,
            buildId: bid,
            sessionId: sid,
            urls: [],
            error: error.message,
          });
        }
      }
    }
  }

  return results;
}

async function writeResults(results, outputPath) {
  const absolutePath = resolve(outputPath);
  await mkdir(dirname(absolutePath), { recursive: true });
  await writeFile(absolutePath, JSON.stringify(results, null, 2) + '\n', 'utf8');
  return absolutePath;
}

async function main() {
  const args = parseArgs(process.argv.slice(2));

  if (args.help || args.h) {
    console.log(`Usage:
  node extractPages.js [options]

Options:
  --projectId <id>   Limit to one project
  --buildId <id>     Start from a build (skips projects)
  --sessionId <id>   Extract pages from one session only
  --buildLimit <n>   Max builds per project (default: 100)
  --out <path>       Write JSON results to this file (default: ${DEFAULT_OUTPUT})
  --quiet            Suppress progress logs
`);
    return;
  }

  const outputPath = args.out || args.output || DEFAULT_OUTPUT;
  const quiet = Boolean(args.quiet);

  const results = await extractPages({
    projectId: args.projectId || process.env.BROWSERSTACK_PROJECT_ID,
    buildId: args.buildId || process.env.BROWSERSTACK_BUILD_ID,
    sessionId: args.sessionId || process.env.BROWSERSTACK_SESSION_ID,
    buildLimit: args.buildLimit ? Number(args.buildLimit) : 100,
    quiet,
  });

  const writtenTo = await writeResults(results, outputPath);
  if (!quiet) {
    console.log(`Wrote ${results.length} result(s) to ${writtenTo}`);
  } else {
    console.log(writtenTo);
  }
}

if (isMainModule(import.meta.url)) {
  main().catch((error) => {
    console.error('Pipeline failed:', error.message);
    process.exit(1);
  });
}