diff --git a/.github/modify_sitemap_new.js b/.github/modify_sitemap_new.js index 8edb194..631cb3a 100644 --- a/.github/modify_sitemap_new.js +++ b/.github/modify_sitemap_new.js @@ -50,6 +50,25 @@ async function readAndParseXml(filePath) { }); } +// The source endpoint has served HTML error pages with a 200 status. Such a +// document parses as valid XML, so without this check every transform below +// silently no-ops and the error page gets published as the sitemap. +const MIN_EXPECTED_URLS = 1000; + +function assertIsSitemap(parsed, filePath) { + if (!parsed || !parsed.urlset) { + const root = Object.keys(parsed || {})[0]; + throw new Error( + `${filePath} is not a sitemap: expected a root element, found ` + + `<${root || "nothing"}>` + ); + } + + if (!Array.isArray(parsed.urlset.url) || parsed.urlset.url.length === 0) { + throw new Error(`${filePath} has a root but no entries`); + } +} + // Function to replace domains in sitemap function replaceDomains(sitemap, newDomain) { if (!sitemap.urlset || !sitemap.urlset.url) { @@ -224,6 +243,8 @@ async function processSitemaps() { try { console.log("Reading staging sitemap file..."); const stagingSitemap = await readAndParseXml(stagingSitemapFile); + assertIsSitemap(stagingSitemap, stagingSitemapFile); + console.log(`Parsed ${stagingSitemap.urlset.url.length} source URLs.`); console.log("Replacing domains in staging sitemap..."); const processedStagingSitemap = replaceDomains(stagingSitemap, newDomain); @@ -239,7 +260,15 @@ async function processSitemaps() { newDomain ); - console.log("Writing sitemap to output file..."); + const remainingUrls = finalSitemap.urlset.url.length; + if (remainingUrls < MIN_EXPECTED_URLS) { + throw new Error( + `Only ${remainingUrls} URLs survived filtering (expected at least ` + + `${MIN_EXPECTED_URLS}); refusing to publish a truncated sitemap` + ); + } + + console.log(`Writing ${remainingUrls} URLs to output file...`); // Clean up URLs to remove carriage returns and line feeds if (finalSitemap.urlset && finalSitemap.urlset.url) { @@ -279,13 +308,13 @@ async function processSitemaps() { "\n $1\n " ); - // Ensure the closing urlset tag is properly formatted + // The root element is guaranteed to be by assertIsSitemap, so a + // missing closing tag means the builder produced something unusable. + // Appending the tag here would mask that, as it once did for an HTML page. if (!xml.endsWith("")) { - if (xml.endsWith("; refusing to write it" + ); } // Ensure there's a newline at the end of the file diff --git a/.github/workflows/generate_new_sitemap.yml b/.github/workflows/generate_new_sitemap.yml index 7a66998..bf98c98 100644 --- a/.github/workflows/generate_new_sitemap.yml +++ b/.github/workflows/generate_new_sitemap.yml @@ -24,10 +24,31 @@ jobs: - name: Download sitemaps shell: bash + env: + SOURCE_ORIGIN: "https://staging-webflow.deriv.com" run: | + set -euo pipefail mkdir -p temp - curl -o temp/staging_sitemap.xml https://staging-webflow.deriv.com/sitemap.xml || exit 1 - curl -o temp/robots.txt https://staging-webflow.deriv.com/robots.txt || exit 1 + # --fail is essential: the origin has served HTML error pages with a + # 200 status, and without it curl saves the error body and exits 0. + curl --fail --silent --show-error --location \ + --retry 5 --retry-delay 10 --retry-all-errors --max-time 300 \ + -o temp/staging_sitemap.xml "$SOURCE_ORIGIN/sitemap.xml" + curl --fail --silent --show-error --location \ + --retry 5 --retry-delay 10 --retry-all-errors --max-time 60 \ + -o temp/robots.txt "$SOURCE_ORIGIN/robots.txt" + + - name: Verify downloaded sitemap + shell: bash + run: | + set -euo pipefail + if ! head -c 4096 temp/staging_sitemap.xml | grep -q ' root element; refusing to publish it." + echo "First 500 bytes of the response:" + head -c 500 temp/staging_sitemap.xml + exit 1 + fi + echo "Source sitemap looks valid ($(wc -c < temp/staging_sitemap.xml) bytes)." - name: Generate Combined Sitemap uses: ./.github/actions/generate_sitemap_and_robots_new