diff --git a/.github/modify_sitemap_new.js b/.github/modify_sitemap_new.js
index 8edb194..631cb3a 100644
--- a/.github/modify_sitemap_new.js
+++ b/.github/modify_sitemap_new.js
@@ -50,6 +50,25 @@ async function readAndParseXml(filePath) {
});
}
+// The source endpoint has served HTML error pages with a 200 status. Such a
+// document parses as valid XML, so without this check every transform below
+// silently no-ops and the error page gets published as the sitemap.
+const MIN_EXPECTED_URLS = 1000;
+
+function assertIsSitemap(parsed, filePath) {
+ if (!parsed || !parsed.urlset) {
+ const root = Object.keys(parsed || {})[0];
+ throw new Error(
+ `${filePath} is not a sitemap: expected a root element, found ` +
+ `<${root || "nothing"}>`
+ );
+ }
+
+ if (!Array.isArray(parsed.urlset.url) || parsed.urlset.url.length === 0) {
+ throw new Error(`${filePath} has a root but no entries`);
+ }
+}
+
// Function to replace domains in sitemap
function replaceDomains(sitemap, newDomain) {
if (!sitemap.urlset || !sitemap.urlset.url) {
@@ -224,6 +243,8 @@ async function processSitemaps() {
try {
console.log("Reading staging sitemap file...");
const stagingSitemap = await readAndParseXml(stagingSitemapFile);
+ assertIsSitemap(stagingSitemap, stagingSitemapFile);
+ console.log(`Parsed ${stagingSitemap.urlset.url.length} source URLs.`);
console.log("Replacing domains in staging sitemap...");
const processedStagingSitemap = replaceDomains(stagingSitemap, newDomain);
@@ -239,7 +260,15 @@ async function processSitemaps() {
newDomain
);
- console.log("Writing sitemap to output file...");
+ const remainingUrls = finalSitemap.urlset.url.length;
+ if (remainingUrls < MIN_EXPECTED_URLS) {
+ throw new Error(
+ `Only ${remainingUrls} URLs survived filtering (expected at least ` +
+ `${MIN_EXPECTED_URLS}); refusing to publish a truncated sitemap`
+ );
+ }
+
+ console.log(`Writing ${remainingUrls} URLs to output file...`);
// Clean up URLs to remove carriage returns and line feeds
if (finalSitemap.urlset && finalSitemap.urlset.url) {
@@ -279,13 +308,13 @@ async function processSitemaps() {
"\n $1\n "
);
- // Ensure the closing urlset tag is properly formatted
+ // The root element is guaranteed to be by assertIsSitemap, so a
+ // missing closing tag means the builder produced something unusable.
+ // Appending the tag here would mask that, as it once did for an HTML page.
if (!xml.endsWith("")) {
- if (xml.endsWith("";
- } else {
- xml += "\n";
- }
+ throw new Error(
+ "Generated XML does not end with ; refusing to write it"
+ );
}
// Ensure there's a newline at the end of the file
diff --git a/.github/workflows/generate_new_sitemap.yml b/.github/workflows/generate_new_sitemap.yml
index 7a66998..bf98c98 100644
--- a/.github/workflows/generate_new_sitemap.yml
+++ b/.github/workflows/generate_new_sitemap.yml
@@ -24,10 +24,31 @@ jobs:
- name: Download sitemaps
shell: bash
+ env:
+ SOURCE_ORIGIN: "https://staging-webflow.deriv.com"
run: |
+ set -euo pipefail
mkdir -p temp
- curl -o temp/staging_sitemap.xml https://staging-webflow.deriv.com/sitemap.xml || exit 1
- curl -o temp/robots.txt https://staging-webflow.deriv.com/robots.txt || exit 1
+ # --fail is essential: the origin has served HTML error pages with a
+ # 200 status, and without it curl saves the error body and exits 0.
+ curl --fail --silent --show-error --location \
+ --retry 5 --retry-delay 10 --retry-all-errors --max-time 300 \
+ -o temp/staging_sitemap.xml "$SOURCE_ORIGIN/sitemap.xml"
+ curl --fail --silent --show-error --location \
+ --retry 5 --retry-delay 10 --retry-all-errors --max-time 60 \
+ -o temp/robots.txt "$SOURCE_ORIGIN/robots.txt"
+
+ - name: Verify downloaded sitemap
+ shell: bash
+ run: |
+ set -euo pipefail
+ if ! head -c 4096 temp/staging_sitemap.xml | grep -q ' root element; refusing to publish it."
+ echo "First 500 bytes of the response:"
+ head -c 500 temp/staging_sitemap.xml
+ exit 1
+ fi
+ echo "Source sitemap looks valid ($(wc -c < temp/staging_sitemap.xml) bytes)."
- name: Generate Combined Sitemap
uses: ./.github/actions/generate_sitemap_and_robots_new