diff --git a/common/changes/@microsoft/rush/fix-build-cache-poisoning-race_2026-09-24.json b/common/changes/@microsoft/rush/fix-build-cache-poisoning-race_2026-09-24.json new file mode 100644 index 0000000000..87048f4639 --- /dev/null +++ b/common/changes/@microsoft/rush/fix-build-cache-poisoning-race_2026-09-24.json @@ -0,0 +1,9 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Fix a build cache poisoning race: skip writing a build cache entry when an operation's tracked input files changed while it was executing.", + "type": "patch" + } + ] +} diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 52f577b98c..4ef1df2f58 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -2,8 +2,9 @@ // See LICENSE in the project root for license information. import * as crypto from 'node:crypto'; +import * as path from 'node:path'; -import { InternalError, NewlineKind, Sort } from '@rushstack/node-core-library'; +import { InternalError, NewlineKind, Sort, Executable } from '@rushstack/node-core-library'; import { CollatedTerminal, type CollatedWriter } from '@rushstack/stream-collator'; import { DiscardStdoutTransform, @@ -28,6 +29,13 @@ import { import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { DisjointSet } from '../cobuild/DisjointSet'; import { PeriodicCallback } from './PeriodicCallback'; +import { + captureInputFilesState, + haveInputFilesChanged, + hasUntrackedGitFiles, + type IInputFilesState +} from './InputFilesStatSignature'; +import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { NullTerminalProvider } from '../../utilities/NullTerminalProvider'; import type { Operation } from './Operation'; import type { IOperationRunnerContext } from './IOperationRunner'; @@ -70,6 +78,11 @@ export interface IOperationBuildCacheContext { periodicCallback: PeriodicCallback; cacheRestored: boolean; isCacheReadAttempted: boolean; + + // The on-disk state of the tracked input files whose hashes produced the cache key, captured right after + // the iteration's inputs snapshot. Used to refuse cache writes if the inputs changed while the operation + // was executing. + inputFilesState?: IInputFilesState; } export interface ICacheableOperationPluginOptions { @@ -102,10 +115,33 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { readonly #options: ICacheableOperationPluginOptions; + #gitPathResolved: boolean = false; + #gitPath: string | undefined; + public constructor(options: ICacheableOperationPluginOptions) { this.#options = options; } + #isNewInput( + newEntryPaths: ReadonlyArray, + rootDirectory: string, + projectFolder: string, + outputFolderNames: ReadonlyArray + ): boolean { + if (!this.#gitPathResolved) { + this.#gitPath = EnvironmentConfiguration.gitBinaryPath || Executable.tryResolve('git'); + this.#gitPathResolved = true; + } + if (!this.#gitPath) { + // Without Git we cannot tell whether the new entries are ignored, so assume they are inputs. + return true; + } + const outputFolderPaths: string[] = outputFolderNames.map((folderName: string) => + path.resolve(projectFolder, folderName) + ); + return hasUntrackedGitFiles(this.#gitPath, rootDirectory, newEntryPaths, outputFolderPaths); + } + public apply(hooks: PhasedCommandHooks): void { const { allowWarningsInSuccessfulBuild, @@ -177,6 +213,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { disjointSet?.add(operation); + const inputFilesState: IInputFilesState | undefined = + cacheWriteEnabled && !cacheDisabledReason && record.enabled + ? captureInputFilesState(inputsSnapshot.rootDirectory, fileHashes.keys()) + : undefined; + const buildCacheContext: IOperationBuildCacheContext = { // Supports cache writes by default for initial operations. // Don't write during watch runs for performance reasons (and to avoid flooding the cache) @@ -193,7 +234,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 }), cacheRestored: false, - isCacheReadAttempted: false + isCacheReadAttempted: false, + inputFilesState }; // Upstream runners may mutate the property of build cache context for downstream runners this.#buildCacheContextByOperation.set(operation, buildCacheContext); @@ -538,6 +580,29 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && operationBuildCache) { setCacheEntryPromise = () => operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal); } + const { inputFilesState } = buildCacheContext; + if ( + !cacheRestored && + isCacheWriteAllowed && + inputFilesState && + haveInputFilesChanged(inputFilesState, (newEntryPaths: ReadonlyArray) => + this.#isNewInput( + newEntryPaths, + inputFilesState.rootDirectory, + project.projectFolder, + buildCacheContext.outputFolderNames + ) + ) + ) { + // The cache key was derived from the iteration's inputs snapshot. Storing outputs produced from + // edited inputs under that key would poison the cache for every consumer of the entry. + // Consumers' cache keys also embed this operation's pre-edit state, so block their writes too. + buildCacheTerminal.writeLine( + 'Input files changed while this operation was executing; not writing a build cache entry.' + ); + buildCacheContext.isCacheWriteAllowed = false; + setCacheEntryPromise = undefined; + } if (!cacheRestored) { const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise?.(); await setCompletedStatePromiseFunction?.(); diff --git a/libraries/rush-lib/src/logic/operations/InputFilesStatSignature.ts b/libraries/rush-lib/src/logic/operations/InputFilesStatSignature.ts new file mode 100644 index 0000000000..aef3eaff54 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/InputFilesStatSignature.ts @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { Executable } from '@rushstack/node-core-library'; + +/** + * The on-disk state of an operation's tracked input files, captured right after the inputs snapshot + * (from which the operation's build cache key is derived) was taken. + */ +export interface IInputFilesState { + /** + * The repository root that relative input file paths were resolved against. + */ + readonly rootDirectory: string; + /** + * Absolute paths of the tracked input files. + */ + readonly filePaths: ReadonlyArray; + /** + * Signature of the size, modification time, and inode of each tracked input file. + */ + readonly statSignature: string; + /** + * For each folder inside the repository that contains a tracked input file, the names of its entries. + * Used to detect files (or folders) that were created after the snapshot was taken. + */ + readonly folderEntries: ReadonlyMap>; +} + +/** + * Given the absolute paths of entries that appeared in input folders after the snapshot was taken, + * returns true if any of them is a potential input of the operation (e.g. an untracked, non-ignored file). + */ +export type IsNewInputCallback = (newEntryPaths: ReadonlyArray) => boolean; + +/** + * Computes a cheap signature of the on-disk identity (size, mtime, inode) of the specified files. + * Missing files are included in the signature, so deleting or creating a listed file also changes it. + */ +export function getInputFilesStatSignature(filePaths: Iterable): string { + const hasher: crypto.Hash = crypto.createHash('sha1'); + for (const filePath of filePaths) { + const stats: fs.BigIntStats | undefined = fs.statSync(filePath, { bigint: true, throwIfNoEntry: false }); + if (stats) { + hasher.update(`${filePath}\0${stats.size}\0${stats.mtimeNs}\0${stats.ino}\n`); + } else { + hasher.update(`${filePath}\0missing\n`); + } + } + return hasher.digest('hex'); +} + +function tryReadFolderEntries(folderPath: string): Set | undefined { + try { + return new Set(fs.readdirSync(folderPath)); + } catch { + return undefined; + } +} + +/** + * Captures the on-disk state of an operation's tracked input files. + * + * @param rootDirectory - The repository root that relative input file paths are resolved against + * @param inputFilePaths - The tracked input file paths. Relative paths are resolved against `rootDirectory`; + * absolute paths (e.g. `dependsOnAdditionalFiles` outside of the repository) are stat'ed but their folders + * are not watched for new entries. + */ +export function captureInputFilesState( + rootDirectory: string, + inputFilePaths: Iterable +): IInputFilesState { + const filePaths: string[] = []; + const folderEntries: Map> = new Map(); + for (const inputFilePath of inputFilePaths) { + const absolutePath: string = path.resolve(rootDirectory, inputFilePath); + filePaths.push(absolutePath); + if (!path.isAbsolute(inputFilePath)) { + const folderPath: string = path.dirname(absolutePath); + if (!folderEntries.has(folderPath)) { + folderEntries.set(folderPath, tryReadFolderEntries(folderPath) ?? new Set()); + } + } + } + return { rootDirectory, filePaths, statSignature: getInputFilesStatSignature(filePaths), folderEntries }; +} + +/** + * Returns the absolute paths of entries that exist now but did not exist when the folder entries were captured. + */ +export function getNewFolderEntries(folderEntries: ReadonlyMap>): string[] { + const newEntryPaths: string[] = []; + for (const [folderPath, originalEntries] of folderEntries) { + const currentEntries: Set | undefined = tryReadFolderEntries(folderPath); + if (currentEntries) { + for (const entry of currentEntries) { + if (!originalEntries.has(entry)) { + newEntryPaths.push(path.join(folderPath, entry)); + } + } + } + } + return newEntryPaths; +} + +/** + * Returns true if any of the operation's tracked input files was modified, deleted, or replaced, or if a + * potential new input file was created in one of the input folders, since the state was captured. + */ +export function haveInputFilesChanged(state: IInputFilesState, isNewInput: IsNewInputCallback): boolean { + if (getInputFilesStatSignature(state.filePaths) !== state.statSignature) { + return true; + } + const newEntryPaths: string[] = getNewFolderEntries(state.folderEntries); + return newEntryPaths.length > 0 && isNewInput(newEntryPaths); +} + +function toGitPathspec(rootDirectory: string, absolutePath: string): string { + return path.relative(rootDirectory, absolutePath).split(path.sep).join('/'); +} + +/** + * Uses Git to determine whether any of the specified paths is, or contains, an untracked file that is not + * ignored by `.gitignore`, excluding the specified folders (typically the operation's output folders). + * If Git fails, conservatively returns true. + */ +export function hasUntrackedGitFiles( + gitPath: string, + rootDirectory: string, + candidatePaths: ReadonlyArray, + excludedFolderPaths: ReadonlyArray +): boolean { + const args: string[] = ['ls-files', '--others', '--exclude-standard', '-z', '--']; + for (const candidatePath of candidatePaths) { + args.push(`:(literal)${toGitPathspec(rootDirectory, candidatePath)}`); + } + for (const excludedFolderPath of excludedFolderPaths) { + args.push(`:(exclude,literal)${toGitPathspec(rootDirectory, excludedFolderPath)}`); + } + const result: ReturnType = Executable.spawnSync(gitPath, args, { + currentWorkingDirectory: rootDirectory + }); + if (result.status !== 0) { + return true; + } + return result.stdout.length > 0; +} \ No newline at end of file diff --git a/libraries/rush-lib/src/logic/operations/test/InputFilesStatSignature.test.ts b/libraries/rush-lib/src/logic/operations/test/InputFilesStatSignature.test.ts new file mode 100644 index 0000000000..0e0839c269 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/InputFilesStatSignature.test.ts @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as child_process from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { + captureInputFilesState, + getNewFolderEntries, + hasUntrackedGitFiles, + haveInputFilesChanged, + type IInputFilesState +} from '../InputFilesStatSignature'; + +describe('InputFilesStatSignature', () => { + let tempFolder: string; + let srcFolder: string; + let fileA: string; + let fileB: string; + let noNewInputs: jest.Mock]>; + + function capture(...absolutePaths: string[]): IInputFilesState { + return captureInputFilesState( + tempFolder, + absolutePaths.map((filePath: string) => path.relative(tempFolder, filePath)) + ); + } + + beforeEach(() => { + tempFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'rush-input-stat-'))); + srcFolder = path.join(tempFolder, 'src'); + fs.mkdirSync(srcFolder); + fileA = path.join(srcFolder, 'a.ts'); + fileB = path.join(srcFolder, 'b.ts'); + fs.writeFileSync(fileA, 'export const a = 1;'); + fs.writeFileSync(fileB, 'export const b = 1;'); + noNewInputs = jest.fn().mockReturnValue(false); + }); + + afterEach(() => { + fs.rmSync(tempFolder, { recursive: true, force: true }); + }); + + describe(haveInputFilesChanged.name, () => { + it('reports no change when the input files are unchanged', () => { + const state: IInputFilesState = capture(fileA, fileB); + expect(state.filePaths).toEqual([fileA, fileB]); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(false); + expect(noNewInputs).not.toHaveBeenCalled(); + }); + + it('detects a modified input file', () => { + const state: IInputFilesState = capture(fileA, fileB); + fs.writeFileSync(fileB, 'export const b = 2; // edited during the build'); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(true); + }); + + it('detects a same-size edit with a different modification time', () => { + const state: IInputFilesState = capture(fileA); + fs.writeFileSync(fileA, 'export const a = 2;'); + const future: Date = new Date(Date.now() + 60 * 1000); + fs.utimesSync(fileA, future, future); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(true); + }); + + it('detects a deleted input file', () => { + const state: IInputFilesState = capture(fileA, fileB); + fs.unlinkSync(fileA); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(true); + }); + + it('asks whether files created in an input folder are inputs', () => { + const state: IInputFilesState = capture(fileA, fileB); + const fileC: string = path.join(srcFolder, 'c.ts'); + fs.writeFileSync(fileC, 'export const c = 1;'); + + expect(haveInputFilesChanged(state, noNewInputs)).toBe(false); + expect(noNewInputs).toHaveBeenCalledWith([fileC]); + + expect(haveInputFilesChanged(state, () => true)).toBe(true); + }); + + it('reports a folder created in an input folder', () => { + const state: IInputFilesState = capture(fileA); + fs.mkdirSync(path.join(srcFolder, 'nested')); + fs.writeFileSync(path.join(srcFolder, 'nested', 'd.ts'), 'export const d = 1;'); + expect(getNewFolderEntries(state.folderEntries)).toEqual([path.join(srcFolder, 'nested')]); + }); + + it('ignores files created outside of the input folders', () => { + const state: IInputFilesState = capture(fileA); + fs.writeFileSync(path.join(tempFolder, 'unrelated.txt'), 'not an input'); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(false); + expect(noNewInputs).not.toHaveBeenCalled(); + }); + + it('resolves absolute input paths as-is and does not watch their folders', () => { + const state: IInputFilesState = captureInputFilesState(path.join(tempFolder, 'other-root'), [fileA]); + expect(state.filePaths).toEqual([fileA]); + expect(state.folderEntries.size).toBe(0); + fs.writeFileSync(fileA, 'export const a = 3; // edited during the build'); + expect(haveInputFilesChanged(state, noNewInputs)).toBe(true); + }); + }); + + describe(hasUntrackedGitFiles.name, () => { + const gitPath: string = 'git'; + + function git(...args: string[]): void { + child_process.execFileSync(gitPath, args, { cwd: tempFolder, stdio: 'ignore' }); + } + + beforeEach(() => { + git('init', '-q'); + fs.writeFileSync(path.join(tempFolder, '.gitignore'), 'temp/\n*.log\n'); + git('add', '-A'); + }); + + it('returns false for ignored files and excluded folders', () => { + fs.mkdirSync(path.join(srcFolder, 'temp')); + fs.writeFileSync(path.join(srcFolder, 'temp', 'x.ts'), ''); + fs.writeFileSync(path.join(srcFolder, 'build.log'), ''); + fs.mkdirSync(path.join(srcFolder, 'lib')); + fs.writeFileSync(path.join(srcFolder, 'lib', 'a.js'), ''); + + expect( + hasUntrackedGitFiles( + gitPath, + tempFolder, + [path.join(srcFolder, 'temp'), path.join(srcFolder, 'build.log'), path.join(srcFolder, 'lib')], + [path.join(srcFolder, 'lib')] + ) + ).toBe(false); + }); + + it('returns true for new untracked files and folders that are not ignored', () => { + fs.writeFileSync(path.join(srcFolder, 'c.ts'), ''); + fs.mkdirSync(path.join(srcFolder, 'nested')); + fs.writeFileSync(path.join(srcFolder, 'nested', 'd.ts'), ''); + + expect(hasUntrackedGitFiles(gitPath, tempFolder, [path.join(srcFolder, 'c.ts')], [])).toBe(true); + expect(hasUntrackedGitFiles(gitPath, tempFolder, [path.join(srcFolder, 'nested')], [])).toBe(true); + }); + }); +}); \ No newline at end of file