diff --git a/CHANGELOG.md b/CHANGELOG.md index dd9b356fb..489d6808e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Versioning](https://semver.org/spec/v2.0.0.html). version, shrinks by one entry per write, and Wireit reminds you once a day while it holds more than twice the limit. To free it at once, run `rm -rf .wireit/*/cache`. See [#71](https://github.com/google/wireit/issues/71). +- Set `WIREIT_CACHE_WORKTREES=true` to store a linked Git worktree's local + cache in the main worktree's corresponding `.wireit` folders. Any other value + keeps each worktree's own cache. Fingerprints and GitHub Actions caching are + unchanged. ### Fixed diff --git a/README.md b/README.md index 005b49f45..662e1008c 100644 --- a/README.md +++ b/README.md @@ -372,6 +372,19 @@ Note the limit is applied per script, so a package with many cached scripts will still use a multiple of this space. To free all of it at once, use `rm -rf .wireit/*/cache .wireit/*/temp .wireit/trash`. +Linked Git worktrees (`git worktree add`) keep their own `.wireit` cache +folders. Set `WIREIT_CACHE_WORKTREES=true` to store a linked worktree's local +cache in the main worktree's corresponding `.wireit` folders. Shared entries +are named from package-relative paths, so both worktrees hit the same entry. +Any other value keeps each worktree's own cache and the same entry names as +before. Worktrees of bare clones, submodules, and checkouts with a separate +git dir also keep their own cache, since the main git dir's parent isn't a +checkout. Lock, fingerprint, and output-manifest files stay in the current +worktree. Fingerprints and GitHub Actions caching are unchanged. + +Restored files are copied verbatim. Wireit does not rewrite output contents +that embed another checkout's absolute path. + ### GitHub Actions caching In _[GitHub Actions](https://github.com/features/actions)_ mode, Wireit caches @@ -895,6 +908,7 @@ The following environment variables affect the behavior of Wireit: | `WIREIT_FAILURES` | [How to handle script failures](#failures-and-errors).

Options:
| | `WIREIT_LOGGER` | How to present progress and results on the command line.

Options:
| | `WIREIT_CACHE_MAX_ENTRIES` | Maximum number of [local cache](#local-caching) entries to retain per script.

Defaults to `10`.

Must be a positive integer or `infinity`. | +| `WIREIT_CACHE_WORKTREES` | Set to `true` to store a linked Git worktree's [local cache](#local-caching) in the main worktree's corresponding `.wireit` folders.

Must be exactly `true`. If unset or any other value, interpreted as `false`. Does not affect fingerprints or GitHub Actions caching. | | `WIREIT_DEBUG_LOG_FILE` | Path to a file which will receive detailed event logging. | | `WIREIT_MAX_OPEN_FILES` | Limits the number of file descriptors Wireit will have open concurrently. Prevents resource exhaustion when checking large numbers of cached files. Set to a lower number if you hit file descriptor limits.

Defaults to `200`.

Must be a positive integer. | | `WIREIT_PARALLEL` | [Maximum number of scripts to run at one time](#parallelism).

Defaults to 2×logical CPU cores.

Must be a positive integer or `infinity`. | diff --git a/package.json b/package.json index 9f7935f16..9f5113d97 100644 --- a/package.json +++ b/package.json @@ -183,7 +183,7 @@ "output": [] }, "test:local-cache": { - "command": "node --test --test-reporter=dot lib/test/local-cache.test.js", + "command": "node --test --test-reporter=dot lib/test/local-cache.test.js lib/test/fingerprint.test.js lib/test/cache-root.test.js", "env": { "NODE_OPTIONS": "--enable-source-maps" }, diff --git a/src/caching/cache.ts b/src/caching/cache.ts index cf78abb51..1adf9f7cd 100644 --- a/src/caching/cache.ts +++ b/src/caching/cache.ts @@ -94,4 +94,7 @@ export interface CacheHit { * before this method is called. */ apply(): Promise; + + /** Release resources held by this hit when it won't be applied. */ + release(): Promise; } diff --git a/src/caching/github-actions-cache.ts b/src/caching/github-actions-cache.ts index 41b697a14..6457ab7e8 100644 --- a/src/caching/github-actions-cache.ts +++ b/src/caching/github-actions-cache.ts @@ -759,6 +759,10 @@ class GitHubActionsCacheHit implements CacheHit { await this.#extract(tarballPath); } + async release(): Promise { + // This cache hit holds no resources before apply. + } + async #download(tarballPath: string): Promise { using requestResult = request(this.#url); const {req, resPromise} = requestResult; diff --git a/src/caching/local-cache.ts b/src/caching/local-cache.ts index ef543be84..d27c9952a 100644 --- a/src/caching/local-cache.ts +++ b/src/caching/local-cache.ts @@ -6,10 +6,12 @@ import * as fs from '../util/fs.js'; import * as pathlib from 'path'; -import {createHash, randomBytes} from 'crypto'; +import {randomBytes} from 'crypto'; +import lockfile from 'proper-lockfile'; import {getPackageDataDir, getScriptDataDir} from '../util/script-data-dir.js'; import {copyEntries} from '../util/copy.js'; import {glob} from '../util/glob.js'; +import {resolveCachePackageDir} from '../util/cache-root.js'; import type {Cache, CacheHit} from './cache.js'; import type {ScriptReference} from '../config.js'; @@ -45,6 +47,36 @@ const REMIND_OVER_LIMIT_FACTOR = 2; /** How often to remind the user about the same package. */ const REMIND_OVER_LIMIT_EVERY_MS = 24 * 60 * 60 * 1000; +/** + * The minimum age of a leftover temp folder before Wireit trashes it. A + * younger folder may belong to a save that is still running in another + * worktree. + */ +const TEMP_LEFTOVER_MIN_AGE_MS = 60 * 60 * 1000; + +/** The name of the lock that an eviction takes in an entry's lock folder. */ +const EVICT_LOCK = 'evict'; + +/** + * The prefix of each restore's lock name. The rest of the name is random. So + * two restores never take the same lock. + */ +const RESTORE_LOCK_PREFIX = 'restore-'; + +const ENTRY_LOCK_OPTIONS = { + // A crashed holder stops updating mtime, so the lock becomes acquirable. + // Same values as the script lock. + stale: 10_000, + update: 2000, + retries: 0, + // Don't resolve the lock's real path. By default proper-lockfile resolves + // it, and that needs a file at the path. Wireit would then have to create + // an empty file for each entry and leave it behind. With this option off, + // no file is needed. The lock is a ".lock" folder, and releasing the lock + // deletes the folder. + realpath: false, +}; + /** * Caches script output to each package's * ".wireit//cache/" folder, keeping only @@ -56,18 +88,31 @@ const REMIND_OVER_LIMIT_EVERY_MS = 24 * 60 * 60 * 1000; * Entries are copied into the script's "temp" folder and then renamed into * place, so a killed Wireit can't leave a partial entry. * - * Eviction needs no lock of its own: it touches only the calling script's cache - * folder, and StandardScriptExecution#acquireSystemLockIfNeeded already holds - * that script's lock, except for an empty "output", where the entries are empty - * directories. The lock also means anything in the script's temp folder was - * left by a killed or failed write, not one in progress. Sweeping is - * deliberately unlocked, so any number of Wireit processes can empty the same - * trash at once and a vanished entry is expected. + * A linked worktree with WIREIT_CACHE_WORKTREES stores its cache in the main + * worktree's cache folder. It does this even when the main worktree does not + * set the flag. The worktrees share the folder, but each worktree has its own + * script lock. So every cache follows two rules, with or without the flag: + * + * - Each entry has locks in a "cache-locks" folder next to the cache. A + * restore holds a lock while it copies. Eviction skips an entry while a + * restore holds a lock on it. So eviction never deletes an entry during a + * restore. + * - Wireit trashes a leftover in the script's "temp" folder only when the + * leftover is over an hour old. A newer leftover may be a save that is still + * running in another worktree. + * + * Eviction needs no other lock. It only changes the calling script's cache + * folder. Within one worktree, the script's lock already covers that folder. + * StandardScriptExecution#acquireSystemLockIfNeeded takes that lock. The one + * exception is an empty "output", whose entries are empty directories. + * Sweeping takes no lock on purpose. Any number of Wireit processes can empty + * the same trash at once. So an entry can disappear during a sweep. */ export class LocalCache implements Cache { readonly #maxEntries: number; + readonly #shareWorktrees: boolean; - /** Packages used this run, whose trash {@link sweepTrash} empties. */ + /** Cache package dirs used this run, whose trash {@link sweepTrash} empties. */ readonly #packageDirs = new Set(); /** Messages for the user, which the next {@link sweepTrash} returns. */ @@ -79,38 +124,94 @@ export class LocalCache implements Cache { */ readonly #remindedPackages = new Set(); - /** @param maxEntries Entries to retain per script, or Infinity for all. */ - constructor(maxEntries: number) { + /** + * Maps each package folder to the package that holds its cache. Finding + * that package reads several git files, so the cache finds it once per run. + */ + readonly #cachePackageDirs = new Map>(); + + /** + * @param maxEntries Entries to retain per script, or Infinity for all. + * @param options When `shareWorktrees` is true, linked git worktrees write + * cache into the main worktree's matching `.wireit`. + */ + constructor(maxEntries: number, options?: {shareWorktrees?: boolean}) { this.#maxEntries = maxEntries; + this.#shareWorktrees = options?.shareWorktrees === true; } async get( script: ScriptReference, fingerprint: Fingerprint, ): Promise { - const cacheDir = this.#getCacheDir(script, fingerprint); + const cachePackageDir = await this.#cachePackageDir(script); + const entryName = fingerprint.localCacheEntryName(this.#shareWorktrees); + const cacheDir = pathlib.join( + this.#getScriptCacheDir(cachePackageDir, script), + entryName, + ); + // A miss takes no lock. + if (!(await exists(cacheDir))) { + return undefined; + } + // Hold a restore lock during the restore. Then eviction in another + // worktree cannot delete the entry while it is being copied. Check that + // the entry still exists after taking the lock. It may have been evicted + // while this run waited. + let release: () => Promise; try { - await fs.access(cacheDir); + release = await this.#lockForRestore( + this.#getEntryLockDir(cachePackageDir, script, entryName), + ); } catch (error) { - if ((error as Error & {code?: string}).code === 'ENOENT') { - return; + if (!cannotWrite(error)) { + throw error; + } + // The cache folder is read-only, such as on a read-only mount. Restore + // without a lock. No run can evict from a folder it cannot write. + release = async () => {}; + } + try { + try { + await fs.access(cacheDir); + } catch (error) { + if ((error as Error & {code?: string}).code === 'ENOENT') { + await release().catch(() => {}); + return; + } + throw error; } + this.#packageDirs.add(cachePackageDir); + const now = new Date(); + try { + await fs.utimes(cacheDir, now, now); + } catch { + // No entry, or one we can't stamp (read-only mount, foreign owner). + // A hit is still a hit; the entry just ages as though only ever + // written. + } + return new LocalCacheHit(cacheDir, script.packageDir, release); + } catch (error) { + await release().catch(() => {}); throw error; } - await this.markEntryRecentlyUsed(script, fingerprint); - return new LocalCacheHit(cacheDir, script.packageDir); } async markEntryRecentlyUsed( script: ScriptReference, fingerprint: Fingerprint, ): Promise { - this.#packageDirs.add(script.packageDir); + const cachePackageDir = await this.#cachePackageDir(script); + this.#packageDirs.add(cachePackageDir); // Recency lives in the mtime, so there is no index file to maintain. atime // won't do, because filesystems are commonly mounted noatime or relatime. const now = new Date(); try { - await fs.utimes(this.#getCacheDir(script, fingerprint), now, now); + await fs.utimes( + this.#getCacheDir(cachePackageDir, script, fingerprint), + now, + now, + ); } catch { // No entry, or one we can't stamp (read-only mount, foreign owner). A hit // is still a hit; the entry just ages as though only ever written. @@ -122,42 +223,53 @@ export class LocalCache implements Cache { fingerprint: Fingerprint, absoluteFiles: AbsoluteEntry[], ): Promise { - this.#packageDirs.add(script.packageDir); - const absCacheDir = this.#getCacheDir(script, fingerprint); + const cachePackageDir = await this.#cachePackageDir(script); + this.#packageDirs.add(cachePackageDir); + const absCacheDir = this.#getCacheDir(cachePackageDir, script, fingerprint); if (absoluteFiles.length === 0) { // No temp folder, because an empty "output" runs without the lock. // - // Note fs.mkdir returns the first created directory, or undefined if no - // directory was created. - const existed = - (await fs.mkdir(absCacheDir, {recursive: true})) === undefined; - if (existed) { - // This is an unexpected error because the Executor should already have - // checked for an existing cache hit. - throw new Error(`Did not expect ${absCacheDir} to already exist.`); - } - await this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)); - return true; + // The entry may already exist. Another worktree may have saved the same + // empty output first. Then this save counts as done. + await fs.mkdir(absCacheDir, {recursive: true}); + } else { + await this.#writeThroughTemp( + cachePackageDir, + script, + absoluteFiles, + absCacheDir, + ); + // This must run after this write's rename. Another worktree may be + // saving into the same temp folder. So only leftovers over an hour old + // are trashed. + await this.#trashLeftoverTemp(cachePackageDir, script); } - await this.#writeThroughTemp(script, absoluteFiles, absCacheDir); - await Promise.all([ - this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)), - this.#trashLeftoverTemp(script), - ]); + await this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)); return true; } + /** + * Copies into a temp folder and renames the copy into place. So a killed + * Wireit can't leave a partial entry. Another worktree may save the same + * entry at the same time, and only one rename can succeed. If the entry + * already exists, the other save won, and this save counts as done. + */ async #writeThroughTemp( + cachePackageDir: string, script: ScriptReference, absoluteFiles: AbsoluteEntry[], absCacheDir: string, ): Promise { // Short, so a path that fits the Windows limit in the cache fits here. + // Next to the cache entry, so the rename stays on one filesystem when a + // linked worktree's cache lives in the main worktree. const tempDir = pathlib.join( - this.#getScriptTempDir(script), + this.#getScriptTempDir(cachePackageDir, script), randomBytes(8).toString('hex'), ); - await fs.mkdir(this.#getScriptCacheDir(script), {recursive: true}); + await fs.mkdir(this.#getScriptCacheDir(cachePackageDir, script), { + recursive: true, + }); try { await copyEntries(absoluteFiles, script.packageDir, tempDir); await fs.rename(tempDir, absCacheDir); @@ -165,22 +277,49 @@ export class LocalCache implements Cache { // Not moved to the trash, because creating the trash folder fails on a // full disk. await fs.rmTree(tempDir).catch(() => {}); + const {code} = error as {code?: string}; + if (code === 'EEXIST' || code === 'ENOTEMPTY' || code === 'EPERM') { + try { + await fs.access(absCacheDir); + return; + } catch { + throw error; + } + } throw error; } } - /** Needs the script's lock, and must run after this write's rename. */ - async #trashLeftoverTemp(script: ScriptReference): Promise { - const tempDir = this.#getScriptTempDir(script); + /** Must run after this write's rename; see {@link set}. */ + async #trashLeftoverTemp( + cachePackageDir: string, + script: ScriptReference, + ): Promise { + const tempDir = this.#getScriptTempDir(cachePackageDir, script); let leftovers; try { leftovers = await fs.readdir(tempDir, {withFileTypes: true}); } catch { return; } + // Another worktree may be saving into this folder right now. Trash only + // the folders that are old enough to be left from an aborted save. + const cutoff = Date.now() - TEMP_LEFTOVER_MIN_AGE_MS; + const names = ( + await Promise.all( + leftovers.map(async ({name}) => { + try { + const {mtimeMs} = await fs.lstat(pathlib.join(tempDir, name)); + return mtimeMs <= cutoff ? name : undefined; + } catch { + return undefined; + } + }), + ) + ).filter((name): name is string => name !== undefined); await Promise.allSettled( - leftovers.map((entry) => - this.#moveToTrash(script.packageDir, pathlib.join(tempDir, entry.name)), + names.map((name) => + this.#moveToTrash(cachePackageDir, pathlib.join(tempDir, name)), ), ); } @@ -219,7 +358,8 @@ export class LocalCache implements Cache { return; } try { - const cacheDir = this.#getScriptCacheDir(script); + const cachePackageDir = await this.#cachePackageDir(script); + const cacheDir = this.#getScriptCacheDir(cachePackageDir, script); const entries = await fs.readdir(cacheDir, {withFileTypes: true}); if (entries.length <= this.#maxEntries) { return; @@ -236,18 +376,44 @@ export class LocalCache implements Cache { })), ); byRecency.sort((a, b) => a.mtimeMs - b.mtimeMs); - const doomed = byRecency.slice( - 0, - Math.min(entries.length - this.#maxEntries, MAX_EVICTIONS_PER_WRITE), - ); - // allSettled, so one entry we can't move (EPERM on Windows, while - // something holds it open) doesn't block evicting the rest. - await Promise.allSettled( - doomed.map(({path}) => this.#moveToTrash(script.packageDir, path)), + // Skip an entry that another run is restoring or evicting. Deleting an + // entry during a restore would corrupt the restore. Evict the next + // oldest entry instead. If one entry fails to move, continue with the + // rest. For example, Windows fails with EPERM while a program has the + // entry open. + const quota = Math.min( + entries.length - this.#maxEntries, + MAX_EVICTIONS_PER_WRITE, ); - const numLeft = entries.length - doomed.length; + let evicted = 0; + for (const {path} of byRecency) { + if (evicted >= quota) { + break; + } + const release = await this.#lockForEviction( + this.#getEntryLockDir( + cachePackageDir, + script, + pathlib.basename(path), + ), + ); + if (release === undefined) { + continue; + } + try { + await this.#moveToTrash(cachePackageDir, path); + evicted++; + } catch { + // Housekeeping: the entry just written is still valid. + } finally { + await release().catch(() => {}); + } + } + const numLeft = entries.length - evicted; if (numLeft > REMIND_OVER_LIMIT_FACTOR * this.#maxEntries) { - await this.#remindOverLimit(script.packageDir); + // Pass the package that holds the cache. With WIREIT_CACHE_WORKTREES, + // a linked worktree's cache is in the main worktree. + await this.#remindOverLimit(cachePackageDir); } } catch { // See above. @@ -350,22 +516,187 @@ export class LocalCache implements Cache { return pathlib.join(getPackageDataDir(packageDir), 'trash'); } - #getScriptCacheDir(script: ScriptReference): string { - return pathlib.join(getScriptDataDir(script), 'cache'); + /** + * Returns the folder of locks for one cache entry. The folder is next to + * the cache folder, so evicting or sweeping the entry never moves the locks. + * + * Each restore takes its own lock in the folder. Its name is + * {@link RESTORE_LOCK_PREFIX} followed by random hex digits. So restores in + * different worktrees never wait for each other. An eviction takes the + * {@link EVICT_LOCK} lock. It evicts the entry only if no restore holds a + * lock. When a restore finds {@link EVICT_LOCK} held, it releases its own + * lock and tries again. + * + * Each side takes its own lock before it checks for the other side's lock. + * This order makes sure that at least one side sees the other. + * + * Each lock is a ".lock" folder. Releasing the last lock also deletes the + * entry's lock folder. So nothing is left when no lock is held. + */ + #getEntryLockDir( + cachePackageDir: string, + script: ScriptReference, + entryName: string, + ): string { + return pathlib.join( + getScriptDataDir({packageDir: cachePackageDir, name: script.name}), + 'cache-locks', + entryName, + ); } - #getScriptTempDir(script: ScriptReference): string { - return pathlib.join(getScriptDataDir(script), 'temp'); + /** Takes a restore lock. Waits while an eviction of the entry is running. */ + async #lockForRestore(lockDir: string): Promise<() => Promise> { + while (true) { + const release = await this.#lockIn( + lockDir, + RESTORE_LOCK_PREFIX + randomBytes(8).toString('hex'), + ); + if ( + !(await lockfile.check( + pathlib.join(lockDir, EVICT_LOCK), + ENTRY_LOCK_OPTIONS, + )) + ) { + return release; + } + await release(); + await new Promise((resolve) => setTimeout(resolve, 200)); + } } - #getCacheDir(script: ScriptReference, fingerprint: Fingerprint): string { + /** + * Takes the eviction lock. Returns undefined if another eviction holds it. + * Also returns undefined if a restore holds a lock. + */ + async #lockForEviction( + lockDir: string, + ): Promise<(() => Promise) | undefined> { + let release; + try { + release = await this.#lockIn(lockDir, EVICT_LOCK); + } catch (error) { + if ((error as {code?: string}).code === 'ELOCKED') { + return undefined; + } + throw error; + } + let names: string[]; + try { + names = (await fs.readdir(lockDir, {withFileTypes: true})).map( + (entry) => entry.name, + ); + } catch (error) { + await release(); + throw error; + } + for (const name of names) { + if (!name.startsWith(RESTORE_LOCK_PREFIX) || !name.endsWith('.lock')) { + continue; + } + const restoreLock = pathlib.join(lockDir, name.slice(0, -'.lock'.length)); + if (await lockfile.check(restoreLock, ENTRY_LOCK_OPTIONS)) { + await release(); + return undefined; + } + // This lock is stale. A restore crashed while it held the lock. Delete + // the lock so that the folder can be deleted later. + await fs.rmdir(pathlib.join(lockDir, name)).catch(() => {}); + } + return release; + } + + /** + * Takes the lock named {@link name} in {@link lockDir}. The returned + * function releases the lock. It also deletes the folder if the folder holds + * no other lock. + */ + async #lockIn(lockDir: string, name: string): Promise<() => Promise> { + while (true) { + await fs.mkdir(lockDir, {recursive: true}); + let release; + try { + release = await lockfile.lock( + pathlib.join(lockDir, name), + ENTRY_LOCK_OPTIONS, + ); + } catch (error) { + // Another run released its lock and deleted the folder after this run + // created it. Create the folder again. + if ((error as {code?: string}).code === 'ENOENT') { + continue; + } + throw error; + } + return async () => { + await release(); + await fs.rmdir(lockDir).catch(() => {}); + }; + } + } + + async #cachePackageDir(script: ScriptReference): Promise { + if (!this.#shareWorktrees) { + return script.packageDir; + } + const {packageDir} = script; + let cachePackageDir = this.#cachePackageDirs.get(packageDir); + if (cachePackageDir === undefined) { + cachePackageDir = resolveCachePackageDir(packageDir, { + shareWorktrees: true, + }); + this.#cachePackageDirs.set(packageDir, cachePackageDir); + // Forget a failed lookup. The next call tries again. + cachePackageDir.catch(() => this.#cachePackageDirs.delete(packageDir)); + } + return cachePackageDir; + } + + #getScriptCacheDir(cachePackageDir: string, script: ScriptReference): string { + return pathlib.join( + getScriptDataDir({packageDir: cachePackageDir, name: script.name}), + 'cache', + ); + } + + #getScriptTempDir(cachePackageDir: string, script: ScriptReference): string { + return pathlib.join( + getScriptDataDir({packageDir: cachePackageDir, name: script.name}), + 'temp', + ); + } + + #getCacheDir( + cachePackageDir: string, + script: ScriptReference, + fingerprint: Fingerprint, + ): string { return pathlib.join( - this.#getScriptCacheDir(script), - createHash('sha256').update(fingerprint.string).digest('hex'), + this.#getScriptCacheDir(cachePackageDir, script), + fingerprint.localCacheEntryName(this.#shareWorktrees), ); } } +/** Whether a call failed because the file system refused to write. */ +const cannotWrite = (error: unknown) => { + const code = (error as {code?: string}).code; + return code === 'EACCES' || code === 'EPERM' || code === 'EROFS'; +}; + +/** Whether {@link path} exists. */ +const exists = async (path: string): Promise => { + try { + await fs.access(path); + return true; + } catch (error) { + if ((error as {code?: string}).code === 'ENOENT') { + return false; + } + throw error; + } +}; + /** * Whether a delete may have failed because another program has the file open. * Windows refuses to delete an open file, with EBUSY, or with EPERM for some @@ -389,20 +720,41 @@ class LocalCacheHit implements CacheHit { */ readonly #destination: string; - constructor(source: string, destination: string) { + /** + * Releases the entry's lock. {@link apply} calls it when the copy is done. + * Until then, eviction in another worktree cannot delete the entry. + */ + #release: (() => Promise) | undefined; + + constructor( + source: string, + destination: string, + release: () => Promise, + ) { this.#source = source; this.#destination = destination; + this.#release = release; } async apply(): Promise { - const entries = await glob(['**'], { - cwd: this.#source, - followSymlinks: false, - includeDirectories: true, - expandDirectories: true, - // Shouldn't ever happen, but would be really weird. - throwIfOutsideCwd: true, - }); - await copyEntries(entries, this.#source, this.#destination); + try { + const entries = await glob(['**'], { + cwd: this.#source, + followSymlinks: false, + includeDirectories: true, + expandDirectories: true, + // Shouldn't ever happen, but would be really weird. + throwIfOutsideCwd: true, + }); + await copyEntries(entries, this.#source, this.#destination); + } finally { + await this.release(); + } + } + + async release(): Promise { + const release = this.#release; + this.#release = undefined; + await release?.().catch(() => {}); } } diff --git a/src/cli-options.ts b/src/cli-options.ts index 282ee3efb..6f1ebe76d 100644 --- a/src/cli-options.ts +++ b/src/cli-options.ts @@ -64,6 +64,7 @@ export interface Options { numWorkers: number; cache: 'local' | 'github' | 'none'; cacheMaxEntries: number; + cacheWorktrees: boolean; failureMode: FailureMode; agent: Agent; logger: Logger; @@ -220,6 +221,9 @@ export const getOptions = async (): Promise> => { } } + // Same rule as CI: only the exact string "true" enables it. + const cacheWorktrees = process.env['WIREIT_CACHE_WORKTREES'] === 'true'; + const failureModeResult = ((): Result => { const str = process.env['WIREIT_FAILURES']; if (!str) { @@ -307,6 +311,7 @@ export const getOptions = async (): Promise> => { numWorkers: numWorkersResult.value, cache: cacheResult.value, cacheMaxEntries: cacheMaxEntriesResult.value, + cacheWorktrees, failureMode: failureModeResult.value, agent, logger, diff --git a/src/cli.ts b/src/cli.ts index 58c133ef0..dbb2860ab 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -62,7 +62,9 @@ const run = async (options: Options): Promise> => { case 'local': { // Import dynamically so that we import fewer unnecessary modules. const {LocalCache} = await import('./caching/local-cache.js'); - cache = new LocalCache(options.cacheMaxEntries); + cache = new LocalCache(options.cacheMaxEntries, { + shareWorktrees: options.cacheWorktrees, + }); break; } case 'github': { diff --git a/src/execution/standard.ts b/src/execution/standard.ts index d090cf9b4..8c1947a55 100644 --- a/src/execution/standard.ts +++ b/src/execution/standard.ts @@ -126,14 +126,18 @@ export class StandardScriptExecution extends BaseExecutionWithCommand = []; + const portableDependencyFingerprints: Array< + [ScriptReferenceString, string] + > = []; for (const [dep, depFingerprint] of dependencyFingerprints) { if (!dep.cascade) { // cascade: false means the fingerprint of the dependency isn't @@ -151,6 +155,19 @@ export class Fingerprint { scriptReferenceToString(dep.config), depFingerprint.hash, ]); + // Package-relative, and hashed from the dependency's portable form, so + // two worktrees of one repo share a local-cache entry. This is not part + // of {@link Fingerprint.string}. + portableDependencyFingerprints.push([ + scriptReferenceToString({ + packageDir: pathRelativeToPackage( + script.packageDir, + dep.config.packageDir, + ), + name: dep.config.name, + }), + depFingerprint.localCacheEntryName(true), + ]); } let fileHashes: Array<[string, FileSha256HexDigest]>; @@ -193,7 +210,7 @@ export class Fingerprint { } erroredFilePaths.push(absolutePath); } - return [file.path, hash.digest('hex') as FileSha256HexDigest]; + return [absolutePath, hash.digest('hex') as FileSha256HexDigest]; }), ); @@ -258,12 +275,36 @@ export class Fingerprint { env: script.env, }; fingerprint.#data = data as FingerprintData; + fingerprint.#portableString = JSON.stringify({ + ...data, + files: Object.fromEntries( + fileHashes + .map(([filePath, hash]): [string, FileSha256HexDigest] => [ + pathRelativeToPackage(script.packageDir, filePath), + hash, + ]) + .sort(([aFile], [bFile]) => aFile.localeCompare(bFile)), + ), + dependencies: Object.fromEntries( + portableDependencyFingerprints.sort(([aRef], [bRef]) => + aRef.localeCompare(bRef), + ), + ), + }); return {ok: true, value: fingerprint}; } #str?: FingerprintString; #data?: FingerprintData; #hash?: FingerprintSha256HexDigest; + /** + * Same fields as {@link string}, but file paths and dependency package dirs + * are package-relative and dependency hashes come from each dependency's + * portable form. Set only by {@link Fingerprint.compute}. Local-cache entry + * names use this when worktree sharing is on. Freshness and GitHub Actions + * caching keep using {@link string}. + */ + #portableString?: string; get string(): FingerprintString { if (this.#str === undefined) { @@ -288,7 +329,31 @@ export class Fingerprint { return this.#hash; } + /** + * Directory name of a local-cache entry. + * + * With worktree sharing off, this is {@link hash}, so existing cache folders + * stay valid. With sharing on, it is the hash of {@link #portableString}. + * A fingerprint built with {@link Fingerprint.fromString} has no portable + * form and falls back to {@link hash}. + */ + localCacheEntryName(shareWorktrees: boolean): string { + const source = + shareWorktrees && this.#portableString !== undefined + ? this.#portableString + : this.string; + return source === this.string + ? this.hash + : createHash('sha256').update(source).digest('hex'); + } + equal(other: Fingerprint): boolean { return this.string === other.string; } } + +/** Package-relative so a shared local-cache entry matches across worktrees. */ +const pathRelativeToPackage = (packageDir: string, path: string): string => { + const relative = pathlib.relative(packageDir, path); + return relative === '' ? '.' : relative; +}; diff --git a/src/test/cache-local.test.ts b/src/test/cache-local.test.ts index 635a8094e..d999279d3 100644 --- a/src/test/cache-local.test.ts +++ b/src/test/cache-local.test.ts @@ -562,7 +562,16 @@ void test( crashed.kill('SIGKILL'); await crashed.exit; assert.deepEqual(await cacheEntriesIfAny(rig, 'a'), []); - assert.equal((await tempEntries(rig)).length, 1); + const leftovers = await tempEntries(rig); + assert.equal(leftovers.length, 1); + // Make the leftover two hours old. A write trashes only leftovers over an + // hour old, because a newer one may be a save in another worktree. + const hoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + await fs.utimes( + pathlib.join(dataDir(rig, 'a'), 'temp', leftovers[0]!), + hoursAgo, + hoursAgo, + ); rig.env = {...rig.env, WIREIT_TEST_FS_GATE: undefined}; // Otherwise the killed run's lock takes 10 seconds to go stale. diff --git a/src/test/cache-root.test.ts b/src/test/cache-root.test.ts new file mode 100644 index 000000000..fae8109c9 --- /dev/null +++ b/src/test/cache-root.test.ts @@ -0,0 +1,204 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {test} from 'node:test'; +import * as assert from 'node:assert'; +import * as pathlib from 'path'; +import {realpath} from '../util/fs.js'; +import {detectWorktree, resolveCachePackageDir} from '../util/cache-root.js'; +import {FilesystemTestRig} from './util/filesystem-test-rig.js'; +import {git, initRepo} from './util/git.js'; + +void test('main worktree uses its own package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + initRepo(rig.temp); + await rig.write('README.md', 'x'); + git(rig.temp, ['add', '.']); + git(rig.temp, ['commit', '-m', 'init']); + const pkg = rig.resolve('packages/foo'); + await rig.mkdir('packages/foo'); + const info = await detectWorktree(pkg); + assert.ok(info); + assert.equal(await realpath(info.worktreeRoot), await realpath(rig.temp)); + assert.equal(await realpath(info.mainWorktreeRoot), await realpath(rig.temp)); + assert.equal(await resolveCachePackageDir(pkg, {shareWorktrees: true}), pkg); +}); + +void test('linked worktree keeps its own cache package dir unless sharing', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + assert.equal(await resolveCachePackageDir(linkedPkg), linkedPkg); +}); + +void test('linked worktree cache package dir maps onto the main tree when sharing', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + const mainPkg = pathlib.join(main, 'packages', 'foo'); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.write( + pathlib.join('main', 'packages', 'foo', 'package.json'), + '{}', + ); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + const info = await detectWorktree(linkedPkg); + assert.ok(info); + assert.equal(await realpath(info.worktreeRoot), await realpath(linked)); + assert.equal(await realpath(info.mainWorktreeRoot), await realpath(main)); + assert.equal( + await realpath( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + ), + await realpath(mainPkg), + ); +}); + +void test('worktree inside main checkout maps cache into matching package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = pathlib.join(main, '.claude', 'worktrees', 'linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + await rig.mkdir(pathlib.join('main', '.claude', 'worktrees')); + git(main, ['worktree', 'add', linked, '-b', 'other']); + const mainPkg = pathlib.join(main, 'packages', 'foo'); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.write( + pathlib.join('main', 'packages', 'foo', 'package.json'), + '{}', + ); + await rig.mkdir( + pathlib.join('.claude', 'worktrees', 'linked', 'packages', 'foo'), + ); + + assert.equal( + await realpath( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + ), + await realpath(mainPkg), + ); +}); + +void test('linked worktree keeps its own cache for a package the main worktree lacks', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + // Only the linked worktree's branch has this package. + const linkedPkg = pathlib.join(linked, 'packages', 'bar'); + await rig.write( + pathlib.join('linked', 'packages', 'bar', 'package.json'), + '{}', + ); + + assert.equal( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + linkedPkg, + ); + assert.equal(await rig.exists(pathlib.join('main', 'packages')), false); +}); + +void test('bare clone worktree keeps its own cache package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const bare = rig.resolve('bare.git'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(rig.temp, ['clone', '--bare', main, bare]); + git(rig.temp, ['--git-dir', bare, 'worktree', 'add', linked, '-b', 'other']); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + // The bare dir's parent isn't a checkout, so sharing would land outside it. + assert.equal( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + linkedPkg, + ); +}); + +void test('separate git dir worktree keeps its own cache package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const gitDir = rig.resolve('sep-git'); + const checkout = rig.resolve('checkout'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(rig.temp, ['clone', '--separate-git-dir', gitDir, main, checkout]); + git(checkout, ['worktree', 'add', linked, '-b', 'other']); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + // The git dir isn't named `.git`, so its parent isn't a checkout. + assert.equal( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + linkedPkg, + ); +}); + +void test('submodule worktree keeps its own cache package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + const subSrc = rig.resolve('sub-src'); + const superRepo = rig.resolve('super'); + const linked = rig.resolve('linked'); + for (const dir of [subSrc, superRepo]) { + await rig.mkdir(pathlib.relative(rig.temp, dir)); + initRepo(dir); + } + await rig.write(pathlib.join('sub-src', 'f.txt'), 'x'); + git(subSrc, ['add', '.']); + git(subSrc, ['commit', '-m', 'init']); + await rig.write(pathlib.join('super', 'README.md'), 'x'); + git(superRepo, ['add', '.']); + git(superRepo, ['commit', '-m', 'init']); + git(superRepo, [ + '-c', + 'protocol.file.allow=always', + 'submodule', + 'add', + subSrc, + 'sub', + ]); + git(superRepo, ['commit', '-m', 'addsub']); + const sub = pathlib.join(superRepo, 'sub'); + git(sub, ['worktree', 'add', linked, '-b', 'other']); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + // The shared config sets `core.worktree`, so sharing would land inside the + // outer repo's git folder. + assert.equal( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + linkedPkg, + ); +}); diff --git a/src/test/cli-options.test.ts b/src/test/cli-options.test.ts index ea81c9b94..6c5e6ad2b 100644 --- a/src/test/cli-options.test.ts +++ b/src/test/cli-options.test.ts @@ -60,6 +60,7 @@ async function assertOptions( watch: false, cache: 'local', cacheMaxEntries: 10, + cacheWorktrees: false, numWorkers: 10, failureMode: 'no-new', logger: 'QuietLogger', @@ -607,4 +608,46 @@ for (const {agent, runCmd, testCmd, startCmd, needsExtraDashes} of commands) { }), ); } + + void test( + `${agent} WIREIT_CACHE_WORKTREES=true`, + rigTest(async ({rig}) => { + await assertOptions( + rig, + `${runCmd} main ${extraDashes}`, + { + agent, + script: { + packageDir: rig.temp, + name: 'main', + }, + cacheWorktrees: true, + }, + { + WIREIT_CACHE_WORKTREES: 'true', + }, + ); + }), + ); + + void test( + `${agent} WIREIT_CACHE_WORKTREES=yes is false`, + rigTest(async ({rig}) => { + await assertOptions( + rig, + `${runCmd} main ${extraDashes}`, + { + agent, + script: { + packageDir: rig.temp, + name: 'main', + }, + cacheWorktrees: false, + }, + { + WIREIT_CACHE_WORKTREES: 'yes', + }, + ); + }), + ); } diff --git a/src/test/fingerprint.test.ts b/src/test/fingerprint.test.ts new file mode 100644 index 000000000..d7ff0d04c --- /dev/null +++ b/src/test/fingerprint.test.ts @@ -0,0 +1,105 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {test} from 'node:test'; +import * as assert from 'node:assert'; +import * as pathlib from 'path'; +import {Fingerprint} from '../fingerprint.js'; +import {scriptReferenceToString} from '../config.js'; +import {FilesystemTestRig} from './util/filesystem-test-rig.js'; + +import type {ArrayNode, JsonAstNode} from '../util/ast.js'; +import type {Dependency, StandardScriptConfig} from '../config.js'; +import type {FingerprintString} from '../fingerprint.js'; + +const stringNode = (value: string): JsonAstNode => ({ + type: 'string', + offset: 0, + length: value.length, + value, +}); + +const arrayNode = (values: string[]): ArrayNode => ({ + node: {type: 'array', offset: 0, length: 0, value: undefined}, + values, +}); + +const scriptAt = (packageDir: string): StandardScriptConfig => ({ + packageDir, + name: 'compile', + state: 'valid', + command: stringNode('tsc'), + extraArgs: [], + clean: true, + files: arrayNode(['src/a.ts', 'input.txt']), + output: arrayNode(['lib/**']), + service: undefined, + env: {}, + dependencies: [], + services: [], + scriptAstNode: undefined, + configAstNode: undefined, + declaringFile: { + path: pathlib.join(packageDir, 'package.json'), + contents: '{}', + }, + failures: [], +}); + +const depAt = ( + packageDir: string, + fingerprint: Fingerprint, +): [Dependency, Fingerprint] => [ + { + cascade: true, + config: scriptAt(packageDir), + specifier: stringNode('compile'), + }, + fingerprint, +]; + +void test('fingerprint stays absolute; shared cache entry name matches', async () => { + await using rig = await FilesystemTestRig.setup(); + await rig.write({ + 'a/packages/foo/src/a.ts': 'export const a = 1;', + 'a/packages/foo/input.txt': 'v0', + 'b/packages/foo/src/a.ts': 'export const a = 1;', + 'b/packages/foo/input.txt': 'v0', + }); + const depFingerprint = Fingerprint.fromString( + '{"fullyTracked":true}' as FingerprintString, + ); + const fooA = rig.resolve('a/packages/foo'); + const fooB = rig.resolve('b/packages/foo'); + const resultA = await Fingerprint.compute(scriptAt(fooA), [ + depAt(rig.resolve('a/packages/dep'), depFingerprint), + ]); + const resultB = await Fingerprint.compute(scriptAt(fooB), [ + depAt(rig.resolve('b/packages/dep'), depFingerprint), + ]); + assert.ok(resultA.ok); + assert.ok(resultB.ok); + assert.notEqual(resultA.value.string, resultB.value.string); + assert.deepEqual( + Object.keys(resultA.value.data.files).sort(), + [pathlib.join(fooA, 'src', 'a.ts'), pathlib.join(fooA, 'input.txt')].sort(), + ); + assert.deepEqual(Object.keys(resultA.value.data.dependencies), [ + scriptReferenceToString({ + packageDir: rig.resolve('a/packages/dep'), + name: 'compile', + }), + ]); + assert.equal(resultA.value.localCacheEntryName(false), resultA.value.hash); + assert.notEqual( + resultA.value.localCacheEntryName(false), + resultB.value.localCacheEntryName(false), + ); + assert.equal( + resultA.value.localCacheEntryName(true), + resultB.value.localCacheEntryName(true), + ); +}); diff --git a/src/test/local-cache.test.ts b/src/test/local-cache.test.ts index 1a7f0279b..e93656cfd 100644 --- a/src/test/local-cache.test.ts +++ b/src/test/local-cache.test.ts @@ -9,11 +9,13 @@ import * as assert from 'node:assert'; import * as fs from 'fs/promises'; import * as pathlib from 'path'; import {createHash} from 'crypto'; +import lockfile from 'proper-lockfile'; import {LocalCache} from '../caching/local-cache.js'; import {Fingerprint} from '../fingerprint.js'; import {getScriptDataDir} from '../util/script-data-dir.js'; import {FilesystemTestRig} from './util/filesystem-test-rig.js'; import {FsGate} from './util/fs-gate.js'; +import {git, initRepo} from './util/git.js'; import type {AbsoluteEntry} from '../util/glob.js'; import type {FingerprintString} from '../fingerprint.js'; @@ -24,7 +26,10 @@ import type {ScriptReference} from '../config.js'; const SCRIPT_NAME = 'a'; -async function setup(maxEntries: number): Promise< +async function setup( + maxEntries: number, + shareWorktrees = false, +): Promise< { rig: FilesystemTestRig; cache: LocalCache; @@ -61,7 +66,7 @@ async function setup(maxEntries: number): Promise< packageDir: rig.resolve('.'), name: SCRIPT_NAME, }; - const cache = new LocalCache(maxEntries); + const cache = new LocalCache(maxEntries, {shareWorktrees}); const cacheDir = pathlib.join(getScriptDataDir(script), 'cache'); const trashDir = rig.resolve(pathlib.join('.wireit', 'trash')); @@ -121,6 +126,10 @@ async function readdirIfExists(dir: string): Promise { } } +/** Takes an entry's lock the same way {@link LocalCache} does. */ +const lockEntry = (lockFile: string) => + lockfile.lock(lockFile, {stale: 10_000, retries: 0, realpath: false}); + /** The cache only keys off the string form, so any distinct string works. */ const fingerprint = (name: string) => Fingerprint.fromString(name as FingerprintString); @@ -184,10 +193,10 @@ void test('evicts the least recently used entry, not the oldest', async () => { await ctx.setRecency('v1', 1_001); // Reading v0 makes it the most recently used, though v1 is the newest. - assert.notEqual( - await ctx.cache.get(ctx.script, fingerprint('v0')), - undefined, - ); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + // A hit holds the entry's lock until it is applied or released. + await hit.release(); await ctx.cacheOutput('v2'); assert.deepEqual(await ctx.entryHashes(), ['v0', 'v2'].map(hashOf).sort()); @@ -485,3 +494,547 @@ void test('a write that fails leaves no entry, and deletes its temp copy', async await ctx.cacheOutput('v0'); assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); }); + +for (const shareWorktrees of [false, true]) { + void test(`a second set of an empty output is a hit (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(10, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + assert.equal(await ctx.cache.set(ctx.script, fingerprint('v0'), []), true); + assert.equal(await ctx.cache.set(ctx.script, fingerprint('v0'), []), true); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); + }); + + void test(`a second set of the same entry is a hit (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(10, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + await ctx.cacheOutput('v0'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); + }); +} + +for (const shareWorktrees of [false, true]) { + void test(`only temp leftovers over an hour old are trashed (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(10, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + const tempDir = pathlib.join(getScriptDataDir(ctx.script), 'temp'); + await fs.mkdir(pathlib.join(tempDir, 'fresh'), {recursive: true}); + await fs.mkdir(pathlib.join(tempDir, 'old'), {recursive: true}); + const ancient = new Date(Date.now() - 2 * 60 * 60 * 1000); + await fs.utimes(pathlib.join(tempDir, 'old'), ancient, ancient); + await ctx.cacheOutput('v0'); + // The fresh folder may be another worktree's save in progress; the old one + // is an aborted save. + assert.deepEqual(await ctx.tempEntries(), ['fresh']); + }); + + void test(`eviction skips an entry whose lock is held (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(2, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + await ctx.setRecency('v0', 1_000); + await ctx.cacheOutput('v1'); + await ctx.setRecency('v1', 1_001); + + // Another worktree is restoring v0. + const restoring = await new LocalCache(10, {shareWorktrees}).get( + ctx.script, + fingerprint('v0'), + ); + assert.ok(restoring); + await ctx.setRecency('v0', 1_000); + try { + await ctx.cacheOutput('v2'); + // v0 is oldest but locked, so the next oldest (v1) goes instead. v2 is + // never evicted, as the entry just written. + assert.deepEqual( + await ctx.entryHashes(), + ['v0', 'v2'].map(hashOf).sort(), + ); + } finally { + await restoring.release(); + } + + await ctx.cacheOutput('v3'); + assert.deepEqual(await ctx.entryHashes(), ['v2', 'v3'].map(hashOf).sort()); + }); +} + +/** + * Returns {@link promise}, but rejects after 5 seconds. A call that waits for a + * held lock does not finish in that time. + */ +const soon = (promise: Promise): Promise => + Promise.race([ + promise, + new Promise((_, reject) => { + setTimeout(() => reject(new Error('Timed out')), 5000).unref(); + }), + ]); + +/** Caches an empty output. This runs eviction without writing a file. */ +const cacheEmpty = ( + ctx: {cache: LocalCache; script: ScriptReference}, + name: string, +) => ctx.cache.set(ctx.script, fingerprint(name), []); + +void test('shared cache hit holds its lock through restore', async () => { + await using ctx = await setup(1, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + await ctx.rig.delete('output'); + + { + using gate = new FsGate({ + functions: ['copyFile'], + path: /[\\/]cache[\\/]/, + }); + const restore = hit.apply(); + await gate.firstCall; + // This write puts the folder over its limit of 1. v0 stays because it is + // being restored. + await cacheEmpty(ctx, 'v1'); + assert.deepEqual(await ctx.entryHashes(), ['v0', 'v1'].map(hashOf).sort()); + gate.release(); + await restore; + } + assert.equal(await ctx.rig.read('output'), 'v0'); + + await cacheEmpty(ctx, 'v2'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v2')]); +}); + +void test('an abandoned shared cache hit releases its lock', async () => { + await using ctx = await setup(1, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + + // StandardScriptExecution releases a hit in its finally block when it + // cancels or fails before apply. + await hit.release(); + await cacheEmpty(ctx, 'v1'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v1')]); +}); + +void test('a failed shared restore releases its lock', async () => { + await using ctx = await setup(1, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + + await ctx.rig.delete('output'); + { + using _gate = new FsGate({ + functions: ['copyFile'], + path: /[\\/]cache[\\/]/, + failWith: 'EIO', + }); + await assert.rejects(hit.apply(), {code: 'EIO'}); + } + await cacheEmpty(ctx, 'v1'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v1')]); +}); + +void test('two worktrees restore the same entry at once', async () => { + await using ctx = await setup(10, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + const first = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(first); + try { + const second = await soon( + new LocalCache(10, {shareWorktrees: true}).get( + ctx.script, + fingerprint('v0'), + ), + ); + assert.ok(second); + const lockNames = await readdirIfExists( + pathlib.join(getScriptDataDir(ctx.script), 'cache-locks', hashOf('v0')), + ); + assert.equal(lockNames.length, 2); + for (const name of lockNames) { + assert.match(name, /^restore-[0-9a-f]{16}\.lock$/); + } + await ctx.rig.delete('output'); + await second.apply(); + assert.equal(await ctx.rig.read('output'), 'v0'); + } finally { + await first.release(); + } +}); + +void test('a restore waits for an eviction of its entry in progress', async () => { + await using ctx = await setup(10, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + const lockDir = pathlib.join( + getScriptDataDir(ctx.script), + 'cache-locks', + hashOf('v0'), + ); + await fs.mkdir(lockDir, {recursive: true}); + const releaseEviction = await lockEntry(pathlib.join(lockDir, 'evict')); + + let settled = false; + const get = ctx.cache.get(ctx.script, fingerprint('v0')).finally(() => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 500)); + assert.equal(settled, false); + + // Finish the eviction by hand. + await fs.rm(pathlib.join(ctx.cacheDir, hashOf('v0')), {recursive: true}); + await releaseEviction(); + assert.equal(await soon(get), undefined); +}); + +for (const shareWorktrees of [false, true]) { + void test(`a lookup that misses takes no lock (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(10, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + assert.equal(await ctx.cache.get(ctx.script, fingerprint('v0')), undefined); + assert.deepEqual( + await readdirIfExists( + pathlib.join(getScriptDataDir(ctx.script), 'cache-locks'), + ), + [], + ); + }); + + void test(`a hit restores from a cache folder that cannot be written (shareWorktrees: ${shareWorktrees})`, async () => { + await using ctx = await setup(10, shareWorktrees); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + await ctx.rig.delete('output'); + // The lock folder cannot be created, as on a read-only mount. + using _gate = new FsGate({ + functions: ['mkdir'], + path: /[\\/]cache-locks/, + failWith: 'EACCES', + }); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + await hit.apply(); + assert.equal(await ctx.rig.read('output'), 'v0'); + }); +} + +/** + * Creates a git repo in "main" and linked worktrees in "linked" and "linked2". + */ +async function addWorktrees( + rig: FilesystemTestRig, +): Promise<{main: string; linked: string; linked2: string}> { + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + const linked2 = rig.resolve('linked2'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'package.json'), '{}'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + git(main, ['worktree', 'add', linked2, '-b', 'other2']); + return {main, linked, linked2}; +} + +/** Returns a file entry in the form that {@link LocalCache.set} takes. */ +const fileEntry = (path: string) => + ({ + path, + dirent: { + isFile: () => true, + isDirectory: () => false, + isSymbolicLink: () => false, + }, + }) as AbsoluteEntry; + +void test('the over-limit reminder names the shared cache folder', async () => { + await using rig = await FilesystemTestRig.setup(); + const {main, linked} = await addWorktrees(rig); + const mainCacheDir = pathlib.join( + getScriptDataDir({packageDir: main, name: SCRIPT_NAME}), + 'cache', + ); + // Fill the main worktree's cache with old entries. A Wireit version without + // a limit could leave this many. + for (const name of ['v0', 'v1', 'v2', 'v3', 'v4', 'v5']) { + await fs.mkdir(pathlib.join(mainCacheDir, hashOf(name)), {recursive: true}); + } + + const cache = new LocalCache(1, {shareWorktrees: true}); + await rig.write(pathlib.join('linked', 'output'), 'n0'); + await cache.set({packageDir: linked, name: SCRIPT_NAME}, fingerprint('n0'), [ + fileEntry(pathlib.join(linked, 'output')), + ]); + + // The message must name the main worktree's folder. The linked worktree has + // no cache of its own. + const messages = await cache.sweepTrash(); + assert.equal(messages.length, 1); + const mainDataDir = pathlib.join(await fs.realpath(main), '.wireit'); + assert.ok(messages[0]!.includes(mainDataDir), messages[0]); + assert.ok(!messages[0]!.includes(linked), messages[0]); +}); + +void test('restoring and evicting entries leave no lock files behind', async () => { + await using ctx = await setup(1); + await ctx.cacheOutput('v0'); + const hit = await ctx.cache.get(ctx.script, fingerprint('v0')); + assert.ok(hit); + await ctx.rig.delete('output'); + await hit.apply(); + // This write evicts v0. + await ctx.cacheOutput('v1'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v1')]); + + assert.deepEqual( + await readdirIfExists( + pathlib.join(getScriptDataDir(ctx.script), 'cache-locks'), + ), + [], + ); +}); + +// A linked worktree with WIREIT_CACHE_WORKTREES stores its cache in the main +// worktree's cache folder. It does this even when the main worktree does not +// set the flag. So two runs share a folder when each run is either in the +// main worktree or in a linked worktree with the flag. Each test below puts a +// run with the flag in a linked worktree. It puts a run without the flag in +// the main worktree. The tests try every combination. + +/** The parts of a Wireit run that the cache uses. */ +interface Run { + cache: LocalCache; + script: ScriptReference; + /** The package folder in the run's worktree. */ + dir: string; +} + +/** + * Creates a run. A run without the flag is in the main worktree. A run with + * the flag is in the linked worktree {@link linked}. + */ +const run = ( + worktrees: {main: string}, + shareWorktrees: boolean, + linked: string, + maxEntries: number, +): Run => { + const dir = shareWorktrees ? linked : worktrees.main; + return { + cache: new LocalCache(maxEntries, {shareWorktrees}), + script: {packageDir: dir, name: SCRIPT_NAME}, + dir, + }; +}; + +const onOff = (flag: boolean) => (flag ? 'on' : 'off'); + +// A fingerprint from Fingerprint.fromString has the same entry name with or +// without the flag. A real fingerprint does too when the script has no input +// files and no dependencies. +for (const first of [false, true]) { + for (const second of [false, true]) { + for (const empty of [false, true]) { + void test(`a second save of an entry is a hit (flag ${onOff(first)} in the first run, ${onOff(second)} in the second run, ${empty ? 'empty output' : 'one file'})`, async () => { + await using rig = await FilesystemTestRig.setup(); + const worktrees = await addWorktrees(rig); + for (const r of [ + run(worktrees, first, worktrees.linked, 10), + run(worktrees, second, worktrees.linked2, 10), + ]) { + const output = pathlib.join(r.dir, 'output'); + await fs.writeFile(output, 'v0'); + assert.equal( + await r.cache.set( + r.script, + fingerprint('v0'), + empty ? [] : [fileEntry(output)], + ), + true, + ); + } + }); + } + } +} + +for (const restoring of [false, true]) { + for (const evicting of [false, true]) { + void test(`an entry being restored is not evicted (flag ${onOff(restoring)} in the restoring run, ${onOff(evicting)} in the evicting run)`, async () => { + await using rig = await FilesystemTestRig.setup(); + const worktrees = await addWorktrees(rig); + const restorer = run(worktrees, restoring, worktrees.linked, 10); + const evictor = run(worktrees, evicting, worktrees.linked2, 1); + + const output = pathlib.join(restorer.dir, 'output'); + await fs.writeFile(output, 'v0'); + await restorer.cache.set(restorer.script, fingerprint('v0'), [ + fileEntry(output), + ]); + const hit = await restorer.cache.get(restorer.script, fingerprint('v0')); + assert.ok(hit); + await fs.rm(output); + + // This run saves an empty output, so it writes no file. A file could + // collide with the restore when both runs are in the main worktree. Its + // limit is 1, so it would evict v0, the only other entry. + await evictor.cache.set(evictor.script, fingerprint('v1'), []); + + await hit.apply(); + assert.equal(await fs.readFile(output, 'utf8'), 'v0'); + }); + } +} + +for (const saving of [false, true]) { + for (const cleaning of [false, true]) { + void test(`a save in progress is not trashed (flag ${onOff(saving)} in the saving run, ${onOff(cleaning)} in the cleaning run)`, async () => { + await using rig = await FilesystemTestRig.setup(); + const worktrees = await addWorktrees(rig); + const saver = run(worktrees, saving, worktrees.linked, 10); + const cleaner = run(worktrees, cleaning, worktrees.linked2, 10); + + // The two runs use different file names. Then the gate pauses only the + // saving run's copy, even when both runs are in the main worktree. + const saved = pathlib.join(saver.dir, 'saved'); + const cleanerOutput = pathlib.join(cleaner.dir, 'output'); + await fs.writeFile(saved, 'v0'); + await fs.writeFile(cleanerOutput, 'v1'); + { + using gate = new FsGate({ + functions: ['copyFile'], + path: /[\\/]saved$/, + }); + const save = saver.cache.set(saver.script, fingerprint('v0'), [ + fileEntry(saved), + ]); + await gate.firstCall; + // This write cleans up the script's temp folder. The saving run's + // copy is still in progress there. + await cleaner.cache.set(cleaner.script, fingerprint('v1'), [ + fileEntry(cleanerOutput), + ]); + gate.release(); + assert.equal(await save, true); + } + + await fs.rm(saved); + const hit = await saver.cache.get(saver.script, fingerprint('v0')); + assert.ok(hit); + await hit.apply(); + assert.equal(await fs.readFile(saved, 'utf8'), 'v0'); + }); + } +} + +void test('a linked worktree finds its shared cache folder once per run', async () => { + await using rig = await FilesystemTestRig.setup(); + const {linked} = await addWorktrees(rig); + const linkedScript = {packageDir: linked, name: SCRIPT_NAME}; + const cache = new LocalCache(10, {shareWorktrees: true}); + await rig.write(pathlib.join('linked', 'output'), 'v0'); + await cache.set(linkedScript, fingerprint('v0'), [ + fileEntry(pathlib.join(linked, 'output')), + ]); + + // Hide the linked worktree's ".git" file. A new lookup would now treat the + // worktree as a separate checkout. + await fs.rename( + pathlib.join(linked, '.git'), + pathlib.join(linked, '.git-hidden'), + ); + // Stop a new lookup's walk at the rig. The checkout that runs the test may + // itself be a linked worktree. + await rig.mkdir('.git'); + const hit = await cache.get(linkedScript, fingerprint('v0')); + assert.ok(hit); + await hit.release(); +}); + +void test('a package that only a linked worktree has keeps its own cache', async () => { + await using rig = await FilesystemTestRig.setup(); + const {linked} = await addWorktrees(rig); + // Only the linked worktree's branch has this package. + const pkg = pathlib.join(linked, 'packages', 'bar'); + await rig.write( + pathlib.join('linked', 'packages', 'bar', 'package.json'), + '{}', + ); + await rig.write(pathlib.join('linked', 'packages', 'bar', 'output'), 'v0'); + const script = {packageDir: pkg, name: SCRIPT_NAME}; + const cache = new LocalCache(10, {shareWorktrees: true}); + await cache.set(script, fingerprint('v0'), [ + fileEntry(pathlib.join(pkg, 'output')), + ]); + + const cacheDir = pathlib.join(getScriptDataDir(script), 'cache'); + assert.deepEqual(await readdirIfExists(cacheDir), [hashOf('v0')]); + assert.equal(await rig.exists(pathlib.join('main', 'packages')), false); +}); + +void test('linked worktree restores from the main worktree local cache', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'package.json'), '{}'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + + const mainScript = {packageDir: main, name: SCRIPT_NAME}; + const linkedScript = {packageDir: linked, name: SCRIPT_NAME}; + const cache = new LocalCache(10, {shareWorktrees: true}); + const outputEntry = { + path: pathlib.join(main, 'output'), + dirent: { + isFile: () => true, + isDirectory: () => false, + isSymbolicLink: () => false, + }, + } as AbsoluteEntry; + await rig.write(pathlib.join('main', 'output'), 'from-main'); + assert.equal( + await cache.set(mainScript, fingerprint('v0'), [outputEntry]), + true, + ); + + const linkedCacheDir = pathlib.join(getScriptDataDir(linkedScript), 'cache'); + await assert.rejects(fs.readdir(linkedCacheDir), {code: 'ENOENT'}); + + const hit = await cache.get(linkedScript, fingerprint('v0')); + assert.ok(hit); + await hit.apply(); + assert.equal(await rig.read(pathlib.join('linked', 'output')), 'from-main'); +}); diff --git a/src/test/util/filesystem-test-rig.ts b/src/test/util/filesystem-test-rig.ts index 48059bcae..072c44409 100644 --- a/src/test/util/filesystem-test-rig.ts +++ b/src/test/util/filesystem-test-rig.ts @@ -31,6 +31,12 @@ export class FilesystemTestRig { } } + static async setup(): Promise { + const rig = new FilesystemTestRig(); + await rig.setup(); + return rig; + } + /** * Initialize the temporary filesystem. */ @@ -40,6 +46,10 @@ export class FilesystemTestRig { await this.mkdir('.'); } + async [Symbol.asyncDispose](): Promise { + await this.cleanup(); + } + /** * Delete the temporary filesystem. */ diff --git a/src/test/util/git.ts b/src/test/util/git.ts new file mode 100644 index 000000000..b345b49bb --- /dev/null +++ b/src/test/util/git.ts @@ -0,0 +1,22 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {execFileSync} from 'child_process'; + +export const git = (cwd: string, args: string[]) => { + execFileSync('git', args, { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +}; + +export const initRepo = (dir: string) => { + git(dir, ['init']); + git(dir, ['config', 'user.email', 'wireit@example.com']); + git(dir, ['config', 'user.name', 'Wireit Test']); + git(dir, ['config', 'commit.gpgsign', 'false']); +}; diff --git a/src/test/util/test-rig.ts b/src/test/util/test-rig.ts index 2372766cc..bec5402fc 100644 --- a/src/test/util/test-rig.ts +++ b/src/test/util/test-rig.ts @@ -44,7 +44,7 @@ export class WireitTestRig extends FilesystemTestRig implements AsyncDisposable { - static async setup(): Promise { + static override async setup(): Promise { const rig = new WireitTestRig(); await rig.setup(); return rig; @@ -273,7 +273,7 @@ export class WireitTestRig return command; } - async [Symbol.asyncDispose]() { + override async [Symbol.asyncDispose]() { await this.cleanup(); } diff --git a/src/util/cache-root.ts b/src/util/cache-root.ts new file mode 100644 index 000000000..641fe05c2 --- /dev/null +++ b/src/util/cache-root.ts @@ -0,0 +1,236 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from './fs.js'; +import * as pathlib from 'path'; + +export interface WorktreeInfo { + worktreeRoot: string; + mainWorktreeRoot: string; +} + +/** + * Directory whose `.wireit/` folder should hold this package's local cache + * (not lock/fingerprint/manifest). + * + * Linked git worktrees share the main worktree only when `shareWorktrees` is + * true. Otherwise `packageDir` (Wireit's historical layout). A package also + * keeps its own cache until the main worktree has the package. + */ +export const resolveCachePackageDir = async ( + packageDir: string, + options?: {shareWorktrees?: boolean}, +): Promise => { + if (options?.shareWorktrees !== true) { + return packageDir; + } + const absPackageDir = pathlib.resolve(packageDir); + const worktree = await detectWorktree(absPackageDir); + if ( + worktree === undefined || + worktree.worktreeRoot === worktree.mainWorktreeRoot + ) { + return absPackageDir; + } + const mainPackageDir = pathlib.join( + worktree.mainWorktreeRoot, + pathlib.relative(worktree.worktreeRoot, absPackageDir), + ); + // Share only when the main worktree has this package. Otherwise Wireit would + // create the package's folder in the main worktree. This happens for a + // package that only this worktree's branch has. + const mainPackageJson = pathlib.join(mainPackageDir, 'package.json'); + if ((await lstatOrUndefined(mainPackageJson)) === undefined) { + return absPackageDir; + } + return mainPackageDir; +}; + +export const detectWorktree = async ( + startDir: string, +): Promise => { + const worktreeRoot = await findGitAncestor(pathlib.resolve(startDir)); + if (worktreeRoot === undefined) { + return undefined; + } + const dotGit = pathlib.join(worktreeRoot, '.git'); + const stat = await lstatOrUndefined(dotGit); + if (stat === undefined) { + return undefined; + } + if (stat.isDirectory()) { + return {worktreeRoot, mainWorktreeRoot: worktreeRoot}; + } + if (!stat.isFile()) { + return undefined; + } + const gitDir = await readGitdirPointer(dotGit, worktreeRoot); + if (gitDir === undefined) { + return undefined; + } + const mainWorktreeRoot = await readMainWorktreeRoot(gitDir); + if (mainWorktreeRoot === undefined) { + return undefined; + } + return {worktreeRoot, mainWorktreeRoot}; +}; + +const findGitAncestor = async (dir: string): Promise => { + // Nearest `.git`. A submodule working tree has its own `.git` file, so a + // package inside one does not walk up into the superproject. + // https://git-scm.com/docs/gitsubmodules + const stat = await lstatOrUndefined(pathlib.join(dir, '.git')); + if (stat !== undefined) { + return dir; + } + const parent = pathlib.dirname(dir); + return parent === dir ? undefined : findGitAncestor(parent); +}; + +const lstatOrUndefined = async ( + path: string, +): Promise> | undefined> => { + try { + return await fs.lstat(path); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return undefined; + } + throw error; + } +}; + +const readFileOrUndefined = async ( + path: string, +): Promise => { + try { + return await fs.readFile(path, 'utf8'); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return undefined; + } + throw error; + } +}; + +const readGitdirPointer = async ( + dotGitFile: string, + worktreeRoot: string, +): Promise => { + const content = await readFileOrUndefined(dotGitFile); + if (content === undefined) { + return undefined; + } + const prefix = 'gitdir: '; + const line = content.split('\n')[0] ?? ''; + if (!line.startsWith(prefix)) { + return undefined; + } + const pointed = line.slice(prefix.length).trim(); + const resolved = pathlib.isAbsolute(pointed) + ? pointed + : pathlib.resolve(worktreeRoot, pointed); + return realpathOrSelf(resolved); +}; + +const readMainWorktreeRoot = async ( + gitDir: string, +): Promise => { + // Linked worktrees point `commondir` at the main git dir. A submodule git + // dir has none, so this returns undefined and the cache stays put. + // https://git-scm.com/docs/gitrepository-layout + const commondir = ( + await readFileOrUndefined(pathlib.join(gitDir, 'commondir')) + )?.trim(); + if (commondir === undefined || commondir === '') { + return undefined; + } + const gitCommon = pathlib.isAbsolute(commondir) + ? commondir + : pathlib.resolve(gitDir, commondir); + const resolved = await realpathOrSelf(gitCommon); + if ( + !isSupportedSharedCacheLayout( + await readFileOrUndefined(pathlib.join(resolved, 'config')), + resolved, + ) + ) { + return undefined; + } + const main = pathlib.dirname(resolved); + return main === resolved ? undefined : main; +}; + +/** + * Whether the shared git dir's parent is a checkout, making + * `//.wireit` a safe cache location. Check the + * config first because bare and submodule git dirs usually aren't named + * `.git` either. + * + * - Bare clone worktrees: the shared config sets `core.bare = true`, so the + * parent of the bare dir isn't a checkout. + * - Worktrees of a submodule: the shared config sets `core.worktree`, so the + * parent is the outer repo's git folder. + * - `git clone --separate-git-dir`: git only names the main git dir `.git` + * when it sits inside the checkout, so any other name means its parent + * isn't a checkout. + */ +const isSupportedSharedCacheLayout = ( + gitConfig: string | undefined, + gitCommon: string, +): boolean => { + if (gitConfig !== undefined) { + const core = parseGitCoreConfig(gitConfig); + if (core.bare === true || core.worktree !== undefined) { + return false; + } + } + return pathlib.basename(gitCommon) === '.git'; +}; + +/** The `[core]` section's `bare` and `worktree` settings, if set. */ +const parseGitCoreConfig = ( + gitConfig: string, +): {bare?: boolean; worktree?: string} => { + const result: {bare?: boolean; worktree?: string} = {}; + let inCore = false; + for (const line of gitConfig.split('\n')) { + const trimmed = line.trim(); + if (trimmed.startsWith('[')) { + inCore = /^\[\s*core\s*[\]]/i.test(trimmed); + continue; + } + if (!inCore) { + continue; + } + const match = /^([^=;#]+?)\s*=\s*(.*?)\s*$/.exec(trimmed); + if (match === null) { + continue; + } + const key = match[1]!.trim().toLowerCase(); + const value = match[2]!.trim(); + if (key === 'bare') { + result.bare = /^(true|yes|on|1)$/i.test(value); + } else if (key === 'worktree') { + result.worktree = value; + } + } + return result; +}; + +const realpathOrSelf = async (path: string): Promise => { + try { + return await fs.realpath(path); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return path; + } + throw error; + } +}; diff --git a/src/util/fs.ts b/src/util/fs.ts index e1dd2f8f8..299a2a4ae 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -141,6 +141,15 @@ export async function lstat(path: string): Promise { } } +export async function realpath(path: string): Promise { + const reservation = await fileBudget.reserve(); + try { + return await fs.realpath(path); + } finally { + reservation[Symbol.dispose](); + } +} + export async function stat(path: string): Promise { const reservation = await fileBudget.reserve(); try {