diff --git a/.pipelines/templates/fetch-test-data-from-blob.yml b/.pipelines/templates/fetch-test-data-from-blob.yml new file mode 100644 index 000000000..95e38384f --- /dev/null +++ b/.pipelines/templates/fetch-test-data-from-blob.yml @@ -0,0 +1,113 @@ +# Download test model data from blob storage into a local directory. +# Uses azcopy with AZCLI auth so the agent identity can read from storage. + +parameters: +- name: destinationPath + type: string + default: '$(Build.SourcesDirectory)/test-data-shared' + +steps: +- task: AzureCLI@2 + displayName: 'Fetch test data from blob' + inputs: + azureSubscription: 'ortcibuild_readonly_mi2-ONNX Runtime-AIFoundryLocal' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $ErrorActionPreference = 'Stop' + + $destination = '${{ parameters.destinationPath }}' + $storageAccount = 'foundrylocalmodels' + $container = 'models' + $prefix = 'foundrylocal/models' + + if (-not (Get-Command azcopy -ErrorAction SilentlyContinue)) { + if ($IsMacOS) { + Write-Host 'azcopy not found on macOS agent. Installing via Homebrew...' + if (-not (Get-Command brew -ErrorAction SilentlyContinue)) { + throw 'Homebrew is required to install azcopy on macOS agents, but brew was not found.' + } + + brew update --quiet + brew install --quiet azcopy + } else { + throw 'azcopy is not installed on this agent.' + } + } + + azcopy --version + + New-Item -ItemType Directory -Path $destination -Force | Out-Null + $publisherRoot = Join-Path $destination 'Microsoft' + New-Item -ItemType Directory -Path $publisherRoot -Force | Out-Null + + function Invoke-AzCopy { + param( + [string]$Source, + [string]$Target + ) + + $parent = Split-Path -Path $Target -Parent + if (-not [string]::IsNullOrEmpty($parent)) { + New-Item -ItemType Directory -Path $parent -Force | Out-Null + } + + $args = @('copy', $Source, $Target, '--recursive') + + & azcopy @args + if ($LASTEXITCODE -ne 0) { + throw "azcopy failed downloading test data from $Source" + } + } + + # The active SDK tests resolve versioned model aliases in cache. + # Keep one target alias per source model to avoid duplicate copies. + $modelMappings = @( + @{ + SourcePath = 'qwen2.5-0.5b-instruct' + TargetAlias = 'qwen2.5-0.5b-instruct-generic-cpu-4' + }, + @{ + SourcePath = 'openai-whisper-tiny' + TargetAlias = 'openai-whisper-tiny-generic-cpu-4' + }, + @{ + SourcePath = 'nemotron-speech-streaming-en-0.6b' + TargetAlias = 'nemotron-speech-streaming-en-0.6b-generic-cpu-3' + }, + @{ + SourcePath = 'qwen3-embedding-0.6b' + TargetAlias = 'qwen3-embedding-0.6b-generic-cpu-1' + }, + @{ + SourcePath = 'qwen3.5-0.8b' + TargetAlias = 'qwen3.5-0.8b-generic-cpu-2' + }, + @{ + SourcePath = 'deepseek-r1-distill-qwen-14b' + TargetAlias = 'deepseek-r1-distill-qwen-14b-generic-cpu-4' + } + ) + + foreach ($model in $modelMappings) { + $requestedUrl = "https://$storageAccount.blob.core.windows.net/$container/$prefix/$($model.SourcePath)/onnx/cpu_and_mobile/*" + Write-Host "Requested blob URL: $requestedUrl" + + $target = Join-Path $publisherRoot $model.TargetAlias + Invoke-AzCopy -Source $requestedUrl -Target $target + } + + $count = @(Get-ChildItem -Path $destination -Recurse -File).Count + + if ($count -eq 0) { + throw "Downloaded test data is empty at $destination" + } + + Write-Host "Downloaded $count files into $destination" + Write-Host "Cache directory contents for $destination" + (Get-ChildItem -Path $destination -Recurse -Force | + Select-Object FullName, Length | + Format-Table -AutoSize | + Out-String).TrimEnd() | Write-Host + env: + AZCOPY_AUTO_LOGIN_TYPE: AZCLI diff --git a/.pipelines/v2/templates/stages-cs.yml b/.pipelines/v2/templates/stages-cs.yml index 3bb83a0e7..b83f48b35 100644 --- a/.pipelines/v2/templates/stages-cs.yml +++ b/.pipelines/v2/templates/stages-cs.yml @@ -67,9 +67,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/cpp-nuget' @@ -98,9 +96,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/cpp-nuget' @@ -128,16 +124,7 @@ stages: steps: - checkout: self clean: true - - bash: | - set -euo pipefail - if ! command -v git-lfs >/dev/null 2>&1; then - brew install git-lfs - fi - git lfs install - displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/cpp-nuget' diff --git a/.pipelines/v2/templates/stages-js.yml b/.pipelines/v2/templates/stages-js.yml index e434f7c74..24308897f 100644 --- a/.pipelines/v2/templates/stages-js.yml +++ b/.pipelines/v2/templates/stages-js.yml @@ -198,14 +198,12 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-js.yml parameters: rid: 'win-x64' prebuildArtifactDir: '$(Pipeline.Workspace)/js-prebuild-win-x64' - testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared' + testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared/Microsoft' - stage: js_test_linux_x64 displayName: 'JS SDK: Test Linux x64' @@ -224,14 +222,12 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-js.yml parameters: rid: 'linux-x64' prebuildArtifactDir: '$(Pipeline.Workspace)/js-prebuild-linux-x64' - testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared' + testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared/Microsoft' - stage: js_test_osx_arm64 displayName: 'JS SDK: Test macOS ARM64' @@ -252,21 +248,12 @@ stages: steps: - checkout: self clean: true - - bash: | - set -euo pipefail - if ! command -v git-lfs >/dev/null 2>&1; then - brew install git-lfs - fi - git lfs install - displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-js.yml parameters: rid: 'osx-arm64' prebuildArtifactDir: '$(Pipeline.Workspace)/js-prebuild-osx-arm64' - testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared' + testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared/Microsoft' # ===================================================================== # Pack stage — runs after all four builds. Pure packaging on Windows. diff --git a/.pipelines/v2/templates/stages-python.yml b/.pipelines/v2/templates/stages-python.yml index e6b859dc9..8aeef0b43 100644 --- a/.pipelines/v2/templates/stages-python.yml +++ b/.pipelines/v2/templates/stages-python.yml @@ -189,9 +189,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-linux-x64' @@ -216,16 +214,7 @@ stages: steps: - checkout: self clean: true - - bash: | - set -euo pipefail - if ! command -v git-lfs >/dev/null 2>&1; then - brew install git-lfs - fi - git lfs install - displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - template: ../../templates/fetch-test-data-from-blob.yml@self - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-osx-arm64' diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index b16b13fdd..f092abc41 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -43,10 +43,9 @@ steps: displayName: 'Append version define' - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self + - template: ../../templates/fetch-test-data-from-blob.yml@self parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + destinationPath: '$(Agent.BuildDirectory)/test-data-shared' - bash: | set -euo pipefail diff --git a/.pipelines/v2/templates/steps-build-macos.yml b/.pipelines/v2/templates/steps-build-macos.yml index 278bdbff8..92f9825e9 100644 --- a/.pipelines/v2/templates/steps-build-macos.yml +++ b/.pipelines/v2/templates/steps-build-macos.yml @@ -51,10 +51,9 @@ steps: displayName: 'Append version define' - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self + - template: ../../templates/fetch-test-data-from-blob.yml@self parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + destinationPath: '$(Agent.BuildDirectory)/test-data-shared' - bash: | set -euo pipefail diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index cc439f54f..5f1d0f761 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -74,10 +74,9 @@ steps: # Tests need shared model files. - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self + - template: ../../templates/fetch-test-data-from-blob.yml@self parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + destinationPath: '$(Agent.BuildDirectory)/test-data-shared' - ${{ if eq(parameters.arch, 'x64') }}: - script: >- diff --git a/.pipelines/v2/templates/steps-test-cs.yml b/.pipelines/v2/templates/steps-test-cs.yml index 4bca80729..adbb10644 100644 --- a/.pipelines/v2/templates/steps-test-cs.yml +++ b/.pipelines/v2/templates/steps-test-cs.yml @@ -2,7 +2,7 @@ # (FoundryLocal.Tests). The caller is responsible for: # - placing the Foundry Local Runtime nupkg in `flNugetDir` # - downloading the `version-info` pipeline artifact -# - checking out `test-data-shared` at `testDataSharedDir` +# - populating the model cache directory at `testDataSharedDir` parameters: - name: flNugetDir @@ -10,7 +10,7 @@ parameters: displayName: 'Path to directory containing the Foundry Local Runtime .nupkg' - name: testDataSharedDir type: string - displayName: 'Absolute path to a checked-out test-data-shared working tree' + displayName: 'Absolute path to the model cache directory used by tests' - name: additionalTestArgs type: string default: '' diff --git a/.pipelines/v2/templates/steps-test-js.yml b/.pipelines/v2/templates/steps-test-js.yml index d161da06e..f33c69850 100644 --- a/.pipelines/v2/templates/steps-test-js.yml +++ b/.pipelines/v2/templates/steps-test-js.yml @@ -14,7 +14,7 @@ parameters: displayName: 'Path to the downloaded js-prebuild- artifact directory (root with prebuilds/ inside)' - name: testDataSharedDir type: string - displayName: 'Path to the checked-out test-data-shared repo' + displayName: 'Path to the model cache directory used by tests' steps: diff --git a/.pipelines/v2/templates/steps-test-python.yml b/.pipelines/v2/templates/steps-test-python.yml index 5dfd4bab6..223ef527f 100644 --- a/.pipelines/v2/templates/steps-test-python.yml +++ b/.pipelines/v2/templates/steps-test-python.yml @@ -3,7 +3,7 @@ # # The caller is responsible for: # * placing the built wheel (.whl) under `wheelDir` -# * checking out test-data-shared at `testDataSharedDir` +# * populating the model cache directory at `testDataSharedDir` parameters: - name: wheelDir diff --git a/sdk_v2/DEVELOPMENT.md b/sdk_v2/DEVELOPMENT.md index febd9a8db..895a5b378 100644 --- a/sdk_v2/DEVELOPMENT.md +++ b/sdk_v2/DEVELOPMENT.md @@ -87,6 +87,24 @@ pwsh ./build_and_test_all.ps1 -ContinueOnError See `pwsh ./build_and_test_all.ps1 -?` for the full parameter list. +## Model cache for tests + +Model-dependent sdk_v2 tests require `FOUNDRY_TEST_DATA_DIR` to point to a +local model cache directory. + +In CI, sdk_v2 pipelines download model data from blob storage into this path. +For local runs, set it explicitly before invoking test commands. + +Example: + +```powershell +$env:FOUNDRY_TEST_DATA_DIR = 'C:\path\to\model-cache' +pwsh ./build_and_test_all.ps1 +``` + +The directory can have any name. What matters is that it already contains the +model files expected by the tests you are running. + ## Troubleshooting * **`cl.exe ... HostX86\x86\cl.exe` during pip install** — your shell has diff --git a/sdk_v2/cpp/test/internal_api/test_model_cache.h b/sdk_v2/cpp/test/internal_api/test_model_cache.h index 7f95e6bc0..ea08b9ba3 100644 --- a/sdk_v2/cpp/test/internal_api/test_model_cache.h +++ b/sdk_v2/cpp/test/internal_api/test_model_cache.h @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Shared helper for tests that need access to the shared test model cache. -// Mirrors the C# test setup pattern: -// - Default: look for "test-data-shared" in the parent of the git repo root -// - Override via FOUNDRY_TEST_DATA_DIR environment variable (absolute or relative path) +// Requires FOUNDRY_TEST_DATA_DIR to be set to an existing model cache path. #pragma once #include "utils/safe_getenv.h" @@ -17,56 +15,27 @@ namespace fs = std::filesystem; namespace fl::test { -/// Walk up from source_dir to find the directory containing .git. -inline fs::path FindRepoRoot(const fs::path& source_dir) { - fs::path dir = source_dir; - - while (!dir.empty() && dir.has_parent_path()) { - if (fs::exists(dir / ".git")) { - return dir; - } - - auto parent = dir.parent_path(); - if (parent == dir) { - break; - } - - dir = parent; - } - - throw std::runtime_error("Could not find git repository root from: " + source_dir.string()); -} - -/// Default test-data-shared directory name. -constexpr const char* kDefaultTestModelCacheDirName = "test-data-shared"; - /// Resolve the test model cache directory. -/// Priority: -/// 1. FOUNDRY_TEST_DATA_DIR environment variable (absolute or relative path) -/// 2. Default: {repo_root}/../test-data-shared/ +/// FOUNDRY_TEST_DATA_DIR is required. inline fs::path GetTestModelCacheDir() { - // Check environment variable first std::string env_value = SafeGetEnv("FOUNDRY_TEST_DATA_DIR"); - if (!env_value.empty()) { - fs::path env_path(env_value); - if (!fs::exists(env_path)) { - throw std::runtime_error("FOUNDRY_TEST_DATA_DIR does not exist: " + env_path.string()); - } - - return fs::canonical(env_path); + if (env_value.empty()) { + throw std::runtime_error( + "FOUNDRY_TEST_DATA_DIR is not set. Set it to a local model cache directory before running tests."); } - // Default: {repo_root}/../test-data-shared/ - fs::path repo_root = FindRepoRoot(fs::path(__FILE__).parent_path()); - fs::path default_path = repo_root.parent_path() / kDefaultTestModelCacheDirName; + fs::path env_path(env_value); + if (!fs::exists(env_path)) { + throw std::runtime_error("FOUNDRY_TEST_DATA_DIR does not exist: " + env_path.string()); + } - if (!fs::exists(default_path)) { - throw std::runtime_error( - "Test model cache directory not found at default location: " + default_path.string() + - "\nSet FOUNDRY_TEST_DATA_DIR environment variable to override."); + fs::path publisher_path = env_path / "Microsoft"; + if (!fs::exists(publisher_path)) { + throw std::runtime_error("FOUNDRY_TEST_DATA_DIR/Microsoft does not exist: " + + publisher_path.string()); } - return fs::canonical(default_path); + return fs::canonical(publisher_path); } /// Returns true when running under a known CI provider. diff --git a/sdk_v2/cs/test/FoundryLocal.Tests/LOCAL_MODEL_TESTING.md b/sdk_v2/cs/test/FoundryLocal.Tests/LOCAL_MODEL_TESTING.md index 1b4a71e78..6f74cb537 100644 --- a/sdk_v2/cs/test/FoundryLocal.Tests/LOCAL_MODEL_TESTING.md +++ b/sdk_v2/cs/test/FoundryLocal.Tests/LOCAL_MODEL_TESTING.md @@ -1,24 +1,13 @@ # Running Local Model Tests -## Configuration - -The test model cache directory name is configured in `sdk/cs/test/FoundryLocal.Tests/appsettings.Test.json`: - -```json -{ - "TestModelCacheDirName": "test-data-shared" -} -``` - -If the value is a directory name it will be resolved as /../{TestModelCacheDirName}. -Otherwise the value will be resolved using Path.GetFullPath, which allows for absolute paths or -relative paths based on the current working directory. +Set `FOUNDRY_TEST_DATA_DIR` to a local model cache path before running tests. +The path can point to any directory layout that contains the models expected by +the test suite. ## Run the tests -The tests will automatically find the models in the configured test model cache directory. - ```bash -cd /path/to/parent-dir/foundry-local-sdk/sdk/cs/test/FoundryLocal.Tests -dotnet test Microsoft.AI.Foundry.Local.Tests.csproj --configuration Release# Running Local Model Tests +export FOUNDRY_TEST_DATA_DIR=/path/to/model-cache +cd /path/to/Foundry-Local/sdk_v2/cs/test/FoundryLocal.Tests +dotnet test Microsoft.AI.Foundry.Local.Tests.csproj --configuration Release ``` diff --git a/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj b/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj index a2071fcc7..0644f330b 100644 --- a/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj +++ b/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj @@ -61,13 +61,6 @@ - - - - PreserveNewest - - - PreserveNewest @@ -86,8 +79,6 @@ all - - diff --git a/sdk_v2/cs/test/FoundryLocal.Tests/Utils.cs b/sdk_v2/cs/test/FoundryLocal.Tests/Utils.cs index e684e1e5d..f05fd8f31 100644 --- a/sdk_v2/cs/test/FoundryLocal.Tests/Utils.cs +++ b/sdk_v2/cs/test/FoundryLocal.Tests/Utils.cs @@ -14,7 +14,6 @@ namespace Microsoft.AI.Foundry.Local.Tests; using System.Text.Json; using Microsoft.AI.Foundry.Local.Detail; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; #pragma warning disable CS0618 // Test helpers exercise PromptTemplate/ModelSettings which are obsolete but still supported. @@ -62,58 +61,24 @@ static Utils() ILogger logger = loggerFactory.CreateLogger("FoundryLocalSdkTest"); - // Read configuration from appsettings.Test.json - logger.LogDebug("Reading configuration from appsettings.Test.json"); - var configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.Test.json", optional: true, reloadOnChange: false) - .Build(); - - // Prefer the FOUNDRY_TEST_DATA_DIR env var when set (used by CI). If it points at - // an existing directory we treat it as an absolute path. Otherwise fall back to - // the appsettings.Test.json TestModelCacheDirName logic for inner-loop / VS use. + // FOUNDRY_TEST_DATA_DIR is required for model-dependent tests. var envCacheDir = Environment.GetEnvironmentVariable("FOUNDRY_TEST_DATA_DIR"); - string testDataSharedPath; - if (!string.IsNullOrWhiteSpace(envCacheDir) && Directory.Exists(envCacheDir)) + if (string.IsNullOrWhiteSpace(envCacheDir)) { - testDataSharedPath = Path.GetFullPath(envCacheDir); - logger.LogInformation( - "Using test model cache directory from FOUNDRY_TEST_DATA_DIR env var: {TestDataSharedPath}", - testDataSharedPath); + logger.LogWarning( + "FOUNDRY_TEST_DATA_DIR is not set. Integration tests will be skipped."); + Console.Error.WriteLine( + "[Utils::Utils] FOUNDRY_TEST_DATA_DIR is not set. Integration tests will be skipped."); + return; } - else - { - if (!string.IsNullOrWhiteSpace(envCacheDir)) - { - logger.LogWarning( - "FOUNDRY_TEST_DATA_DIR is set to '{EnvCacheDir}' but the directory does not exist; falling back to appsettings.Test.json.", - envCacheDir); - } - var testModelCacheDirName = configuration["TestModelCacheDirName"] ?? "test-data-shared"; - if (Path.IsPathRooted(testModelCacheDirName) || - testModelCacheDirName.Contains(Path.DirectorySeparatorChar) || - testModelCacheDirName.Contains(Path.AltDirectorySeparatorChar)) - { - // It's a relative or complete filepath, resolve from current directory - testDataSharedPath = Path.GetFullPath(testModelCacheDirName); - } - else - { - // It's just a directory name, combine with repo root parent - testDataSharedPath = Path.GetFullPath(Path.Combine(GetRepoRoot(), "..", testModelCacheDirName)); - } - - logger.LogInformation( - "Using test model cache directory from appsettings.Test.json: {TestDataSharedPath}", - testDataSharedPath); - } + string testDataSharedPath = Path.GetFullPath(Path.Combine(envCacheDir, "Microsoft")); + logger.LogInformation( + "Using test model cache directory from FOUNDRY_TEST_DATA_DIR/Microsoft: {TestDataSharedPath}", + testDataSharedPath); if (!Directory.Exists(testDataSharedPath)) { - // Do NOT throw — that would abort every test in the assembly, including pure unit - // tests that don't touch the manager. Instead leave IntegrationTestsAvailable=false - // so integration tests skip via [SkipUnlessIntegration] while unit tests still run. logger.LogWarning( "Test model cache directory does not exist: {TestDataSharedPath}. Integration tests will be skipped. See LOCAL_MODEL_TESTING.md.", testDataSharedPath); @@ -127,7 +92,7 @@ static Utils() // exactly which path we resolved. Critical when diagnosing initialization // failures from CI logs only. Console.WriteLine($"[Utils::Utils] FOUNDRY_TEST_DATA_DIR env: '{envCacheDir}'"); - Console.WriteLine($"[Utils::Utils] Resolved test model cache: '{testDataSharedPath}'"); + Console.WriteLine($"[Utils::Utils] Resolved test model cache (FOUNDRY_TEST_DATA_DIR/Microsoft): '{testDataSharedPath}'"); var config = new Configuration { diff --git a/sdk_v2/cs/test/FoundryLocal.Tests/appsettings.Test.json b/sdk_v2/cs/test/FoundryLocal.Tests/appsettings.Test.json deleted file mode 100644 index d42d87893..000000000 --- a/sdk_v2/cs/test/FoundryLocal.Tests/appsettings.Test.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "TestModelCacheDirName": "test-data-shared" -} diff --git a/sdk_v2/js/test/_fixtures/realModelManager.ts b/sdk_v2/js/test/_fixtures/realModelManager.ts index 786bf427e..8a9ff919b 100644 --- a/sdk_v2/js/test/_fixtures/realModelManager.ts +++ b/sdk_v2/js/test/_fixtures/realModelManager.ts @@ -75,31 +75,58 @@ export async function setupRealModelManager(opts: RealModelManagerOptions = {}): }); const catalog = manager.catalog; const namePref = opts.namePreference ?? "qwen2.5-0.5b"; + const task = opts.task ?? "chat-completion"; + const all = await catalog.getModels(); + + const normalizeVersionSuffix = (value: string): string => value.replace(/-\d+$/, ""); + const nameMatchesPreference = (candidateName: string, preference: string): boolean => { + if (candidateName === preference) { + return true; + } + + const candidateBase = normalizeVersionSuffix(candidateName); + const prefBase = normalizeVersionSuffix(preference); + + // Handles versioned/unversioned pairs in either direction, e.g. + // - preference: openai-whisper-tiny-generic-cpu + // - candidate : openai-whisper-tiny-generic-cpu-4 + return candidateBase === prefBase; + }; + + const bySize = (a: IModel, b: IModel): number => + (a.info.fileSizeMb ?? Number.POSITIVE_INFINITY) - (b.info.fileSizeMb ?? Number.POSITIVE_INFINITY); + + const preferCachedThenSize = (models: IModel[]): IModel | undefined => { + const cached = models.filter((m) => m.isCached); + if (cached.length > 0) { + return cached.sort(bySize)[0]; + } + return models.sort(bySize)[0]; + }; // Preference 1: exact name / alias hit (V1 throws on miss, so swallow). let model: IModel | undefined; try { - model = await catalog.getModel(namePref); + const exact = await catalog.getModel(namePref); + if (exact.info.deviceType === "CPU") { + model = exact; + } } catch { model = undefined; } - // Preference 1b: catalog entries often carry a `-N` version suffix - // (e.g. `nemotron-speech-streaming-en-0.6b-generic-cpu-3`). If the exact - // hit missed, try matching by prefix before falling back to the task - // filter — caller-specified names should win over "smallest by task". - if (model === undefined && opts.namePreference !== undefined) { - const all = await catalog.getModels(); - const prefixed = all.find((m) => m.info.name.startsWith(namePref)); - if (prefixed !== undefined) { - model = prefixed; - } + // Preference 1b: robust preference matching with CPU + cache-first behavior. + // Handles versioned/unversioned names in either direction. + if (model === undefined) { + const preferredMatches = all.filter((m) => { + const info = m.info; + return info.deviceType === "CPU" && nameMatchesPreference(info.name, namePref); + }); + model = preferCachedThenSize(preferredMatches); } // Preference 2: smallest model matching the task filter. if (model === undefined) { - const task = opts.task ?? "chat-completion"; - const all = await catalog.getModels(); const matching = all.filter((m) => { const info = m.info; return info.task === task && info.deviceType === "CPU"; @@ -110,11 +137,7 @@ export async function setupRealModelManager(opts: RealModelManagerOptions = {}): `No catalog model matches task='${task}' deviceType='CPU' (and preference '${namePref}' missing)`, ); } - matching.sort( - (a, b) => - (a.info.filesizeMb ?? Number.POSITIVE_INFINITY) - (b.info.filesizeMb ?? Number.POSITIVE_INFINITY), - ); - model = matching[0]; + model = preferCachedThenSize(matching); } if (model === undefined) { manager.dispose(); diff --git a/sdk_v2/js/test/audio-client.test.ts b/sdk_v2/js/test/audio-client.test.ts index 0cf818311..6b3eb9d3c 100644 --- a/sdk_v2/js/test/audio-client.test.ts +++ b/sdk_v2/js/test/audio-client.test.ts @@ -51,7 +51,7 @@ describe.skipIf(!haveTestModelCache)("AudioClient (real whisper-tiny model, V1 O beforeAll(async () => { fixture = await setupRealModelManager({ - namePreference: "whisper-tiny", + namePreference: "openai-whisper-tiny-generic-cpu", task: "automatic-speech-recognition", }); if (fixture !== undefined) { diff --git a/sdk_v2/js/test/audio-session.test.ts b/sdk_v2/js/test/audio-session.test.ts index a2a7fcb7d..72b522710 100644 --- a/sdk_v2/js/test/audio-session.test.ts +++ b/sdk_v2/js/test/audio-session.test.ts @@ -257,7 +257,7 @@ describe.skipIf(!haveTestModelCache)("AudioSession (real whisper-tiny model)", ( beforeAll(async () => { fixture = await setupRealModelManager({ - namePreference: "whisper-tiny", + namePreference: "openai-whisper-tiny-generic-cpu", task: "automatic-speech-recognition", }); }, 5 * 60_000); diff --git a/sdk_v2/python/test/conftest.py b/sdk_v2/python/test/conftest.py index 5f521efed..a1fdccfce 100644 --- a/sdk_v2/python/test/conftest.py +++ b/sdk_v2/python/test/conftest.py @@ -38,9 +38,7 @@ def is_running_in_ci() -> bool: IS_CI: bool = is_running_in_ci() -# Optional override that points the SDK at a pre-staged model cache so -# integration tests can find cached models without downloading. -# Mirrors the C++ FOUNDRY_TEST_DATA_DIR env var. +# Required model cache path for sdk_v2 Python tests. FOUNDRY_TEST_DATA_DIR: str | None = os.environ.get("FOUNDRY_TEST_DATA_DIR") or None @@ -66,13 +64,18 @@ def manager(): # and must not close it on teardown. created_here = False + if not FOUNDRY_TEST_DATA_DIR: + pytest.skip("FOUNDRY_TEST_DATA_DIR is required for sdk_v2 Python tests.") + + if not os.path.isdir(FOUNDRY_TEST_DATA_DIR): + pytest.skip(f"FOUNDRY_TEST_DATA_DIR does not exist: {FOUNDRY_TEST_DATA_DIR}") + if FoundryLocalManager.instance is None: config_kwargs = { "app_name": "FoundryLocalPythonTests", "log_level": LogLevel.WARNING, + "model_cache_dir": FOUNDRY_TEST_DATA_DIR, } - if FOUNDRY_TEST_DATA_DIR: - config_kwargs["model_cache_dir"] = FOUNDRY_TEST_DATA_DIR config = Configuration(**config_kwargs) FoundryLocalManager.initialize(config)