diff --git a/.vscode/launch.json b/.vscode/launch.json index 830cdbfdc..b93b69ffb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,39 +19,6 @@ "/app": "${workspaceFolder}" }, }, - { - "name": "Echo Docker", - "type": "docker", - "containerName": "echo_cntr", - "request": "attach", - "platform": "netCore", - "processName": "EchoEngine", - "sourceFileMap": { - "/app": "${workspaceFolder}" - }, - }, - { - "name": "Machine Engine Docker", - "type": "docker", - "containerName": "machine-engine-cntr", - "request": "attach", - "platform": "netCore", - "processName": "Serval.Machine.EngineServer", - "sourceFileMap": { - "/app": "${workspaceFolder}" - }, - }, - { - "name": "Machine Job Docker", - "type": "docker", - "containerName": "machine-job-cntr", - "request": "attach", - "platform": "netCore", - "processName": "Serval.Machine.JobServer", - "sourceFileMap": { - "/app": "${workspaceFolder}" - }, - }, { "name": "Launch Serval", "type": "coreclr", @@ -72,26 +39,6 @@ "ASPNETCORE_ENVIRONMENT": "Development" }, }, - { - "name": "Launch Echo", - "type": "coreclr", - "request": "launch", - "preLaunchTask": "build", - "program": "${workspaceFolder}/src/Echo/src/EchoEngine/bin/Debug/net10.0/EchoEngine.dll", - "args": [], - "cwd": "${workspaceFolder}/src/Echo/src/EchoEngine", - "stopAtEntry": false, - "console": "internalConsole", - "justMyCode": false, - "symbolOptions": { - "searchPaths": [], - "searchMicrosoftSymbolServer": true, - "searchNuGetOrgSymbolServer": true - }, - "env": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - }, { "name": ".NET Core Attach", "type": "coreclr", @@ -99,23 +46,4 @@ "processId": "${command:pickProcess}" } ], - "compounds": [ - { - "name": "ServalComb", - "configurations": [ - "Launch Serval", - "Launch Echo" - ] - }, - { - "name": "DockerComb", - "justMyCode": false, - "configurations": [ - "ServalApi Docker", - "Echo Docker", - "Machine Engine Docker", - "Machine Job Docker" - ] - } - ] } \ No newline at end of file diff --git a/README.md b/README.md index 8c4346e1c..ea4f4e0b9 100644 --- a/README.md +++ b/README.md @@ -9,57 +9,48 @@ For the REST documentation use the Swagger site [here](https://prod.serval-api.o # Serval Architecture -Serval is designed using a microservice architecture with the following deployments: -* Serval.APIServer - * This is the top level API layer. All REST calls are made through this layer. - * It primarily handles the definition of files and corpora, per-client permissions, validation of requests and assembling NLP requests from files and database entries, such as inserting pretranslations into the USFM structure. - * This includes all components under `./src/Serval/src` except for the auto-generated Client and gRPC projects, which are used to handle communication with the API layer. -* Serval.Machine.EngineServer - * Exposes multiple engine types to the APIServer - * Can be directly invoked by the APIServer - * Handles short-lived NLP requests, including loading SMT models for on-demand word graphs. - * Queues up jobs for the JobServer - * Primary functionality is defined in Serval.Machine.Shared, with the deployment configuration is defined in Serval.Machine.EngineServer - * Reports back status to the APILayer over gRPC -* Serval.Machine.JobServer - * Executes Hangfire NLP jobs - * Queues up ClearML NLP jobs, using the ClearML API and the S3 bucket - * Preprocesses training and inferencing data - * Postprocesses inferencing results and models from the ClearML job - * Primary functionality is defined in Serval.Machine.Shared, with the deployment configuration is defined in Serval.Machine.JobServer - * Reports back status to the APILayer over gRPC -* Echo.EchoTranslationEngine - * The echo engine is for testing both the API layer and the deployment in general. -Other components of the microservice are: -* SIL.DataAccess - * Abstracts all MongoDB operations - * Enables in-memory database for testing purposes - * Replicates the functionality of [EF Core](https://learn.microsoft.com/en-us/ef/core/) -* ServiceToolkit - * Defines common models, configurations and services both used by the Serval.APIServal deployment and the Serval.Machine deployments. - +Serval is designed as a modular monolith with a single deployable unit: + +- Serval.ApiServer + - The sole deployment — all REST calls are made through this layer. + - Hosts all domain modules in-process; modules communicate via interfaces rather than over the network. + - Domain modules under `./src/Serval/src`: + - `Serval.Translation` — translation engine management and pretranslation assembly + - `Serval.WordAlignment` — word alignment engine management + - `Serval.DataFiles` — file and corpus management + - `Serval.Webhooks` — webhook delivery + - `Serval.Shared` — common models, configuration, and services shared across modules + - Engine implementations (also hosted in-process): + - `Serval.Machine.Shared` — NMT, SMT Transfer, and Statistical engine implementations; also runs Hangfire background build jobs and queues ClearML GPU training jobs + - `EchoEngine` — echo engine for testing (translation and word alignment stubs) + - External runtime dependencies: MongoDB (persistence), Hangfire (job scheduling), ClearML (GPU training jobs), S3 (shared file storage for training data and models) +- SIL.DataAccess + - Abstracts all MongoDB operations + - Enables in-memory database for testing purposes + - Replicates the functionality of [EF Core](https://learn.microsoft.com/en-us/ef/core/) # Development ## Setting up Your Environment -* Use VS Code with all the recommended extensions -* Development is supported in Ubuntu and Windows WSL2 - * For Ubuntu, use [microsoft's distribution of .net](https://learn.microsoft.com/en-us/dotnet/core/install/linux-ubuntu) - * Ubuntu 22.04 and 24.04 are currently supported -* Install the repositories: - * To develop Serval, you will also likely need to make changes to the [Machine repo](https://github.com/sillsdev/machine) as well - they are intricately tied. - * To enable Serval to use your current edits in Machine (rather than the nuget package) you need to install Machine in an adjacent folder to Serval - * i.e., if your serval repo is cloned into /home/username/repos/serval, machine should be in /home/username/repos/machine - * Make sure that you build Machine using before you build Serval +- Use VS Code with all the recommended extensions +- Development is supported in Ubuntu and Windows WSL2 + - For Ubuntu, use [microsoft's distribution of .net](https://learn.microsoft.com/en-us/dotnet/core/install/linux-ubuntu) + - Ubuntu 22.04 and 24.04 are currently supported +- Install the repositories: + - To develop Serval, you will also likely need to make changes to the [Machine repo](https://github.com/sillsdev/machine) as well - they are intricately tied. + - To enable Serval to use your current edits in Machine (rather than the nuget package) you need to install Machine in an adjacent folder to Serval + - i.e., if your serval repo is cloned into /home/username/repos/serval, machine should be in /home/username/repos/machine + - Make sure that you build Machine using before you build Serval ## Option 1: Docker Compose local testing deployment These instructions are for developing/testing using docker-compose (rather than locally/bare metal) With both the serval and machine repos installed, in the serval root folder, run `./docker_deploy.sh` -To debug in VSCode, launch "DockerComb" after to containers come up (about 5 -10 seconds). This will allow you to debug all 4 serval containers. +To debug in VSCode, launch "ServalApi Docker" after to containers come up (about 5 -10 seconds). This will allow you to debug all 4 serval containers. ## Option 2: Bare metal local testing deployment + Alternatively, you can develop without containerizing Serval. Install MongoDB 8.0 as a replica set run it on localhost:27017. (You can run `docker compose -f docker-compose.mongo.yml up` from the root of the serval repo to do so). @@ -73,26 +64,31 @@ Open "Serval.sln" and debug the ApiServer. Coding guidelines are documented [on the wiki](https://github.com/sillsdev/serval/wiki/Development-Guide) ## (Optional) Get your machine.py images setup -When jobs are run, they are queued up on ClearML. If you want to have your own agents for integration testing (and you have a GPU with 24GB RAM), you can do the following: -* clone the [machine.py repo](https://github.com/sillsdev/machine.py) -* Build the docker image with `docker build . -t local.mpy` for a GPU image or `docker build . -f dockerfile.cpu_only -t local.mpy.cpu_only` for a CPU only image. -* Register your machine as a ClearML agent (see dev team for details) - * Make sure you do NOT "always pull the image"! The images you are building are stored locally. -* Set the following environment variables: + +When jobs are run, they are queued up on ClearML. If you want to have your own agents for integration testing (and you have a GPU with 24GB RAM), you can do the following: + +- clone the [machine.py repo](https://github.com/sillsdev/machine.py) +- Build the docker image with `docker build . -t local.mpy` for a GPU image or `docker build . -f dockerfile.cpu_only -t local.mpy.cpu_only` for a CPU only image. +- Register your machine as a ClearML agent (see dev team for details) + - Make sure you do NOT "always pull the image"! The images you are building are stored locally. +- Set the following environment variables: + ``` export MACHINE_PY_IMAGE=local.mpy export MACHINE_PY_CPU_IMAGE=local.mpy.cpu_only ``` ### Running the API E2E Tests + In order to run the E2E tests, you will need to have the appropriate credentials + - Get Client ID and Client Secret from auth0.com - Login, go to Applications-> Applications -> "Machine API (Test Application)" or similar - Copy `Client ID` into Environment variable `SERVAL_CLIENT_ID` - Copy `Client Secret` into Environment variable `SERVAL_CLIENT_SECRET` - Copy the auth0 url into Environment variable `SERVAL_AUTH_URL` (e.g. `SERVAL_AUTH_URL=https://sil-appbuilder.auth0.com`) - Set `SERVAL_HOST_URL` to the api's URL (e.g. `SERVAL_HOST_URL=http://localhost`) -Now, when you run the tests from `Serval.E2ETests`, the token will automatically be retrieved from Auth0. + Now, when you run the tests from `Serval.E2ETests`, the token will automatically be retrieved from Auth0. ## Special thanks to diff --git a/Serval.sln b/Serval.sln index ade31e68f..5ebc9643c 100644 --- a/Serval.sln +++ b/Serval.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32126.317 +# Visual Studio Version 18 +VisualStudioVersion = 18.3.11520.95 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{C2CEC737-240C-4FE7-9BD4-B7548E8F5829}" ProjectSection(SolutionItems) = preProject @@ -13,8 +13,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Grpc", "src\Serval\src\Serval.Grpc\Serval.Grpc.csproj", "{443D8A33-62F5-4FE0-A972-E32E280E15C8}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.ApiServer", "src\Serval\src\Serval.ApiServer\Serval.ApiServer.csproj", "{7FAE5F93-C1BA-4280-8086-7E6207FE335E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Client", "src\Serval\src\Serval.Client\Serval.Client.csproj", "{EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}" @@ -66,18 +64,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{40C225C2-1 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared", "src\Machine\src\Serval.Machine.Shared\Serval.Machine.Shared.csproj", "{090ECB69-464F-42C8-B92C-0808BE2802FA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.EngineServer", "src\Machine\src\Serval.Machine.EngineServer\Serval.Machine.EngineServer.csproj", "{C02494FB-663E-4430-9F2D-41F1A740B271}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer", "src\Machine\src\Serval.Machine.JobServer\Serval.Machine.JobServer.csproj", "{BC766753-E560-4ADF-9923-C7A96076EA47}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ServiceToolkit", "ServiceToolkit", "{EA69B41C-49EF-4017-A687-44B9DF37FF98}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A654-4604-818C-4E683DD45A51}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment", "src\Serval\src\Serval.WordAlignment\Serval.WordAlignment.csproj", "{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{D201886D-9299-4758-80E8-694DBCF8DF93}" @@ -88,130 +76,347 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoEngine", "src\Echo\src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment.Tests", "src\Serval\test\Serval.WordAlignment.Tests\Serval.WordAlignment.Tests.csproj", "{5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit.Tests", "src\ServiceToolkit\test\SIL.ServiceToolkit.Tests\SIL.ServiceToolkit.Tests.csproj", "{C50ED15A-876D-42BF-980A-388E8C49C78D}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.IntegrationTests", "src\Serval\test\Serval.IntegrationTests\Serval.IntegrationTests.csproj", "{5FC2A081-9C4A-4761-BCE1-9753C942D597}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.Translation.Contracts", "src\Serval\src\Serval.Translation.Contracts\Serval.Translation.Contracts.csproj", "{F788A056-30F1-E0D4-81AE-B74270DF47A1}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.Machine.IntegrationTests", "src\Machine\test\Serval.Machine.IntegrationTests\Serval.Machine.IntegrationTests.csproj", "{D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.Shared.Contracts", "src\Serval\src\Serval.Shared.Contracts\Serval.Shared.Contracts.csproj", "{ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.DataFiles.Contracts", "src\Serval\src\Serval.DataFiles.Contracts\Serval.DataFiles.Contracts.csproj", "{991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment.Contracts", "src\Serval\src\Serval.WordAlignment.Contracts\Serval.WordAlignment.Contracts.csproj", "{FDE92669-CE99-429D-AB07-5E42A4C75579}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.DataAccess.Abstractions", "src\DataAccess\src\SIL.DataAccess.Abstractions\SIL.DataAccess.Abstractions.csproj", "{008B9A54-7482-4F84-A400-D9FC5F024EFD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {443D8A33-62F5-4FE0-A972-E32E280E15C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {443D8A33-62F5-4FE0-A972-E32E280E15C8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {443D8A33-62F5-4FE0-A972-E32E280E15C8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {443D8A33-62F5-4FE0-A972-E32E280E15C8}.Release|Any CPU.Build.0 = Release|Any CPU {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|x64.Build.0 = Debug|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Debug|x86.Build.0 = Debug|Any CPU {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|Any CPU.ActiveCfg = Release|Any CPU {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|Any CPU.Build.0 = Release|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|x64.ActiveCfg = Release|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|x64.Build.0 = Release|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|x86.ActiveCfg = Release|Any CPU + {7FAE5F93-C1BA-4280-8086-7E6207FE335E}.Release|x86.Build.0 = Release|Any CPU {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|x64.Build.0 = Debug|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|x86.ActiveCfg = Debug|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Debug|x86.Build.0 = Debug|Any CPU {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|Any CPU.ActiveCfg = Release|Any CPU {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|Any CPU.Build.0 = Release|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|x64.ActiveCfg = Release|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|x64.Build.0 = Release|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|x86.ActiveCfg = Release|Any CPU + {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B}.Release|x86.Build.0 = Release|Any CPU {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|x64.ActiveCfg = Debug|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|x64.Build.0 = Debug|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|x86.ActiveCfg = Debug|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Debug|x86.Build.0 = Debug|Any CPU {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|Any CPU.ActiveCfg = Release|Any CPU {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|Any CPU.Build.0 = Release|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|x64.ActiveCfg = Release|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|x64.Build.0 = Release|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|x86.ActiveCfg = Release|Any CPU + {C5F942A3-8534-43FD-A75C-0160624FD456}.Release|x86.Build.0 = Release|Any CPU {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|Any CPU.Build.0 = Debug|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|x64.ActiveCfg = Debug|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|x64.Build.0 = Debug|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|x86.ActiveCfg = Debug|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Debug|x86.Build.0 = Debug|Any CPU {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|Any CPU.ActiveCfg = Release|Any CPU {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|Any CPU.Build.0 = Release|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|x64.ActiveCfg = Release|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|x64.Build.0 = Release|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|x86.ActiveCfg = Release|Any CPU + {31106018-56D7-4E52-A438-8FD2E12D2D47}.Release|x86.Build.0 = Release|Any CPU {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|x64.ActiveCfg = Debug|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|x64.Build.0 = Debug|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|x86.ActiveCfg = Debug|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Debug|x86.Build.0 = Debug|Any CPU {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|Any CPU.ActiveCfg = Release|Any CPU {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|Any CPU.Build.0 = Release|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|x64.ActiveCfg = Release|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|x64.Build.0 = Release|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|x86.ActiveCfg = Release|Any CPU + {9ADE4727-79F3-4A7E-912A-5DBA7219DC4E}.Release|x86.Build.0 = Release|Any CPU {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|x64.ActiveCfg = Debug|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|x64.Build.0 = Debug|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|x86.ActiveCfg = Debug|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Debug|x86.Build.0 = Debug|Any CPU {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|Any CPU.ActiveCfg = Release|Any CPU {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|Any CPU.Build.0 = Release|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|x64.ActiveCfg = Release|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|x64.Build.0 = Release|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|x86.ActiveCfg = Release|Any CPU + {B3BAEC3C-0E95-422A-A8CF-D91F9874985F}.Release|x86.Build.0 = Release|Any CPU {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|x64.ActiveCfg = Debug|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|x64.Build.0 = Debug|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|x86.ActiveCfg = Debug|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Debug|x86.Build.0 = Debug|Any CPU {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|Any CPU.ActiveCfg = Release|Any CPU {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|Any CPU.Build.0 = Release|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|x64.ActiveCfg = Release|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|x64.Build.0 = Release|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|x86.ActiveCfg = Release|Any CPU + {8FC30758-37FC-4819-805D-8EFF3DEDF05F}.Release|x86.Build.0 = Release|Any CPU {539598C5-8634-4273-8714-A684622DDCFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {539598C5-8634-4273-8714-A684622DDCFC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Debug|x64.ActiveCfg = Debug|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Debug|x64.Build.0 = Debug|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Debug|x86.ActiveCfg = Debug|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Debug|x86.Build.0 = Debug|Any CPU {539598C5-8634-4273-8714-A684622DDCFC}.Release|Any CPU.ActiveCfg = Release|Any CPU {539598C5-8634-4273-8714-A684622DDCFC}.Release|Any CPU.Build.0 = Release|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Release|x64.ActiveCfg = Release|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Release|x64.Build.0 = Release|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Release|x86.ActiveCfg = Release|Any CPU + {539598C5-8634-4273-8714-A684622DDCFC}.Release|x86.Build.0 = Release|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|x64.ActiveCfg = Debug|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|x64.Build.0 = Debug|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|x86.ActiveCfg = Debug|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|x86.Build.0 = Debug|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.ActiveCfg = Release|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.Build.0 = Release|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|x64.ActiveCfg = Release|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|x64.Build.0 = Release|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|x86.ActiveCfg = Release|Any CPU + {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|x86.Build.0 = Release|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|x64.ActiveCfg = Debug|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|x64.Build.0 = Debug|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|x86.ActiveCfg = Debug|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|x86.Build.0 = Debug|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|Any CPU.ActiveCfg = Release|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|Any CPU.Build.0 = Release|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|x64.ActiveCfg = Release|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|x64.Build.0 = Release|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|x86.ActiveCfg = Release|Any CPU + {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|x86.Build.0 = Release|Any CPU {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|x64.ActiveCfg = Debug|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|x64.Build.0 = Debug|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|x86.ActiveCfg = Debug|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Debug|x86.Build.0 = Debug|Any CPU {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|Any CPU.ActiveCfg = Release|Any CPU {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|Any CPU.Build.0 = Release|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|x64.ActiveCfg = Release|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|x64.Build.0 = Release|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|x86.ActiveCfg = Release|Any CPU + {1F020042-D7B8-4541-9691-26ECFD1FFC73}.Release|x86.Build.0 = Release|Any CPU {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|Any CPU.Build.0 = Debug|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|x64.ActiveCfg = Debug|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|x64.Build.0 = Debug|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|x86.ActiveCfg = Debug|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Debug|x86.Build.0 = Debug|Any CPU {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|Any CPU.ActiveCfg = Release|Any CPU {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|Any CPU.Build.0 = Release|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|x64.ActiveCfg = Release|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|x64.Build.0 = Release|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|x86.ActiveCfg = Release|Any CPU + {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}.Release|x86.Build.0 = Release|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|x64.ActiveCfg = Debug|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|x64.Build.0 = Debug|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|x86.ActiveCfg = Debug|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|x86.Build.0 = Debug|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|Any CPU.ActiveCfg = Release|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|Any CPU.Build.0 = Release|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|x64.ActiveCfg = Release|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|x64.Build.0 = Release|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|x86.ActiveCfg = Release|Any CPU + {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|x86.Build.0 = Release|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|x64.ActiveCfg = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|x64.Build.0 = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|x86.ActiveCfg = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|x86.Build.0 = Debug|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.Build.0 = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|x64.ActiveCfg = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|x64.Build.0 = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|x86.ActiveCfg = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|x86.Build.0 = Release|Any CPU {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|x64.ActiveCfg = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|x64.Build.0 = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|x86.ActiveCfg = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|x86.Build.0 = Debug|Any CPU {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.ActiveCfg = Release|Any CPU {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.Build.0 = Release|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.Build.0 = Release|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.Build.0 = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|x64.ActiveCfg = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|x64.Build.0 = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|x86.ActiveCfg = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|x86.Build.0 = Release|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|x64.ActiveCfg = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|x64.Build.0 = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|x86.ActiveCfg = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|x86.Build.0 = Debug|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU - {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|x64.ActiveCfg = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|x64.Build.0 = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|x86.ActiveCfg = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|x86.Build.0 = Release|Any CPU {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|x64.ActiveCfg = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|x64.Build.0 = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|x86.ActiveCfg = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|x86.Build.0 = Debug|Any CPU {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.ActiveCfg = Release|Any CPU {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.Build.0 = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|x64.ActiveCfg = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|x64.Build.0 = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|x86.ActiveCfg = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|x86.Build.0 = Release|Any CPU {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|x64.ActiveCfg = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|x64.Build.0 = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|x86.ActiveCfg = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|x86.Build.0 = Debug|Any CPU {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.ActiveCfg = Release|Any CPU {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.Build.0 = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|x64.ActiveCfg = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|x64.Build.0 = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|x86.ActiveCfg = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|x86.Build.0 = Release|Any CPU {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|x64.ActiveCfg = Debug|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|x64.Build.0 = Debug|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|x86.ActiveCfg = Debug|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Debug|x86.Build.0 = Debug|Any CPU {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|Any CPU.ActiveCfg = Release|Any CPU {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|Any CPU.Build.0 = Release|Any CPU - {C50ED15A-876D-42BF-980A-388E8C49C78D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C50ED15A-876D-42BF-980A-388E8C49C78D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.Build.0 = Release|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|x64.ActiveCfg = Release|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|x64.Build.0 = Release|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|x86.ActiveCfg = Release|Any CPU + {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5}.Release|x86.Build.0 = Release|Any CPU {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|x64.ActiveCfg = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|x64.Build.0 = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|x86.ActiveCfg = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|x86.Build.0 = Debug|Any CPU {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|Any CPU.ActiveCfg = Release|Any CPU {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|Any CPU.Build.0 = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|x64.ActiveCfg = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|x64.Build.0 = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|x86.ActiveCfg = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|x86.Build.0 = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|x64.ActiveCfg = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|x64.Build.0 = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|x86.ActiveCfg = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Debug|x86.Build.0 = Debug|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|Any CPU.Build.0 = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|x64.ActiveCfg = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|x64.Build.0 = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|x86.ActiveCfg = Release|Any CPU + {F788A056-30F1-E0D4-81AE-B74270DF47A1}.Release|x86.Build.0 = Release|Any CPU {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|x64.ActiveCfg = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|x64.Build.0 = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|x86.ActiveCfg = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|x86.Build.0 = Debug|Any CPU {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|Any CPU.ActiveCfg = Release|Any CPU {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|Any CPU.Build.0 = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|x64.ActiveCfg = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|x64.Build.0 = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|x86.ActiveCfg = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|x86.Build.0 = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|x64.ActiveCfg = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|x64.Build.0 = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|x86.ActiveCfg = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Debug|x86.Build.0 = Debug|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|Any CPU.Build.0 = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|x64.ActiveCfg = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|x64.Build.0 = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|x86.ActiveCfg = Release|Any CPU + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF}.Release|x86.Build.0 = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|x64.ActiveCfg = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|x64.Build.0 = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|x86.ActiveCfg = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Debug|x86.Build.0 = Debug|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|Any CPU.Build.0 = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|x64.ActiveCfg = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|x64.Build.0 = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|x86.ActiveCfg = Release|Any CPU + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B}.Release|x86.Build.0 = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|x64.ActiveCfg = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|x64.Build.0 = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|x86.ActiveCfg = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Debug|x86.Build.0 = Debug|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|Any CPU.Build.0 = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|x64.ActiveCfg = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|x64.Build.0 = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|x86.ActiveCfg = Release|Any CPU + {FDE92669-CE99-429D-AB07-5E42A4C75579}.Release|x86.Build.0 = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|x64.ActiveCfg = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|x64.Build.0 = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|x86.ActiveCfg = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Debug|x86.Build.0 = Debug|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|Any CPU.Build.0 = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|x64.ActiveCfg = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|x64.Build.0 = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|x86.ActiveCfg = Release|Any CPU + {008B9A54-7482-4F84-A400-D9FC5F024EFD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {443D8A33-62F5-4FE0-A972-E32E280E15C8} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {7FAE5F93-C1BA-4280-8086-7E6207FE335E} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {EF43AA91-4FA2-4F96-8EEB-C1D03943FD2B} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {C5F942A3-8534-43FD-A75C-0160624FD456} = {92805246-5285-4F0A-9BF8-6EE4A027A41B} @@ -233,19 +438,18 @@ Global {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} {090ECB69-464F-42C8-B92C-0808BE2802FA} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} - {C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} - {BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} - {C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98} - {0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51} {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {0904BA95-D5BF-4AC2-A919-20A785EF45F5} = {D201886D-9299-4758-80E8-694DBCF8DF93} {929FF600-8C7E-4498-A2A3-5534F3A3481E} = {0904BA95-D5BF-4AC2-A919-20A785EF45F5} {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5} = {3E753B99-7C31-42AC-B02E-012B802F58DB} - {1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126} = {EA69B41C-49EF-4017-A687-44B9DF37FF98} - {C50ED15A-876D-42BF-980A-388E8C49C78D} = {1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126} {5FC2A081-9C4A-4761-BCE1-9753C942D597} = {3E753B99-7C31-42AC-B02E-012B802F58DB} + {F788A056-30F1-E0D4-81AE-B74270DF47A1} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} + {ABFE5135-66C9-9F29-AAAB-50CEA2AB5BDF} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} + {991D4DEF-5AE8-4061-82F8-2BCB27D7A14B} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} + {FDE92669-CE99-429D-AB07-5E42A4C75579} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} + {008B9A54-7482-4F84-A400-D9FC5F024EFD} = {92805246-5285-4F0A-9BF8-6EE4A027A41B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/DataAccess/src/SIL.DataAccess/Attempt.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/Attempt.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/Attempt.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/Attempt.cs diff --git a/src/DataAccess/src/SIL.DataAccess/DuplicateKeyException.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/DuplicateKeyException.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/DuplicateKeyException.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/DuplicateKeyException.cs diff --git a/src/DataAccess/src/SIL.DataAccess/EntityChange.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/EntityChange.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/EntityChange.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/EntityChange.cs diff --git a/src/DataAccess/src/SIL.DataAccess/IDataAccessContext.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IDataAccessContext.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/IDataAccessContext.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IDataAccessContext.cs diff --git a/src/DataAccess/src/SIL.DataAccess/IEntity.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IEntity.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/IEntity.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IEntity.cs diff --git a/src/DataAccess/src/SIL.DataAccess/IIdGenerator.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IIdGenerator.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/IIdGenerator.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IIdGenerator.cs diff --git a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfigurator.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IMemoryDataAccessBuilder.cs similarity index 68% rename from src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfigurator.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IMemoryDataAccessBuilder.cs index b4a6dce62..fbe079f1e 100644 --- a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfigurator.cs +++ b/src/DataAccess/src/SIL.DataAccess.Abstractions/IMemoryDataAccessBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -public interface IMongoDataAccessConfigurator +public interface IMemoryDataAccessBuilder { IServiceCollection Services { get; } } diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfigurator.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IMongoDataAccessBuilder.cs similarity index 71% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfigurator.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IMongoDataAccessBuilder.cs index 2a2d5188d..f23427d4a 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfigurator.cs +++ b/src/DataAccess/src/SIL.DataAccess.Abstractions/IMongoDataAccessBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -public interface IOutboxConfigurator +public interface IMongoDataAccessBuilder { IServiceCollection Services { get; } } diff --git a/src/DataAccess/src/SIL.DataAccess/IRepository.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IRepository.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/IRepository.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IRepository.cs diff --git a/src/DataAccess/src/SIL.DataAccess.Abstractions/IRepositoryExtensions.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IRepositoryExtensions.cs new file mode 100644 index 000000000..64f52bea3 --- /dev/null +++ b/src/DataAccess/src/SIL.DataAccess.Abstractions/IRepositoryExtensions.cs @@ -0,0 +1,93 @@ +namespace SIL.DataAccess; + +public static class IRepositoryExtensions +{ + public static async Task GetAsync( + this IRepository repo, + string id, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + Attempt attempt = await repo.TryGetAsync(id, cancellationToken).ConfigureAwait(false); + if (attempt.Success) + return attempt.Result; + return default; + } + + public static async Task> GetAllAsync( + this IRepository repo, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + return await repo.GetAllAsync(e => true, cancellationToken).ConfigureAwait(false); + } + + public static async Task> TryGetAsync( + this IRepository repo, + string id, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + T? entity = await repo.GetAsync(e => e.Id == id, cancellationToken).ConfigureAwait(false); + return new Attempt(entity != null, entity); + } + + public static async Task ExistsAsync( + this IRepository repo, + string id, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + return await repo.ExistsAsync(e => e.Id == id, cancellationToken).ConfigureAwait(false); + } + + public static Task UpdateAsync( + this IRepository repo, + string id, + Action> update, + bool upsert = false, + bool returnOriginal = false, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + return repo.UpdateAsync(e => e.Id == id, update, upsert, returnOriginal, cancellationToken); + } + + public static Task UpdateAsync( + this IRepository repo, + T entity, + Action> update, + bool upsert = false, + bool returnOriginal = false, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + return repo.UpdateAsync(e => e.Id == entity.Id, update, upsert, returnOriginal, cancellationToken); + } + + public static Task DeleteAsync( + this IRepository repo, + string id, + CancellationToken cancellationToken = default + ) + where T : IEntity + { + return repo.DeleteAsync(e => e.Id == id, cancellationToken); + } + + public static async Task DeleteAsync( + this IRepository repo, + T entity, + CancellationToken cancellationToken = default + ) + where T : class, IEntity + { + return await repo.DeleteAsync(e => e.Id == entity.Id, cancellationToken).ConfigureAwait(false) != null; + } +} diff --git a/src/DataAccess/src/SIL.DataAccess/ISubscription.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/ISubscription.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/ISubscription.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/ISubscription.cs diff --git a/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/IUpdateBuilder.cs similarity index 100% rename from src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs rename to src/DataAccess/src/SIL.DataAccess.Abstractions/IUpdateBuilder.cs diff --git a/src/DataAccess/src/SIL.DataAccess.Abstractions/SIL.DataAccess.Abstractions.csproj b/src/DataAccess/src/SIL.DataAccess.Abstractions/SIL.DataAccess.Abstractions.csproj new file mode 100644 index 000000000..6b7e865ed --- /dev/null +++ b/src/DataAccess/src/SIL.DataAccess.Abstractions/SIL.DataAccess.Abstractions.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + SIL.DataAccess + + + + + + + + + diff --git a/src/DataAccess/src/SIL.DataAccess.Abstractions/Usings.cs b/src/DataAccess/src/SIL.DataAccess.Abstractions/Usings.cs new file mode 100644 index 000000000..b6a354116 --- /dev/null +++ b/src/DataAccess/src/SIL.DataAccess.Abstractions/Usings.cs @@ -0,0 +1,2 @@ +global using System.Diagnostics.CodeAnalysis; +global using System.Linq.Expressions; diff --git a/src/DataAccess/src/SIL.DataAccess/DataAccessExtensions.cs b/src/DataAccess/src/SIL.DataAccess/DataAccessExtensions.cs index e58a47755..499fb6870 100644 --- a/src/DataAccess/src/SIL.DataAccess/DataAccessExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/DataAccessExtensions.cs @@ -2,95 +2,6 @@ public static class DataAccessExtensions { - public static async Task GetAsync( - this IRepository repo, - string id, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - Attempt attempt = await repo.TryGetAsync(id, cancellationToken).ConfigureAwait(false); - if (attempt.Success) - return attempt.Result; - return default; - } - - public static async Task> GetAllAsync( - this IRepository repo, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - return await repo.GetAllAsync(e => true, cancellationToken).ConfigureAwait(false); - } - - public static async Task> TryGetAsync( - this IRepository repo, - string id, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - T? entity = await repo.GetAsync(e => e.Id == id, cancellationToken).ConfigureAwait(false); - return new Attempt(entity != null, entity); - } - - public static async Task ExistsAsync( - this IRepository repo, - string id, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - return await repo.ExistsAsync(e => e.Id == id, cancellationToken).ConfigureAwait(false); - } - - public static Task UpdateAsync( - this IRepository repo, - string id, - Action> update, - bool upsert = false, - bool returnOriginal = false, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - return repo.UpdateAsync(e => e.Id == id, update, upsert, returnOriginal, cancellationToken); - } - - public static Task UpdateAsync( - this IRepository repo, - T entity, - Action> update, - bool upsert = false, - bool returnOriginal = false, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - return repo.UpdateAsync(e => e.Id == entity.Id, update, upsert, returnOriginal, cancellationToken); - } - - public static Task DeleteAsync( - this IRepository repo, - string id, - CancellationToken cancellationToken = default - ) - where T : IEntity - { - return repo.DeleteAsync(e => e.Id == id, cancellationToken); - } - - public static async Task DeleteAsync( - this IRepository repo, - T entity, - CancellationToken cancellationToken = default - ) - where T : class, IEntity - { - return await repo.DeleteAsync(e => e.Id == entity.Id, cancellationToken).ConfigureAwait(false) != null; - } - public static async Task CreateOrUpdateAsync(this IMongoIndexManager indexes, CreateIndexModel indexModel) { try diff --git a/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfiguratorExtensions.cs b/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessBuilderExtensions.cs similarity index 54% rename from src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfiguratorExtensions.cs rename to src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessBuilderExtensions.cs index 5c49deecc..9bf63b64d 100644 --- a/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfiguratorExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessBuilderExtensions.cs @@ -1,8 +1,8 @@ namespace Microsoft.Extensions.DependencyInjection; -public static class IMemoryDataAccessConfiguratorExtensions +public static class IMemoryDataAccessBuilderExtensions { - public static IMemoryDataAccessConfigurator AddRepository(this IMemoryDataAccessConfigurator configurator) + public static IMemoryDataAccessBuilder AddRepository(this IMemoryDataAccessBuilder configurator) where T : IEntity { configurator.Services.AddSingleton, MemoryRepository>(); diff --git a/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfigurator.cs b/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfigurator.cs deleted file mode 100644 index 919b064c9..000000000 --- a/src/DataAccess/src/SIL.DataAccess/IMemoryDataAccessConfigurator.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public interface IMemoryDataAccessConfigurator -{ - IServiceCollection Services { get; } -} diff --git a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs b/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessBuilderExtensions.cs similarity index 92% rename from src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs rename to src/DataAccess/src/SIL.DataAccess/IMongoDataAccessBuilderExtensions.cs index ae089d3a4..cefd45a7f 100644 --- a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessBuilderExtensions.cs @@ -1,9 +1,9 @@ namespace Microsoft.Extensions.DependencyInjection; -public static class IMongoDataAccessConfiguratorExtensions +public static class IMongoDataAccessBuilderExtensions { - public static IMongoDataAccessConfigurator AddRepository( - this IMongoDataAccessConfigurator configurator, + public static IMongoDataAccessBuilder AddRepository( + this IMongoDataAccessBuilder configurator, string collectionName, Action>? mapSetup = null, IReadOnlyList, Task>>? init = null diff --git a/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs b/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs index 6d77002ca..9d5792a9d 100644 --- a/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs @@ -2,22 +2,17 @@ public static class IServiceCollectionExtensions { - public static IServiceCollection AddMemoryDataAccess( - this IServiceCollection services, - Action configure - ) + public static IMemoryDataAccessBuilder AddMemoryDataAccess(this IServiceCollection services) { services.TryAddTransient(); services.TryAddScoped(); - configure(new MemoryDataAccessConfigurator(services)); - return services; + return new MemoryDataAccessBuilder(services); } - public static IServiceCollection AddMongoDataAccess( + public static IMongoDataAccessBuilder AddMongoDataAccess( this IServiceCollection services, string connectionString, - string entityNamespace, - Action configure + string entityNamespace ) { DataAccessClassMap.RegisterConventions( @@ -56,10 +51,10 @@ Action configure services.TryAddScoped(); services.TryAddScoped(sp => sp.GetRequiredService()); services.AddHostedService(); - var configurator = new MongoDataAccessConfigurator(services); + MongoDataAccessBuilder builder = new(services); // Configure the schema_versions repository - configurator.AddRepository( + builder.AddRepository( "schema_versions", init: [ @@ -72,7 +67,6 @@ Action configure ] ); - configure(configurator); - return services; + return builder; } } diff --git a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessConfigurator.cs b/src/DataAccess/src/SIL.DataAccess/MemoryDataAccessBuilder.cs similarity index 54% rename from src/DataAccess/src/SIL.DataAccess/MongoDataAccessConfigurator.cs rename to src/DataAccess/src/SIL.DataAccess/MemoryDataAccessBuilder.cs index c3d1e30cb..bce177e4a 100644 --- a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessConfigurator.cs +++ b/src/DataAccess/src/SIL.DataAccess/MemoryDataAccessBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -public class MongoDataAccessConfigurator(IServiceCollection services) : IMongoDataAccessConfigurator +public class MemoryDataAccessBuilder(IServiceCollection services) : IMemoryDataAccessBuilder { public IServiceCollection Services { get; } = services; } diff --git a/src/DataAccess/src/SIL.DataAccess/MemoryDataAccessConfigurator.cs b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessBuilder.cs similarity index 53% rename from src/DataAccess/src/SIL.DataAccess/MemoryDataAccessConfigurator.cs rename to src/DataAccess/src/SIL.DataAccess/MongoDataAccessBuilder.cs index 1be4df8b2..dfd1de2bd 100644 --- a/src/DataAccess/src/SIL.DataAccess/MemoryDataAccessConfigurator.cs +++ b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -public class MemoryDataAccessConfigurator(IServiceCollection services) : IMemoryDataAccessConfigurator +public class MongoDataAccessBuilder(IServiceCollection services) : IMongoDataAccessBuilder { public IServiceCollection Services { get; } = services; } diff --git a/src/DataAccess/src/SIL.DataAccess/SIL.DataAccess.csproj b/src/DataAccess/src/SIL.DataAccess/SIL.DataAccess.csproj index 2c83f8b0e..5a0be0fd1 100644 --- a/src/DataAccess/src/SIL.DataAccess/SIL.DataAccess.csproj +++ b/src/DataAccess/src/SIL.DataAccess/SIL.DataAccess.csproj @@ -24,4 +24,8 @@ + + + + diff --git a/src/DataAccess/src/SIL.DataAccess/Usings.cs b/src/DataAccess/src/SIL.DataAccess/Usings.cs index f1f9e4adf..3c46618ff 100644 --- a/src/DataAccess/src/SIL.DataAccess/Usings.cs +++ b/src/DataAccess/src/SIL.DataAccess/Usings.cs @@ -1,6 +1,5 @@ global using System.Collections; global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; global using System.Linq.Expressions; global using System.Reflection; global using Microsoft.Extensions.DependencyInjection.Extensions; diff --git a/src/Echo/src/EchoEngine/EchoEngine.csproj b/src/Echo/src/EchoEngine/EchoEngine.csproj index ced13544c..a118f7066 100644 --- a/src/Echo/src/EchoEngine/EchoEngine.csproj +++ b/src/Echo/src/EchoEngine/EchoEngine.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -13,15 +13,13 @@ - - - - + + - - + + diff --git a/src/Echo/src/EchoEngine/HealthServiceV1.cs b/src/Echo/src/EchoEngine/HealthServiceV1.cs deleted file mode 100644 index 025b2feed..000000000 --- a/src/Echo/src/EchoEngine/HealthServiceV1.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Serval.Health.V1; - -namespace EchoEngine; - -public class HealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase -{ - private readonly HealthCheckService _healthCheckService = healthCheckService; - - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } -} diff --git a/src/Echo/src/EchoEngine/IServalConfiguratorExtensions.cs b/src/Echo/src/EchoEngine/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..b7cb45ef8 --- /dev/null +++ b/src/Echo/src/EchoEngine/IServalConfiguratorExtensions.cs @@ -0,0 +1,13 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddEchoEngines(this IServalConfigurator configurator) + { + configurator.Services.AddHostedService(); + configurator.Services.AddSingleton(); + configurator.AddTranslationEngine("Echo"); + configurator.AddWordAlignmentEngine("EchoWordAlignment"); + return configurator; + } +} diff --git a/src/Echo/src/EchoEngine/Program.cs b/src/Echo/src/EchoEngine/Program.cs deleted file mode 100644 index 796260736..000000000 --- a/src/Echo/src/EchoEngine/Program.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Serval.Translation.V1; -using Serval.WordAlignment.V1; - -WebApplicationBuilder builder = WebApplication.CreateBuilder(args); - -// Add services to the container. -builder.Services.AddGrpcClient( - "Translation", - o => - { - o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!); - } -); -builder.Services.AddGrpcClient( - "WordAlignment", - o => - { - o.Address = new Uri(builder.Configuration.GetConnectionString("WordAlignmentPlatformApi")!); - } -); - -builder.Services.AddGrpc(); - -builder.Services.AddHostedService(); -builder.Services.AddSingleton(); - -builder.Services.AddParallelCorpusService(); - -builder.Services.AddHealthChecks().AddCheck("Live", () => HealthCheckResult.Healthy()); - -builder.Services.Configure(builder.Configuration.GetSection("Bugsnag")); -builder.Services.AddBugsnag(); -builder.Services.AddDiagnostics(); - -WebApplication app = builder.Build(); - -app.MapGrpcService(); -app.MapGrpcService(); - -app.MapGrpcService(); - -app.Run(); diff --git a/src/Echo/src/EchoEngine/Properties/launchSettings.json b/src/Echo/src/EchoEngine/Properties/launchSettings.json deleted file mode 100644 index 5d4c345dd..000000000 --- a/src/Echo/src/EchoEngine/Properties/launchSettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/launchsettings.json", - "profiles": { - "EchoEngine": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": false, - "applicationUrl": "https://localhost:8055", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Echo/src/EchoEngine/TranslationEngineService.cs b/src/Echo/src/EchoEngine/TranslationEngineService.cs new file mode 100644 index 000000000..e07fd36d5 --- /dev/null +++ b/src/Echo/src/EchoEngine/TranslationEngineService.cs @@ -0,0 +1,281 @@ +using Serval.Translation.Contracts; + +namespace EchoEngine; + +public class TranslationEngineService(BackgroundTaskQueue taskQueue, IParallelCorpusService parallelCorpusService) + : ITranslationEngineService +{ + private readonly BackgroundTaskQueue _taskQueue = taskQueue; + private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; + + public Task CreateAsync( + string engineId, + string sourceLanguage, + string targetLanguage, + string? engineName = null, + bool? isModelPersisted = null, + CancellationToken cancellationToken = default + ) + { + if (sourceLanguage != targetLanguage) + throw new InvalidOperationException("Source and target languages must be the same"); + return Task.CompletedTask; + } + + public Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + if (_taskQueue.ActiveBuilds.TryRemove(engineId, out (string buildId, CancellationTokenSource cts) build)) + { + build.cts.Cancel(); + return Task.FromResult(build.buildId); + } + + return Task.FromResult(null); + } + + public Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpdateAsync( + string engineId, + string? sourceLanguage, + string? targetLanguage, + CancellationToken cancellationToken = default + ) + { + if (sourceLanguage != targetLanguage) + throw new InvalidOperationException("Source and target languages must be the same"); + return Task.CompletedTask; + } + + public Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + string[] tokens = segment.Split(); + IReadOnlyList results = + [ + new TranslationResultContract + { + Translation = segment, + SourceTokens = tokens, + TargetTokens = tokens, + Confidences = Enumerable.Repeat(1.0, tokens.Length).ToArray(), + Sources = Enumerable + .Repeat>( + new HashSet { TranslationSource.Primary }, + tokens.Length + ) + .ToList(), + Alignment = Enumerable + .Range(0, tokens.Length) + .Select(i => new AlignedWordPairContract { SourceIndex = i, TargetIndex = i }) + .ToList(), + Phrases = + [ + new PhraseContract + { + SourceSegmentStart = 0, + SourceSegmentEnd = tokens.Length, + TargetSegmentCut = tokens.Length, + }, + ], + }, + ]; + return Task.FromResult(results); + } + + public Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + string[] tokens = segment.Split(); + var wordGraph = new WordGraphContract + { + InitialStateScore = 0.0, + SourceTokens = tokens, + FinalStates = new HashSet { tokens.Length }, + Arcs = Enumerable + .Range(0, tokens.Length - 1) + .Select(index => new WordGraphArcContract + { + PrevState = index, + NextState = index + 1, + Score = 1.0, + TargetTokens = [tokens[index]], + Confidences = [1.0], + SourceSegmentStart = index, + SourceSegmentEnd = index + 1, + Alignment = [new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }], + Sources = [new HashSet { TranslationSource.Primary }], + }) + .ToList(), + }; + return Task.FromResult(wordGraph); + } + + public Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) => Task.CompletedTask; + + public Task GetModelDownloadUrlAsync( + string engineId, + CancellationToken cancellationToken = default + ) + { + return Task.FromResult( + new ModelDownloadUrlContract + { + Url = "https://example.com/model", + ModelRevision = 1, + ExpiresAt = DateTime.UtcNow.AddHours(1), + } + ); + } + + public Task GetQueueSizeAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); + + public Task GetLanguageInfoAsync( + string language, + CancellationToken cancellationToken = default + ) + { + return Task.FromResult(new LanguageInfoContract { InternalCode = language + "_echo", IsNative = true }); + } + + public async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + string? options = null, + CancellationToken cancellationToken = default + ) + { + var cts = new CancellationTokenSource(); + if (!_taskQueue.ActiveBuilds.TryAdd(engineId, (buildId, cts))) + { + await _taskQueue.QueueBackgroundWorkItemAsync( + async (services, token) => + { + ITranslationPlatformService platform = services.GetRequiredService(); + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + ); + return; + } + + await _taskQueue.QueueBackgroundWorkItemAsync( + async (services, backgroundCt) => + { + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(backgroundCt, cts.Token); + ITranslationPlatformService platform = services.GetRequiredService(); + + try + { + await platform.BuildStartedAsync(buildId, linkedCts.Token); + + int trainCount = 0; + int pretranslateCount = 0; + + List pretranslations = []; + await _parallelCorpusService.PreprocessAsync( + corpora, + (row, _) => + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + trainCount++; + return Task.CompletedTask; + }, + (row, isInTrainingData, corpusId) => + { + string[] tokens = row.SourceSegment.Split(); + pretranslations.Add( + new PretranslationContract + { + CorpusId = corpusId, + TextId = row.TextId, + SourceRefs = row.SourceRefs.Select(r => r.ToString()!).ToArray(), + TargetRefs = row.TargetRefs.Select(r => r.ToString()!).ToArray(), + Translation = row.SourceSegment, + SourceTokens = tokens, + TranslationTokens = tokens, + Alignment = tokens + .Select( + (_, i) => new AlignedWordPairContract { SourceIndex = i, TargetIndex = i } + ) + .ToList(), + Confidence = 1.0, + } + ); + if (row.SourceSegment.Length > 0 && !isInTrainingData) + pretranslateCount++; + if (cts.IsCancellationRequested) + throw new OperationCanceledException(cts.Token); + return Task.CompletedTask; + }, + false + ); + + await platform.InsertPretranslationsAsync( + engineId, + pretranslations.ToAsyncEnumerable(), + linkedCts.Token + ); + + string sourceLanguage = + corpora.Count > 0 && corpora[0].SourceCorpora.Count > 0 + ? corpora[0].SourceCorpora[0].Language + : string.Empty; + string targetLanguage = + corpora.Count > 0 && corpora[0].TargetCorpora.Count > 0 + ? corpora[0].TargetCorpora[0].Language + : string.Empty; + + await platform.UpdateBuildExecutionDataAsync( + engineId, + buildId, + new ExecutionDataContract + { + TrainCount = trainCount, + PretranslateCount = pretranslateCount, + EngineSourceLanguageTag = sourceLanguage, + EngineTargetLanguageTag = targetLanguage, + ResolvedSourceLanguage = sourceLanguage, + ResolvedTargetLanguage = targetLanguage, + }, + linkedCts.Token + ); + + await platform.BuildCompletedAsync(buildId, 0, 1.0, CancellationToken.None); + } + catch (OperationCanceledException) + { + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + catch (Exception e) + { + if (cts.IsCancellationRequested) + { + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + else + { + await platform.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + } + } + finally + { + _taskQueue.ActiveBuilds.TryRemove(engineId, out _); + } + } + ); + } +} diff --git a/src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs deleted file mode 100644 index 53ca037cb..000000000 --- a/src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs +++ /dev/null @@ -1,386 +0,0 @@ -using Serval.Translation.V1; - -namespace EchoEngine; - -public class TranslationEngineServiceV1( - BackgroundTaskQueue taskQueue, - IParallelCorpusService parallelCorpusService, - TranslationPlatformApi.TranslationPlatformApiClient platformApiClient -) : TranslationEngineApi.TranslationEngineApiBase -{ - private static readonly Empty Empty = new(); - private readonly BackgroundTaskQueue _taskQueue = taskQueue; - private readonly TranslationPlatformApi.TranslationPlatformApiClient _platformApiClient = platformApiClient; - - private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; - - public override Task Create(CreateRequest request, ServerCallContext context) - { - if (request.SourceLanguage != request.TargetLanguage) - { - var status = new Status(StatusCode.InvalidArgument, "Source and target languages must be the same"); - throw new RpcException(status); - } - return Task.FromResult(Empty); - } - - public override Task CancelBuild(CancelBuildRequest request, ServerCallContext context) - { - if ( - _taskQueue.ActiveBuilds.TryRemove(request.EngineId, out (string buildId, CancellationTokenSource cts) build) - ) - { - build.cts.Cancel(); - return Task.FromResult(new CancelBuildResponse { BuildId = build.buildId }); - } - - throw new RpcException(new Status(StatusCode.FailedPrecondition, "No build running")); - } - - public override Task Delete(DeleteRequest request, ServerCallContext context) - { - return Task.FromResult(Empty); - } - - public override Task Update(UpdateRequest request, ServerCallContext context) - { - if (request.SourceLanguage != request.TargetLanguage) - { - var status = new Status(StatusCode.InvalidArgument, "Source and target languages must be the same"); - throw new RpcException(status); - } - - return Task.FromResult(Empty); - } - - public override Task Translate(TranslateRequest request, ServerCallContext context) - { - string[] tokens = request.Segment.Split(); - var response = new TranslateResponse - { - Results = - { - new TranslationResult - { - Translation = request.Segment, - SourceTokens = { tokens }, - TargetTokens = { tokens }, - Confidences = { Enumerable.Repeat(1.0, tokens.Length) }, - Sources = - { - Enumerable.Repeat( - new TranslationSources { Values = { TranslationSource.Primary } }, - tokens.Length - ), - }, - Alignment = - { - Enumerable - .Range(0, tokens.Length) - .Select(i => new AlignedWordPair { SourceIndex = i, TargetIndex = i }), - }, - Phrases = - { - new Phrase - { - SourceSegmentStart = 0, - SourceSegmentEnd = tokens.Length, - TargetSegmentCut = tokens.Length, - }, - }, - }, - }, - }; - return Task.FromResult(response); - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - var cts = new CancellationTokenSource(); - if (!_taskQueue.ActiveBuilds.TryAdd(request.EngineId, (request.BuildId, cts))) - { - await _platformApiClient.BuildCanceledAsync( - new BuildCanceledRequest { BuildId = request.BuildId }, - cancellationToken: context.CancellationToken - ); - return Empty; - } - - await _taskQueue.QueueBackgroundWorkItemAsync( - async (services, cancellationToken) => - { - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cts.Token); - TranslationPlatformApi.TranslationPlatformApiClient client = - services.GetRequiredService(); - - try - { - await client.BuildStartedAsync( - new BuildStartedRequest { BuildId = request.BuildId }, - cancellationToken: linkedCts.Token - ); - - int trainCount = 0; - int pretranslateCount = 0; - - List pretranslationsRequests = []; - await _parallelCorpusService.PreprocessAsync( - request.Corpora.Select(Map), - (row, _) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - return Task.CompletedTask; - }, - (row, isInTrainingData, corpusId) => - { - string[] tokens = row.SourceSegment.Split(); - pretranslationsRequests.Add( - new InsertPretranslationsRequest - { - EngineId = request.EngineId, - CorpusId = corpusId, - TextId = row.TextId, - SourceRefs = { row.SourceRefs.Select(r => r.ToString()) }, - TargetRefs = { row.TargetRefs.Select(r => r.ToString()) }, - Translation = row.SourceSegment, - SourceTokens = { tokens }, - TranslationTokens = { tokens }, - Alignment = - { - tokens.Select( - (_, i) => new AlignedWordPair() { SourceIndex = i, TargetIndex = i } - ), - }, - Confidence = 1.0, - } - ); - if (row.SourceSegment.Length > 0 && !isInTrainingData) - pretranslateCount++; - if (cts.IsCancellationRequested) - throw new OperationCanceledException(cts.Token); - - return Task.CompletedTask; - }, - false - ); - using ( - AsyncClientStreamingCall call = - client.InsertPretranslations(cancellationToken: linkedCts.Token) - ) - { - foreach (InsertPretranslationsRequest request in pretranslationsRequests) - { - await call.RequestStream.WriteAsync(request, linkedCts.Token); - } - - await call.RequestStream.CompleteAsync(); - await call; - } - - string sourceLanguage = - request.Corpora.FirstOrDefault()?.SourceCorpora.FirstOrDefault()?.Language ?? string.Empty; - string targetLanguage = - request.Corpora.FirstOrDefault()?.TargetCorpora.FirstOrDefault()?.Language ?? string.Empty; - await client.UpdateBuildExecutionDataAsync( - new UpdateBuildExecutionDataRequest - { - EngineId = request.EngineId, - BuildId = request.BuildId, - ExecutionData = new ExecutionData - { - TrainCount = trainCount, - PretranslateCount = pretranslateCount, - EngineSourceLanguageTag = sourceLanguage, - EngineTargetLanguageTag = targetLanguage, - ResolvedSourceLanguage = sourceLanguage, - ResolvedTargetLanguage = targetLanguage, - }, - }, - cancellationToken: linkedCts.Token - ); - - await client.BuildCompletedAsync( - new BuildCompletedRequest { BuildId = request.BuildId, Confidence = 1.0 }, - cancellationToken: CancellationToken.None - ); - } - catch (OperationCanceledException) - { - await client.BuildCanceledAsync( - new BuildCanceledRequest { BuildId = request.BuildId }, - cancellationToken: CancellationToken.None - ); - } - catch (Exception e) - { - if (cts.IsCancellationRequested) - { - // This will be an RpcException resulting from the token cancellation - // occurring during an RPC call. - await client.BuildCanceledAsync( - new BuildCanceledRequest { BuildId = request.BuildId }, - cancellationToken: CancellationToken.None - ); - } - else - { - await client.BuildFaultedAsync( - new BuildFaultedRequest { BuildId = request.BuildId, Message = e.Message }, - cancellationToken: CancellationToken.None - ); - } - } - finally - { - _taskQueue.ActiveBuilds.TryRemove(request.EngineId, out _); - } - } - ); - - return Empty; - } - - public override Task TrainSegmentPair(TrainSegmentPairRequest request, ServerCallContext context) - { - return Task.FromResult(Empty); - } - - public override Task GetWordGraph(GetWordGraphRequest request, ServerCallContext context) - { - string[] tokens = request.Segment.Split(); - return Task.FromResult( - new GetWordGraphResponse - { - WordGraph = new WordGraph - { - InitialStateScore = 0.0, - SourceTokens = { tokens }, - Arcs = - { - Enumerable - .Range(0, tokens.Length - 1) - .Select(index => new WordGraphArc - { - PrevState = index, - NextState = index + 1, - Score = 1.0, - TargetTokens = { tokens[index] }, - Confidences = { 1.0 }, - SourceSegmentStart = index, - SourceSegmentEnd = index + 1, - Alignment = - { - new AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - }, - }), - }, - FinalStates = { tokens.Length }, - }, - } - ); - } - - public override Task GetModelDownloadUrl( - GetModelDownloadUrlRequest request, - ServerCallContext context - ) - { - var response = new GetModelDownloadUrlResponse - { - Url = "https://example.com/model", - ModelRevision = 1, - ExpiresAt = DateTime.UtcNow.AddHours(1).ToTimestamp(), - }; - return Task.FromResult(response); - } - - public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) - { - return Task.FromResult(new GetQueueSizeResponse { Size = 0 }); - } - - public override Task GetLanguageInfo( - GetLanguageInfoRequest request, - ServerCallContext context - ) - { - return Task.FromResult( - new GetLanguageInfoResponse { InternalCode = request.Language + "_echo", IsNative = true } - ); - } - - private static SIL.ServiceToolkit.Models.ParallelCorpus Map(ParallelCorpus source) - { - return new SIL.ServiceToolkit.Models.ParallelCorpus - { - Id = source.Id, - SourceCorpora = source.SourceCorpora.Select(Map).ToList(), - TargetCorpora = source.TargetCorpora.Select(Map).ToList(), - }; - } - - private static SIL.ServiceToolkit.Models.MonolingualCorpus Map(MonolingualCorpus source) - { - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); - FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll); - - var pretranslateChapters = source.PretranslateChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var pretranslateTextIds = source.PretranslateTextIds.ToHashSet(); - FilterChoice pretranslateFilter = GetFilterChoice( - pretranslateChapters, - pretranslateTextIds, - source.PretranslateAll - ); - - return new SIL.ServiceToolkit.Models.MonolingualCorpus - { - Id = source.Id, - Language = source.Language, - Files = source.Files.Select(Map).ToList(), - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? trainOnTextIds : null, - InferenceChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, - InferenceTextIds = pretranslateFilter == FilterChoice.TextIds ? pretranslateTextIds : null, - }; - } - - private static SIL.ServiceToolkit.Models.CorpusFile Map(CorpusFile source) - { - return new SIL.ServiceToolkit.Models.CorpusFile - { - Location = source.Location, - Format = (SIL.ServiceToolkit.Models.FileFormat)source.Format, - TextId = source.TextId, - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None, - } - - private static FilterChoice GetFilterChoice( - IReadOnlyDictionary> chapters, - HashSet textIds, - bool noFilter - ) - { - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (noFilter || chapters is null && textIds is null) - return FilterChoice.None; - if (chapters is null || chapters.Count == 0) - return FilterChoice.TextIds; - return FilterChoice.Chapters; - } -} diff --git a/src/Echo/src/EchoEngine/Usings.cs b/src/Echo/src/EchoEngine/Usings.cs index 3a57dd603..3f993582e 100644 --- a/src/Echo/src/EchoEngine/Usings.cs +++ b/src/Echo/src/EchoEngine/Usings.cs @@ -1,8 +1,7 @@ global using System.Collections.Concurrent; global using System.Threading.Channels; -global using Bugsnag.AspNet.Core; global using EchoEngine; -global using Google.Protobuf.WellKnownTypes; -global using Grpc.Core; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using SIL.ServiceToolkit.Services; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Serval.Shared.Contracts; diff --git a/src/Echo/src/EchoEngine/WordAlignmentEngineService.cs b/src/Echo/src/EchoEngine/WordAlignmentEngineService.cs new file mode 100644 index 000000000..491fdec5c --- /dev/null +++ b/src/Echo/src/EchoEngine/WordAlignmentEngineService.cs @@ -0,0 +1,193 @@ +using Serval.WordAlignment.Contracts; + +namespace EchoEngine; + +public class WordAlignmentEngineService(BackgroundTaskQueue taskQueue, IParallelCorpusService parallelCorpusService) + : IWordAlignmentEngineService +{ + private readonly BackgroundTaskQueue _taskQueue = taskQueue; + private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; + + public Task CreateAsync( + string engineId, + string sourceLanguage, + string targetLanguage, + string? engineName = null, + CancellationToken cancellationToken = default + ) + { + if (sourceLanguage != targetLanguage) + throw new InvalidOperationException("Source and target languages must be the same"); + return Task.CompletedTask; + } + + public Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task AlignAsync( + string engineId, + string sourceSegment, + string targetSegment, + CancellationToken cancellationToken = default + ) + { + string[] sourceTokens = sourceSegment.Split(); + string[] targetTokens = targetSegment.Split(); + int minLength = Math.Min(sourceTokens.Length, targetTokens.Length); + + var result = new WordAlignmentResultContract + { + SourceTokens = sourceTokens, + TargetTokens = targetTokens, + Alignment = Enumerable + .Range(0, minLength) + .Select(i => new AlignedWordPairContract + { + SourceIndex = i, + TargetIndex = i, + Score = 1.0, + }) + .ToList(), + }; + return Task.FromResult(result); + } + + public Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + if (_taskQueue.ActiveBuilds.TryRemove(engineId, out (string buildId, CancellationTokenSource cts) build)) + { + build.cts.Cancel(); + return Task.FromResult(build.buildId); + } + + return Task.FromResult(null); + } + + public Task GetQueueSizeAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); + + public async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + string? options = null, + CancellationToken cancellationToken = default + ) + { + var cts = new CancellationTokenSource(); + if (!_taskQueue.ActiveBuilds.TryAdd(engineId, (buildId, cts))) + { + await _taskQueue.QueueBackgroundWorkItemAsync( + async (services, token) => + { + IWordAlignmentPlatformService platform = + services.GetRequiredService(); + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + ); + return; + } + + await _taskQueue.QueueBackgroundWorkItemAsync( + async (services, backgroundCt) => + { + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(backgroundCt, cts.Token); + IWordAlignmentPlatformService platform = services.GetRequiredService(); + + try + { + await platform.BuildStartedAsync(buildId, linkedCts.Token); + + int trainCount = 0; + int wordAlignCount = 0; + + List wordAlignments = []; + await _parallelCorpusService.PreprocessAsync( + corpora, + (row, _) => + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + trainCount++; + return Task.CompletedTask; + }, + (row, isInTrainingData, corpusId) => + { + string[] sourceTokens = row.SourceSegment.Split(); + string[] targetTokens = row.TargetSegment.Split(); + int minLength = Math.Min(sourceTokens.Length, targetTokens.Length); + + wordAlignments.Add( + new WordAlignmentContract + { + CorpusId = corpusId, + TextId = row.TextId, + SourceRefs = row.SourceRefs.Select(r => r.ToString()!).ToArray(), + TargetRefs = row.TargetRefs.Select(r => r.ToString()!).ToArray(), + SourceTokens = sourceTokens, + TargetTokens = targetTokens, + Alignment = Enumerable + .Range(0, minLength) + .Select(i => new AlignedWordPairContract { SourceIndex = i, TargetIndex = i }) + .ToList(), + } + ); + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) + wordAlignCount++; + if (cts.IsCancellationRequested) + throw new OperationCanceledException(cts.Token); + return Task.CompletedTask; + }, + false + ); + + await platform.InsertWordAlignmentsAsync( + engineId, + wordAlignments.ToAsyncEnumerable(), + linkedCts.Token + ); + + string sourceLanguage = + corpora.Count > 0 && corpora[0].SourceCorpora.Count > 0 + ? corpora[0].SourceCorpora[0].Language + : string.Empty; + string targetLanguage = + corpora.Count > 0 && corpora[0].TargetCorpora.Count > 0 + ? corpora[0].TargetCorpora[0].Language + : string.Empty; + + await platform.UpdateBuildExecutionDataAsync( + engineId, + buildId, + new ExecutionDataContract + { + TrainCount = trainCount, + WordAlignCount = wordAlignCount, + EngineSourceLanguageTag = sourceLanguage, + EngineTargetLanguageTag = targetLanguage, + }, + linkedCts.Token + ); + + await platform.BuildCompletedAsync(buildId, 0, 1.0, CancellationToken.None); + } + catch (OperationCanceledException) + { + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + catch (Exception e) + { + if (cts.IsCancellationRequested) + { + await platform.BuildCanceledAsync(buildId, CancellationToken.None); + } + else + { + await platform.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + } + } + finally + { + _taskQueue.ActiveBuilds.TryRemove(engineId, out _); + } + } + ); + } +} diff --git a/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs b/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs deleted file mode 100644 index 74ad1f07d..000000000 --- a/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs +++ /dev/null @@ -1,241 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace EchoEngine; - -public class WordAlignmentEngineServiceV1(BackgroundTaskQueue taskQueue, IParallelCorpusService parallelCorpusService) - : WordAlignmentEngineApi.WordAlignmentEngineApiBase -{ - private static readonly Empty Empty = new(); - private readonly BackgroundTaskQueue _taskQueue = taskQueue; - private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; - - public override Task Create(CreateRequest request, ServerCallContext context) - { - if (request.SourceLanguage != request.TargetLanguage) - { - var status = new Status(StatusCode.InvalidArgument, "Source and target languages must be the same"); - throw new RpcException(status); - } - return Task.FromResult(Empty); - } - - public override Task Delete(DeleteRequest request, ServerCallContext context) - { - return Task.FromResult(Empty); - } - - public static IEnumerable GenerateAlignedWordPairs(int number) - { - if (number < 0) - throw new ArgumentOutOfRangeException(nameof(number), "Number must be non-negative"); - return Enumerable - .Range(0, number) - .Select(i => new AlignedWordPair - { - SourceIndex = i, - TargetIndex = i, - Score = 1.0, - }); - } - - public override Task GetWordAlignment( - GetWordAlignmentRequest request, - ServerCallContext context - ) - { - string[] sourceTokens = request.SourceSegment.Split(); - string[] targetTokens = request.TargetSegment.Split(); - int minLength = Math.Min(sourceTokens.Length, targetTokens.Length); - - var response = new GetWordAlignmentResponse - { - Result = new WordAlignmentResult - { - SourceTokens = { sourceTokens }, - TargetTokens = { targetTokens }, - Alignment = { GenerateAlignedWordPairs(minLength) }, - }, - }; - return Task.FromResult(response); - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - await _taskQueue.QueueBackgroundWorkItemAsync( - async (services, cancellationToken) => - { - WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client = - services.GetRequiredService(); - await client.BuildStartedAsync( - new BuildStartedRequest { BuildId = request.BuildId }, - cancellationToken: cancellationToken - ); - - try - { - int trainCount = 0; - int wordAlignCount = 0; - List wordAlignmentsRequests = []; - await _parallelCorpusService.PreprocessAsync( - request.Corpora.Select(Map), - (row, _) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - return Task.CompletedTask; - }, - (row, isInTrainingData, corpusId) => - { - wordAlignmentsRequests.Add( - new InsertWordAlignmentsRequest - { - EngineId = request.EngineId, - CorpusId = corpusId, - TextId = row.TextId, - SourceRefs = { row.SourceRefs.Select(r => r.ToString()) }, - TargetRefs = { row.TargetRefs.Select(r => r.ToString()) }, - SourceTokens = { row.SourceSegment.Split() }, - TargetTokens = { row.TargetSegment.Split() }, - Alignment = - { - row - .SourceSegment.Split() - .Select( - (_, i) => new AlignedWordPair() { SourceIndex = i, TargetIndex = i } - ), - }, - } - ); - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) - wordAlignCount++; - return Task.CompletedTask; - }, - false - ); - using ( - AsyncClientStreamingCall call = client.InsertWordAlignments( - cancellationToken: cancellationToken - ) - ) - { - foreach (InsertWordAlignmentsRequest request in wordAlignmentsRequests) - { - await call.RequestStream.WriteAsync(request, cancellationToken); - } - } - - string sourceLanguage = - request.Corpora.FirstOrDefault()?.SourceCorpora.FirstOrDefault()?.Language ?? string.Empty; - string targetLanguage = - request.Corpora.FirstOrDefault()?.TargetCorpora.FirstOrDefault()?.Language ?? string.Empty; - await client.UpdateBuildExecutionDataAsync( - new UpdateBuildExecutionDataRequest - { - EngineId = request.EngineId, - BuildId = request.BuildId, - ExecutionData = new ExecutionData - { - TrainCount = trainCount, - WordAlignCount = wordAlignCount, - EngineSourceLanguageTag = sourceLanguage, - EngineTargetLanguageTag = targetLanguage, - }, - }, - cancellationToken: cancellationToken - ); - - await client.BuildCompletedAsync( - new BuildCompletedRequest { BuildId = request.BuildId, Confidence = 1.0 }, - cancellationToken: CancellationToken.None - ); - } - catch (OperationCanceledException) - { - await client.BuildCanceledAsync( - new BuildCanceledRequest { BuildId = request.BuildId }, - cancellationToken: CancellationToken.None - ); - } - catch (Exception e) - { - await client.BuildFaultedAsync( - new BuildFaultedRequest { BuildId = request.BuildId, Message = e.Message }, - cancellationToken: CancellationToken.None - ); - } - } - ); - - return Empty; - } - - private static SIL.ServiceToolkit.Models.ParallelCorpus Map(ParallelCorpus source) - { - return new SIL.ServiceToolkit.Models.ParallelCorpus - { - Id = source.Id, - SourceCorpora = source.SourceCorpora.Select(Map).ToList(), - TargetCorpora = source.TargetCorpora.Select(Map).ToList(), - }; - } - - private static SIL.ServiceToolkit.Models.MonolingualCorpus Map(MonolingualCorpus source) - { - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); - FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll); - - var wordAlignChapters = source.WordAlignOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var wordAlignTextIds = source.WordAlignOnTextIds.ToHashSet(); - FilterChoice wordAlignFilter = GetFilterChoice(wordAlignChapters, wordAlignTextIds, source.WordAlignOnAll); - - return new SIL.ServiceToolkit.Models.MonolingualCorpus - { - Id = source.Id, - Language = source.Language, - Files = source.Files.Select(Map).ToList(), - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? trainOnTextIds : null, - InferenceChapters = wordAlignFilter == FilterChoice.Chapters ? wordAlignChapters : null, - InferenceTextIds = wordAlignFilter == FilterChoice.TextIds ? wordAlignTextIds : null, - }; - } - - private static SIL.ServiceToolkit.Models.CorpusFile Map(CorpusFile source) - { - return new SIL.ServiceToolkit.Models.CorpusFile - { - Location = source.Location, - Format = (SIL.ServiceToolkit.Models.FileFormat)source.Format, - TextId = source.TextId, - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None, - } - - private static FilterChoice GetFilterChoice( - IReadOnlyDictionary> chapters, - HashSet textIds, - bool noFilter - ) - { - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (noFilter || chapters is null && textIds is null) - return FilterChoice.None; - if (chapters is null || chapters.Count == 0) - return FilterChoice.TextIds; - return FilterChoice.Chapters; - } -} diff --git a/src/Echo/src/EchoEngine/appsettings.Development.json b/src/Echo/src/EchoEngine/appsettings.Development.json deleted file mode 100644 index 98ef11002..000000000 --- a/src/Echo/src/EchoEngine/appsettings.Development.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "ConnectionStrings": { - "TranslationPlatformApi": "https://localhost:8444", - "WordAlignmentPlatformApi": "https://localhost:8444" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Development" - } -} \ No newline at end of file diff --git a/src/Echo/src/EchoEngine/appsettings.Production.json b/src/Echo/src/EchoEngine/appsettings.Production.json deleted file mode 100644 index 2ed02f22c..000000000 --- a/src/Echo/src/EchoEngine/appsettings.Production.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Production" - } -} \ No newline at end of file diff --git a/src/Echo/src/EchoEngine/appsettings.Staging.json b/src/Echo/src/EchoEngine/appsettings.Staging.json deleted file mode 100644 index 05d79b0c2..000000000 --- a/src/Echo/src/EchoEngine/appsettings.Staging.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Staging" - } -} \ No newline at end of file diff --git a/src/Echo/src/EchoEngine/appsettings.json b/src/Echo/src/EchoEngine/appsettings.json deleted file mode 100644 index cee50463a..000000000 --- a/src/Echo/src/EchoEngine/appsettings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient": "Warning" - } - }, - "Bugsnag": { - "ApiKey": "3fdfe53eddd9e876a50dcc075823c19e", - "NotifyReleaseStages": [ - "Staging", - "QA", - "Production" - ] - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/Program.cs b/src/Machine/src/Serval.Machine.EngineServer/Program.cs deleted file mode 100644 index 3520c98b0..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Program.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Hangfire; -using OpenTelemetry.Trace; - -WebApplicationBuilder builder = WebApplication.CreateBuilder(args); - -// Add services to the container. -builder - .Services.AddMachine(builder.Configuration) - .AddServalTranslationEngineService() - .AddServalWordAlignmentEngineService() - .AddServalTranslationPlatformService() - .AddServalWordAlignmentPlatformService() - .AddModelCleanupService() - .AddMessageOutboxDeliveryService(); - -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -WebApplication app = builder.Build(); - -app.MapServalTranslationEngineService(); -app.MapServalWordAlignmentEngineService(); -app.MapHangfireDashboard(); - -app.Run(); diff --git a/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json deleted file mode 100644 index 34eb2e94b..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.EngineServer": { - "commandName": "Project", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9000" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj b/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj deleted file mode 100644 index eea857fa3..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj +++ /dev/null @@ -1,34 +0,0 @@ - - - - net10.0 - enable - enable - 34e222a9-ef76-48f9-869e-338547f9bd25 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json deleted file mode 100644 index 171a5820c..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://silnlp/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Development" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json deleted file mode 100644 index 2ed02f22c..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Production" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json deleted file mode 100644 index 05d79b0c2..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Staging" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json deleted file mode 100644 index 401bf1baa..000000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_engine" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "WordAlignmentEngines": [ - "Statistical" - ], - "BuildJob": { - "ClearML": [ - { - "EngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "EngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "jobs_backlog.cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "EngineType": "Statistical", - "ModelType": "thot", - "Queue": "jobs_backlog.cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ] - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "StatisticalEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": true - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient": "Warning" - } - }, - "Bugsnag": { - "ApiKey": "3fdfe53eddd9e876a50dcc075823c19e", - "NotifyReleaseStages": [ - "Staging", - "QA", - "Production" - ] - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Program.cs b/src/Machine/src/Serval.Machine.JobServer/Program.cs deleted file mode 100644 index 7bd7a1891..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/Program.cs +++ /dev/null @@ -1,28 +0,0 @@ -using OpenTelemetry.Trace; - -WebApplicationBuilder builder = WebApplication.CreateBuilder(args); - -builder - .Services.AddMachine(builder.Configuration) - .AddHangfireJobServer() - .AddServalTranslationPlatformService() - .AddServalWordAlignmentPlatformService(); - -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -WebApplication app = builder.Build(); - -app.Run(); diff --git a/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json deleted file mode 100644 index f636d0c39..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.JobServer": { - "commandName": "Project", - "launchBrowser": false, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9100" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj b/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj deleted file mode 100644 index 7a893ba73..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - net10.0 - enable - enable - aa9e7440-5a04-4de6-ba51-bab9ef4a62e1 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json deleted file mode 100644 index cef2edc8d..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://silnlp/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient.Default": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Development" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json deleted file mode 100644 index 2ed02f22c..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Production" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json deleted file mode 100644 index 05d79b0c2..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "Bugsnag": { - "ReleaseStage": "Staging" - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.json deleted file mode 100644 index 7a23f04c4..000000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_job" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "WordAlignmentEngines": [ - "Statistical" - ], - "BuildJob": { - "ClearML": [ - { - "EngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "EngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "jobs_backlog.cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "EngineType": "Statistical", - "ModelType": "thot", - "Queue": "jobs_backlog.cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ], - "PreserveBuildFiles": true - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "StatisticalEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": false - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient": "Warning" - } - }, - "Bugsnag": { - "ApiKey": "3fdfe53eddd9e876a50dcc075823c19e", - "NotifyReleaseStages": [ - "Staging", - "QA", - "Production" - ] - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs deleted file mode 100644 index 1392a7912..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Microsoft.AspNetCore.Builder; - -public static class IEndpointRouteBuilderExtensions -{ - public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - builder.MapGrpcService(); - - return builder; - } - - public static IEndpointRouteBuilder MapServalWordAlignmentEngineService(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - - return builder; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs deleted file mode 100644 index ce0180b52..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public interface IMachineBuilder -{ - IServiceCollection Services { get; } - IConfiguration Configuration { get; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs deleted file mode 100644 index ae000d3d8..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ /dev/null @@ -1,542 +0,0 @@ -using Serval.Translation.V1; -using Serval.WordAlignment.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddStatisticalEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddDistributedReaderWriterLockOptions( - this IMachineBuilder build, - IConfiguration config - ) - { - build.Services.Configure(config); - return build; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddBuildJobOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - return builder.AddThotSmtModel(builder.Configuration.GetSection(ThotSmtModelOptions.Key)); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddWordAlignmentModel(this IMachineBuilder builder) - { - builder.Services.Configure( - builder.Configuration.GetSection(ThotWordAlignmentModelOptions.Key) - ); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("ClearML"); - if (connectionString is null) - throw new InvalidOperationException("ClearML connection string is required"); - - builder - .Services.AddHttpClient("ClearML") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) - .AddPolicyHandler( - (serviceProvider, _) => - Policy - .Handle() - .OrTransientHttpStatusCode() - .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests) - .WaitAndRetryAsync( - 7, - retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt), // total 56, less than the 1 minute limit - onRetryAsync: (outcome, timespan, retryAttempt, context) => - { - if (retryAttempt < 3) - return Task.CompletedTask; - // Log the retry attempt - var logger = serviceProvider.GetRequiredService>(); - logger.LogInformation( - "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}", - retryAttempt, - timespan, - outcome.Exception?.Message - ); - return Task.CompletedTask; - } - ) - ); - - builder.Services.AddSingleton(); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder - .Services.AddHttpClient("ClearML-NoRetry") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); - - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - return builder; - } - - private static MongoStorageOptions GetMongoStorageOptions() - { - var mongoStorageOptions = new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy(), - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - }; - return mongoStorageOptions; - } - - public static IMachineBuilder AddMongoHangfireJobClient(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("Hangfire"); - if (connectionString is null) - throw new InvalidOperationException("Hangfire connection string is required"); - - builder.Services.AddHangfire(c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage(connectionString, GetMongoStorageOptions()) - .UseFilter(new AutomaticRetryAttribute { Attempts = 0 }) - ); - builder.Services.AddHealthChecks().AddHangfire(); - return builder; - } - - public static IMachineBuilder AddHangfireJobServer(this IMachineBuilder builder) - { - IEnumerable engineTypes = ( - builder.Configuration.GetSection("TranslationEngines").Get() - ?? [EngineType.SmtTransfer, EngineType.Nmt] - ).Concat( - builder.Configuration.GetSection("WordAlignmentEngines").Get() ?? [EngineType.Statistical] - ); - var queues = new List(); - foreach (EngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case EngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.AddThotSmtTransferEngine(); - queues.Add("smt_transfer"); - break; - case EngineType.Nmt: - queues.Add("nmt"); - break; - case EngineType.Statistical: - builder.Services.AddSingleton(); - builder.AddThotStatisticalWordAlignment(); - queues.Add("statistical"); - break; - default: - throw new ArgumentOutOfRangeException(engineType.ToString()); - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("Mongo"); - if (connectionString is null) - throw new InvalidOperationException("Mongo connection string is required"); - builder.Services.AddMongoDataAccess( - connectionString, - "Serval.Machine.Shared.Models", - o => - { - o.AddRepository( - "translation_engines", - init: - [ - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.EngineId) - ) - ), - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) - ) - ), - ] - ); - o.AddRepository( - "word_alignment_engines", - init: - [ - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.EngineId) - ) - ), - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => - e.CurrentBuild!.BuildJobRunner - ) - ) - ), - ] - ); - o.AddRepository("locks"); - o.AddRepository( - "train_segment_pairs", - init: - [ - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ), - ] - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddMongoOutbox(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("Mongo"); - if (connectionString is null) - throw new InvalidOperationException("Mongo connection string is required"); - builder.Services.AddOutbox(builder.Configuration, x => x.UseMongo(connectionString)); - builder.Services.AddHealthChecks().AddOutbox(); - return builder; - } - - public static IMachineBuilder AddMessageOutboxDeliveryService(this IMachineBuilder builder) - { - builder.Services.AddOutbox(x => x.UseDeliveryService()); - return builder; - } - - public static IMachineBuilder AddServalTranslationPlatformService(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("Serval"); - if (connectionString is null) - throw new InvalidOperationException("Serval connection string is required"); - - builder.Services.AddKeyedScoped(EngineGroup.Translation); - - builder.Services.AddOutbox(x => - { - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - }); - - builder - .Services.AddGrpcClient(o => - { - o.Address = new Uri(connectionString); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable }, - }, - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateTranslationBuildStatus", - }, - }, - }, - }, - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalWordAlignmentPlatformService(this IMachineBuilder builder) - { - string? connectionString = builder.Configuration.GetConnectionString("Serval"); - if (connectionString is null) - throw new InvalidOperationException("Serval connection string is required"); - - builder.Services.AddKeyedScoped( - EngineGroup.WordAlignment - ); - - builder.Services.AddOutbox(x => - { - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - }); - - builder - .Services.AddGrpcClient(o => - { - o.Address = new Uri(connectionString); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable }, - }, - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.word_alignment.v1.WordAlignmentPlatformApi", - Method = "UpdateWordAlignmentBuildStatus", - }, - }, - }, - }, - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService(this IMachineBuilder builder) - { - builder.Services.AddGrpc(options => - { - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - }); - - IEnumerable engineTypes = - builder.Configuration.GetSection("TranslationEngines").Get() - ?? [EngineType.SmtTransfer, EngineType.Nmt]; - foreach (EngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case EngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtTransferEngine(); - builder.Services.AddScoped(); - break; - case EngineType.Nmt: - builder.Services.AddScoped(); - break; - default: - throw new InvalidEnumArgumentException(nameof(engineType), (int)engineType, typeof(EngineType)); - } - } - - return builder; - } - - public static IMachineBuilder AddServalWordAlignmentEngineService(this IMachineBuilder builder) - { - builder.Services.AddGrpc(options => - { - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - options.Interceptors.Add(); - }); - - IEnumerable engineTypes = - builder.Configuration.GetSection("WordAlignmentEngines").Get() ?? [EngineType.Statistical]; - - foreach (EngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case EngineType.Statistical: - builder.Services.AddSingleton(); - builder.AddThotStatisticalWordAlignment(); - builder.Services.AddScoped(); - builder.Services.AddHostedService(); - break; - default: - throw new ArgumentOutOfRangeException(engineType.ToString()); - } - } - - return builder; - } - - public static IMachineBuilder AddThotStatisticalWordAlignment(this IMachineBuilder builder) - { - builder.AddWordAlignmentModel(); - return builder; - } - - public static IMachineBuilder AddThotSmtTransferEngine(this IMachineBuilder builder) - { - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - return builder; - } - - public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder) - { - builder.Services.AddScoped, TranslationBuildJobService>(); - builder.Services.AddScoped, BuildJobService>(); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - builder.Services.AddSingleton(); - builder.Services.AddSingleton(x => x.GetRequiredService()); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - var smtTransferEngineOptions = new SmtTransferEngineOptions(); - builder.Configuration.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); - string? smtDriveLetter = Path.GetPathRoot(smtTransferEngineOptions.EnginesDir)?[..1]; - var statisticalEngineOptions = new StatisticalEngineOptions(); - builder.Configuration.GetSection(StatisticalEngineOptions.Key).Bind(statisticalEngineOptions); - string? statisticsDriveLetter = Path.GetPathRoot(statisticalEngineOptions.EnginesDir)?[..1]; - if (smtDriveLetter is null || statisticsDriveLetter is null) - throw new InvalidOperationException("SMT Engine and Statistical directory is required"); - if (smtDriveLetter != statisticsDriveLetter) - throw new InvalidOperationException("SMT Engine and Statistical directory must be on the same drive"); - // add health check for disk storage capacity - builder - .Services.AddHealthChecks() - .AddDiskStorageHealthCheck( - x => x.AddDrive(smtDriveLetter, 1_000), // 1GB - "SMT and Statistical Engine Storage Capacity", - HealthStatus.Degraded - ); - - return builder; - } - - public static IMachineBuilder AddModelCleanupService(this IMachineBuilder builder) - { - builder.Services.AddHostedService(); - return builder; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IServalConfiguratorExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..16846d559 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IServalConfiguratorExtensions.cs @@ -0,0 +1,242 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddMachineEngines(this IServalConfigurator configurator) + { + IConfiguration configuration = configurator.Configuration; + IServiceCollection services = configurator.Services; + + if (!Sldr.IsInitialized) + Sldr.Initialize(); + + services.AddMemoryCache(); + services.AddSingleton(); + services.AddHealthChecks().AddCheck("S3 Bucket"); + + services.AddSingleton(); + + services.AddScoped(); + services.AddStartupTask( + (sp, cancellationToken) => + sp.GetRequiredService().InitAsync(cancellationToken) + ); + + services.Configure(configuration.GetSection(ServiceOptions.Key)); + services.Configure(configuration.GetSection(SharedFileOptions.Key)); + services.Configure(configuration.GetSection(SmtTransferEngineOptions.Key)); + services.Configure(configuration.GetSection(StatisticalEngineOptions.Key)); + services.Configure(configuration.GetSection(ClearMLOptions.Key)); + services.Configure( + configuration.GetSection(DistributedReaderWriterLockOptions.Key) + ); + services.Configure(configuration.GetSection(BuildJobOptions.Key)); + + services.AddHostedService(); + + configurator.AddBuildJobService(); + configurator.AddMachineDataAccess(); + configurator.AddClearMLService(); + + configurator.AddTranslationEngines(); + configurator.AddWordAlignmentEngines(); + + return configurator; + } + + private static IServalConfigurator AddTranslationEngines(this IServalConfigurator configurator) + { + configurator.Services.AddKeyedScoped( + EngineGroup.Translation + ); + + // SMT Transfer Engine + configurator.Services.AddSingleton(); + configurator.Services.AddHostedService(); + configurator.Services.Configure( + configurator.Configuration.GetSection(ThotSmtModelOptions.Key) + ); + configurator.Services.AddSingleton(); + configurator.Services.AddSingleton(); + configurator.Services.AddSingleton(); + configurator.AddTranslationEngine(EngineType.SmtTransfer.ToString()); + configurator.JobQueues.Add(BuildJobQueues.SmtTransfer); + + // NMT Engine + configurator.AddTranslationEngine(EngineType.Nmt.ToString()); + configurator.JobQueues.Add(BuildJobQueues.Nmt); + + return configurator; + } + + private static IServalConfigurator AddWordAlignmentEngines(this IServalConfigurator configurator) + { + configurator.Services.AddKeyedScoped( + EngineGroup.WordAlignment + ); + + // Statistical Engine + configurator.Services.AddSingleton(); + configurator.Services.Configure( + configurator.Configuration.GetSection(ThotWordAlignmentModelOptions.Key) + ); + configurator.Services.AddSingleton(); + configurator.AddWordAlignmentEngine(EngineType.Statistical.ToString()); + configurator.Services.AddHostedService(); + configurator.JobQueues.Add(BuildJobQueues.Statistical); + + return configurator; + } + + private static IServiceCollection AddStartupTask( + this IServiceCollection services, + Func startupTask + ) + { + services.AddHostedService(sp => new StartupTask(sp, startupTask)); + return services; + } + + private static IServalConfigurator AddClearMLService(this IServalConfigurator builder) + { + string? connectionString = builder.Configuration.GetConnectionString("ClearML"); + if (connectionString is null) + throw new InvalidOperationException("ClearML connection string is required"); + + builder + .Services.AddHttpClient("ClearML") + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) + .AddPolicyHandler( + (serviceProvider, _) => + Policy + .Handle() + .OrTransientHttpStatusCode() + .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests) + .WaitAndRetryAsync( + 7, + retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt), // total 56, less than the 1 minute limit + onRetryAsync: (outcome, timespan, retryAttempt, context) => + { + if (retryAttempt < 3) + return Task.CompletedTask; + // Log the retry attempt + var logger = serviceProvider.GetRequiredService>(); + logger.LogInformation( + "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}", + retryAttempt, + timespan, + outcome.Exception?.Message + ); + return Task.CompletedTask; + } + ) + ); + + builder.Services.AddSingleton(); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder + .Services.AddHttpClient("ClearML-NoRetry") + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); + + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + return builder; + } + + public static IServalConfigurator AddMachineDataAccess(this IServalConfigurator configurator) + { + configurator.DataAccess.AddRepository( + "machine.translation_engines", + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.EngineId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) + ) + ), + ] + ); + configurator.DataAccess.AddRepository( + "machine.word_alignment_engines", + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.EngineId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) + ) + ), + ] + ); + configurator.DataAccess.AddRepository("machine.locks"); + configurator.DataAccess.AddRepository( + "machine.train_segment_pairs", + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ), + ] + ); + return configurator; + } + + private static IServalConfigurator AddBuildJobService(this IServalConfigurator configurator) + { + configurator.Services.AddScoped, TranslationBuildJobService>(); + configurator.Services.AddScoped, BuildJobService>(); + + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + + configurator.Services.AddSingleton(); + configurator.Services.AddSingleton(x => x.GetRequiredService()); + configurator.Services.AddHostedService(p => p.GetRequiredService()); + + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + + var smtTransferEngineOptions = new SmtTransferEngineOptions(); + configurator.Configuration.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); + string? smtDriveLetter = Path.GetPathRoot(smtTransferEngineOptions.EnginesDir)?[..1]; + var statisticalEngineOptions = new StatisticalEngineOptions(); + configurator.Configuration.GetSection(StatisticalEngineOptions.Key).Bind(statisticalEngineOptions); + string? statisticsDriveLetter = Path.GetPathRoot(statisticalEngineOptions.EnginesDir)?[..1]; + if (smtDriveLetter is null || statisticsDriveLetter is null) + throw new InvalidOperationException("SMT Engine and Statistical directory is required"); + if (smtDriveLetter != statisticsDriveLetter) + throw new InvalidOperationException("SMT Engine and Statistical directory must be on the same drive"); + // add health check for disk storage capacity + configurator + .Services.AddHealthChecks() + .AddDiskStorageHealthCheck( + x => x.AddDrive(smtDriveLetter, 1_000), // 1GB + "SMT and Statistical Engine Storage Capacity", + HealthStatus.Degraded + ); + return configurator; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs deleted file mode 100644 index f5d82f55a..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServiceCollectionExtensions -{ - public static IMachineBuilder AddMachine(this IServiceCollection services, IConfiguration configuration) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - - services.AddMemoryCache(); - services.AddSingleton(); - services.AddHealthChecks().AddCheck("S3 Bucket"); - - services.AddSingleton(); - services.AddFileSystem(); - - services.AddScoped(); - services.AddStartupTask( - (sp, cancellationToken) => - sp.GetRequiredService().InitAsync(cancellationToken) - ); - services.AddParallelCorpusService(); - services.Configure(configuration.GetSection("Bugsnag")); - services.AddBugsnag(); - services.AddDiagnostics(); - - var builder = new MachineBuilder(services, configuration); - - builder.AddServiceOptions(configuration.GetSection(ServiceOptions.Key)); - builder.AddSharedFileOptions(configuration.GetSection(SharedFileOptions.Key)); - builder.AddSmtTransferEngineOptions(configuration.GetSection(SmtTransferEngineOptions.Key)); - builder.AddStatisticalEngineOptions(configuration.GetSection(StatisticalEngineOptions.Key)); - builder.AddClearMLOptions(configuration.GetSection(ClearMLOptions.Key)); - builder.AddDistributedReaderWriterLockOptions(configuration.GetSection(DistributedReaderWriterLockOptions.Key)); - builder.AddBuildJobOptions(configuration.GetSection(BuildJobOptions.Key)); - - builder.AddBuildJobService(); - builder.AddMongoDataAccess(); - builder.AddMongoHangfireJobClient(); - builder.AddClearMLService(); - builder.AddMongoOutbox(); - - return builder; - } - - private static IServiceCollection AddStartupTask( - this IServiceCollection services, - Func startupTask - ) - { - services.AddHostedService(sp => new StartupTask(sp, startupTask)); - return services; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs deleted file mode 100644 index 392e33740..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public class MachineBuilder(IServiceCollection services, IConfiguration configuration) : IMachineBuilder -{ - public IServiceCollection Services { get; } = services; - public IConfiguration Configuration { get; } = configuration; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/ServalPlatformConsumerBase.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/ServalPlatformConsumerBase.cs deleted file mode 100644 index 465f16a50..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/ServalPlatformConsumerBase.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace Serval.Machine.Shared.Consumers; - -public abstract class ServalPlatformConsumerBase( - string outboxId, - string method, - Func> platformFunc -) : OutboxConsumerBase(outboxId, method) -{ - private readonly Func< - T, - Metadata?, - DateTime?, - CancellationToken, - AsyncUnaryCall - > _platformFunc = platformFunc; - - protected override async Task HandleMessageAsync(T content, Stream? stream, CancellationToken cancellationToken) - { - await _platformFunc(content, null, null, cancellationToken); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCanceledConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCanceledConsumer.cs deleted file mode 100644 index 08f69c581..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCanceledConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationBuildCanceledConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.BuildCanceled, - client.BuildCanceledAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCompletedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCompletedConsumer.cs deleted file mode 100644 index 12761304f..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildCompletedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationBuildCompletedConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.BuildCompleted, - client.BuildCompletedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildFaultedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildFaultedConsumer.cs deleted file mode 100644 index 031a40ff3..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildFaultedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationBuildFaultedConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.BuildFaulted, - client.BuildFaultedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildRestartingConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildRestartingConsumer.cs deleted file mode 100644 index 76f0a2500..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildRestartingConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationBuildRestartingConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.BuildRestarting, - client.BuildRestartingAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildStartedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildStartedConsumer.cs deleted file mode 100644 index 23f0f86fc..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationBuildStartedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationBuildStartedConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.BuildStarted, - client.BuildStartedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationIncrementEngineCorpusSizeConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationIncrementEngineCorpusSizeConsumer.cs deleted file mode 100644 index 189ba2ea5..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationIncrementEngineCorpusSizeConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationIncrementEngineCorpusSizeConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.IncrementEngineCorpusSize, - client.IncrementEngineCorpusSizeAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationInsertPretranslationsConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationInsertPretranslationsConsumer.cs deleted file mode 100644 index 816448b97..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationInsertPretranslationsConsumer.cs +++ /dev/null @@ -1,161 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationInsertPretranslationsConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : OutboxConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.InsertPretranslations - ) -{ - private static readonly JsonSerializerOptions JsonSerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - Converters = { new PretranslationConverter() }, - }; - - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - - protected override async Task HandleMessageAsync( - string content, - Stream? stream, - CancellationToken cancellationToken - ) - { - ArgumentNullException.ThrowIfNull(stream); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable(stream, JsonSerializerOptions, cancellationToken) - .OfType(); - - using var call = _client.InsertPretranslations(cancellationToken: cancellationToken); - await foreach (Pretranslation pretranslation in pretranslations) - { - InsertPretranslationsRequest request = new InsertPretranslationsRequest - { - EngineId = content, - CorpusId = pretranslation.CorpusId, - TextId = pretranslation.TextId, - SourceRefs = { pretranslation.SourceRefs }, - TargetRefs = { pretranslation.TargetRefs }, - Translation = pretranslation.Translation, - SourceTokens = { pretranslation.SourceTokens }, - TranslationTokens = { pretranslation.TranslationTokens }, - Confidence = pretranslation.Confidence, - }; - if (pretranslation.Alignment is not null) - request.Alignment.Add(pretranslation.Alignment.Select(Map)); - - await call.RequestStream.WriteAsync(request, cancellationToken); - } - - await call.RequestStream.CompleteAsync(); - await call; - } - - private static Translation.V1.AlignedWordPair Map(SIL.Machine.Corpora.AlignedWordPair alignedWordPair) - { - return new Translation.V1.AlignedWordPair() - { - SourceIndex = alignedWordPair.SourceIndex, - TargetIndex = alignedWordPair.TargetIndex, - }; - } - - private class PretranslationConverter : JsonConverter - { - public override Pretranslation Read( - ref Utf8JsonReader reader, - Type typeToConvert, - JsonSerializerOptions options - ) - { - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException( - $"Expected StartObject token at the beginning of Pretranslation object but instead encountered {reader.TokenType}" - ); - } - string corpusId = "", - textId = "", - translation = ""; - IReadOnlyList sourceRefs = [], - targetRefs = [], - sourceTokens = [], - translationTokens = []; - IReadOnlyList alignedWordPairs = []; - double confidence = 0.0; - while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) - { - if (reader.TokenType == JsonTokenType.PropertyName) - { - string s = reader.GetString()!; - switch (s) - { - case "corpusId": - reader.Read(); - corpusId = reader.GetString()!; - break; - case "textId": - reader.Read(); - textId = reader.GetString()!; - break; - case "refs": - reader.Read(); - targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "sourceRefs": - reader.Read(); - sourceRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "targetRefs": - reader.Read(); - targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "translation": - reader.Read(); - translation = reader.GetString()!; - break; - case "sourceTokens": - reader.Read(); - sourceTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "translationTokens": - reader.Read(); - translationTokens = JsonSerializer - .Deserialize>(ref reader, options)! - .ToArray(); - break; - case "alignment": - reader.Read(); - alignedWordPairs = SIL.Machine.Corpora.AlignedWordPair.Parse(reader.GetString()).ToArray(); - break; - case "sequenceConfidence": - reader.Read(); - confidence = reader.GetDouble(); - break; - default: - throw new JsonException( - $"Unexpected property name {s} when deserializing Pretranslation object" - ); - } - } - } - return new Pretranslation() - { - CorpusId = corpusId, - TextId = textId, - SourceRefs = sourceRefs, - TargetRefs = targetRefs, - Translation = translation, - Alignment = alignedWordPairs, - SourceTokens = sourceTokens, - TranslationTokens = translationTokens, - Confidence = confidence, - }; - } - - public override void Write(Utf8JsonWriter writer, Pretranslation value, JsonSerializerOptions options) => - throw new NotSupportedException(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateBuildExecutionDataConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateBuildExecutionDataConsumer.cs deleted file mode 100644 index 7d221b3b6..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateBuildExecutionDataConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationUpdateBuildExecutionDataConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.UpdateBuildExecutionData, - client.UpdateBuildExecutionDataAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateQuoteConventionConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateQuoteConventionConsumer.cs deleted file mode 100644 index 478cb1bd5..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateQuoteConventionConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class TranslationUpdateTargetQuoteConventionConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalTranslationPlatformOutboxConstants.OutboxId, - ServalTranslationPlatformOutboxConstants.UpdateTargetQuoteConvention, - client.UpdateTargetQuoteConventionAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCanceledConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCanceledConsumer.cs deleted file mode 100644 index c632d204d..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCanceledConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentBuildCanceledConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.BuildCanceled, - client.BuildCanceledAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCompletedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCompletedConsumer.cs deleted file mode 100644 index c3c9db853..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildCompletedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentBuildCompletedConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.BuildCompleted, - client.BuildCompletedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildFaultedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildFaultedConsumer.cs deleted file mode 100644 index f366794f7..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildFaultedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentBuildFaultedConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.BuildFaulted, - client.BuildFaultedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildRestartingConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildRestartingConsumer.cs deleted file mode 100644 index 7ea0fd0ae..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildRestartingConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentBuildRestartingConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.BuildRestarting, - client.BuildRestartingAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildStartedConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildStartedConsumer.cs deleted file mode 100644 index 91fbe5bb2..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentBuildStartedConsumer.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentBuildStartedConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.BuildStarted, - client.BuildStartedAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentIncrementEngineCorpusSizeConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentIncrementEngineCorpusSizeConsumer.cs deleted file mode 100644 index 0fbbac309..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentIncrementEngineCorpusSizeConsumer.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentIncrementEngineCorpusSizeConsumer( - WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client -) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.IncrementTrainEngineCorpusSize, - client.IncrementEngineCorpusSizeAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentInsertWordAlignmentsConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentInsertWordAlignmentsConsumer.cs deleted file mode 100644 index e0b0ddc9f..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentInsertWordAlignmentsConsumer.cs +++ /dev/null @@ -1,151 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentInsertWordAlignmentsConsumer(WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client) - : OutboxConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.InsertWordAlignments - ) -{ - private static readonly JsonSerializerOptions JsonSerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - Converters = { new WordAlignmentConverter() }, - }; - - private readonly WordAlignmentPlatformApi.WordAlignmentPlatformApiClient _client = client; - - protected override async Task HandleMessageAsync( - string content, - Stream? stream, - CancellationToken cancellationToken - ) - { - ArgumentNullException.ThrowIfNull(stream); - - IAsyncEnumerable wordAlignments = JsonSerializer - .DeserializeAsyncEnumerable(stream, JsonSerializerOptions, cancellationToken) - .OfType(); - - using var call = _client.InsertWordAlignments(cancellationToken: cancellationToken); - await foreach (Models.WordAlignment wordAlignment in wordAlignments) - { - await call.RequestStream.WriteAsync( - new InsertWordAlignmentsRequest - { - EngineId = content, - CorpusId = wordAlignment.CorpusId, - TextId = wordAlignment.TextId, - SourceRefs = { wordAlignment.SourceRefs }, - TargetRefs = { wordAlignment.TargetRefs }, - SourceTokens = { wordAlignment.SourceTokens }, - TargetTokens = { wordAlignment.TargetTokens }, - Alignment = { Map(wordAlignment.Alignment) }, - }, - cancellationToken - ); - } - - await call.RequestStream.CompleteAsync(); - await call; - } - - private static IEnumerable Map( - IEnumerable alignedWordPairs - ) - { - foreach (SIL.Machine.Corpora.AlignedWordPair pair in alignedWordPairs) - { - yield return new WordAlignment.V1.AlignedWordPair - { - SourceIndex = pair.SourceIndex, - TargetIndex = pair.TargetIndex, - Score = pair.TranslationScore, - }; - } - } - - private class WordAlignmentConverter : JsonConverter - { - public override Models.WordAlignment Read( - ref Utf8JsonReader reader, - Type typeToConvert, - JsonSerializerOptions options - ) - { - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException( - $"Expected StartObject token at the beginning of WordAlignment object but instead encountered {reader.TokenType}" - ); - } - string corpusId = "", - textId = ""; - IReadOnlyList sourceRefs = [], - targetRefs = [], - sourceTokens = [], - targetTokens = []; - IReadOnlyList alignedWordPairs = []; - while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) - { - if (reader.TokenType == JsonTokenType.PropertyName) - { - string s = reader.GetString()!; - switch (s) - { - case "corpusId": - reader.Read(); - corpusId = reader.GetString()!; - break; - case "textId": - reader.Read(); - textId = reader.GetString()!; - break; - case "refs": - reader.Read(); - targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "sourceRefs": - reader.Read(); - sourceRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "targetRefs": - reader.Read(); - targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "sourceTokens": - reader.Read(); - sourceTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "targetTokens": - reader.Read(); - targetTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); - break; - case "alignment": - reader.Read(); - alignedWordPairs = SIL.Machine.Corpora.AlignedWordPair.Parse(reader.GetString()).ToArray(); - break; - default: - throw new JsonException( - $"Unexpected property name {s} when deserializing WordAlignment object" - ); - } - } - } - return new Models.WordAlignment() - { - CorpusId = corpusId, - TextId = textId, - SourceRefs = sourceRefs, - TargetRefs = targetRefs, - Alignment = alignedWordPairs, - SourceTokens = sourceTokens, - TargetTokens = targetTokens, - }; - } - - public override void Write(Utf8JsonWriter writer, Models.WordAlignment value, JsonSerializerOptions options) => - throw new NotSupportedException(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentUpdateBuildExecutionDataConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentUpdateBuildExecutionDataConsumer.cs deleted file mode 100644 index 7e2444423..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentUpdateBuildExecutionDataConsumer.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -public class WordAlignmentUpdateBuildExecutionDataConsumer( - WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client -) - : ServalPlatformConsumerBase( - ServalWordAlignmentPlatformOutboxConstants.OutboxId, - ServalWordAlignmentPlatformOutboxConstants.UpdateBuildExecutionData, - client.UpdateBuildExecutionDataAsync - ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index 573964300..9d0f74bad 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -11,6 +11,8 @@ $(NoWarn);CS1591;CS1573 + + @@ -32,10 +34,7 @@ - - - - + @@ -48,11 +47,11 @@ - - + + diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobQueues.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobQueues.cs new file mode 100644 index 000000000..dd66f6e81 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobQueues.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Services; + +public static class BuildJobQueues +{ + public const string Nmt = "nmt"; + public const string SmtTransfer = "smt_transfer"; + public const string Statistical = "statistical"; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs index 5a8c92adc..90323234d 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -84,10 +84,8 @@ await _clearMLService.GetTasksByIdAsync( { var tasksPerEngineType = tasks .Where(kvp => - engineToBuildServiceDict - .Where(te => te.Key.CurrentBuild?.JobId == kvp.Key) - .FirstOrDefault() - .Key?.Type == engineType + engineToBuildServiceDict.FirstOrDefault(te => te.Key.CurrentBuild?.JobId == kvp.Key).Key?.Type + == engineType ) .Select(kvp => kvp.Value) .UnionBy(await _clearMLService.GetTasksForQueueAsync(queueName, cancellationToken), t => t.Id) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/FailedPreconditionInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/FailedPreconditionInterceptor.cs deleted file mode 100644 index aa536654a..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/FailedPreconditionInterceptor.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class FailedPreconditionInterceptor : Interceptor -{ - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (EngineNotBuiltException e) - { - throw new RpcException(new Status(StatusCode.FailedPrecondition, e.Message, e)); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs index 05d5df502..0b40ee6e5 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs @@ -9,6 +9,7 @@ ILogger> logger ) : HangfireBuildJob(platformService, engines, dataAccessContext, buildJobService, logger) where TEngine : ITrainingEngine { + [AutomaticRetry(Attempts = 0)] public virtual Task RunAsync( string engineId, string buildId, @@ -35,6 +36,7 @@ ILogger> logger protected IBuildJobService BuildJobService { get; } = buildJobService; protected ILogger> Logger { get; } = logger; + [AutomaticRetry(Attempts = 0)] public virtual async Task RunAsync( string engineId, string buildId, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs index 1f9bec290..8108e1a98 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs @@ -1,8 +1,9 @@ -namespace Serval.Machine.Shared.Services; +using Serval.Translation.Contracts; + +namespace Serval.Machine.Shared.Services; public class NmtEngineService( - [FromKeyedServices(EngineGroup.Translation)] IPlatformService platformService, - IDataAccessContext dataAccessContext, + ITranslationPlatformService platformService, IRepository engines, IBuildJobService buildJobService, ILanguageTagService languageTagService, @@ -10,8 +11,7 @@ public class NmtEngineService( ISharedFileService sharedFileService ) : ITranslationEngineService { - private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly ITranslationPlatformService _platformService = platformService; private readonly IRepository _engines = engines; private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; @@ -24,15 +24,13 @@ public static string GetModelPath(string engineId, int buildRevision) return $"{ModelDirectory}{engineId}_{buildRevision}.tar.gz"; } - public EngineType Type => EngineType.Nmt; - private const int MinutesToExpire = 60; public async Task CreateAsync( string engineId, - string? engineName, string sourceLanguage, string targetLanguage, + string? engineName = null, bool? isModelPersisted = null, CancellationToken cancellationToken = default ) @@ -58,9 +56,8 @@ public async Task CreateAsync( public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) { await CancelBuildJobAsync(engineId, cancellationToken); - await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); + await _buildJobService.DeleteEngineAsync(engineId, cancellationToken); } public async Task UpdateAsync( @@ -88,30 +85,24 @@ await _engines.UpdateAsync( public async Task StartBuildAsync( string engineId, string buildId, - string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, + string? options = null, CancellationToken cancellationToken = default ) { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - bool building = !await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - EngineType.Nmt, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - ct - ); - // If there is a pending/running build, then no need to start a new one. - if (building) - await _platformService.BuildCanceledAsync(buildId, ct); - }, - cancellationToken: cancellationToken + bool building = !await _buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + EngineType.Nmt, + engineId, + buildId, + BuildStage.Preprocess, + corpora, + options, + cancellationToken ); + // If there is a pending/running build, then no need to start a new one. + if (building) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); } public Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) @@ -119,7 +110,7 @@ await _dataAccessContext.WithTransactionAsync( return CancelBuildJobAsync(engineId, cancellationToken); } - public async Task GetModelDownloadUrlAsync( + public async Task GetModelDownloadUrlAsync( string engineId, CancellationToken cancellationToken = default ) @@ -139,8 +130,8 @@ public async Task GetModelDownloadUrlAsync( bool fileExists = await _sharedFileService.ExistsAsync(filepath, cancellationToken); if (!fileExists) throw new FileNotFoundException($"The model for build revision , {engine.BuildRevision}, does not exist."); - var expiresAt = DateTime.UtcNow.AddMinutes(MinutesToExpire); - var modelInfo = new ModelDownloadUrl + DateTime expiresAt = DateTime.UtcNow.AddMinutes(MinutesToExpire); + var modelInfo = new ModelDownloadUrlContract { Url = await _sharedFileService.GetDownloadUrlAsync(filepath, expiresAt), ModelRevision = engine.BuildRevision, @@ -149,7 +140,7 @@ public async Task GetModelDownloadUrlAsync( return modelInfo; } - public Task> TranslateAsync( + public Task> TranslateAsync( string engineId, int n, string segment, @@ -159,7 +150,7 @@ public Task> TranslateAsync( throw new NotSupportedException(); } - public Task GetWordGraphAsync( + public Task GetWordGraphAsync( string engineId, string segment, CancellationToken cancellationToken = default @@ -179,12 +170,21 @@ public Task TrainSegmentPairAsync( throw new NotSupportedException(); } - public int GetQueueSize() + public Task GetQueueSizeAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(_clearMLQueueService.GetQueueSize(EngineType.Nmt)); + } + + public Task GetLanguageInfoAsync( + string language, + CancellationToken cancellationToken = default + ) { - return _clearMLQueueService.GetQueueSize(Type); + bool isNative = IsLanguageNativeToModel(language, out string internalCode); + return Task.FromResult(new LanguageInfoContract { IsNative = isNative, InternalCode = internalCode }); } - public bool IsLanguageNativeToModel(string language, out string internalCode) + private bool IsLanguageNativeToModel(string language, out string internalCode) { return _languageTagService.ConvertToFlores200Code(language, out internalCode) == Flores200Support.LanguageAndScript; @@ -192,16 +192,12 @@ public bool IsLanguageNativeToModel(string language, out string internalCode) private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken + (string? buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync( + engineId, + cancellationToken ); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); return buildId; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs index b1444d73a..c9a4a5b07 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs @@ -10,17 +10,15 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? { return stage switch { - BuildStage.Preprocess => CreateJob>( - engineId, - buildId, - "nmt", - data, - buildOptions - ), + BuildStage.Preprocess => CreateJob< + TranslationEngine, + NmtPreprocessBuildJob, + IReadOnlyList + >(engineId, buildId, BuildJobQueues.Nmt, data, buildOptions), BuildStage.Postprocess => CreateJob( engineId, buildId, - "nmt", + BuildJobQueues.Nmt, data, buildOptions ), diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index 2b5f06393..bdc176ff1 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -33,13 +33,13 @@ private bool ResolveLanguageCode(string languageCode, out string resolvedCode) protected override async Task UpdateTargetQuoteConventionAsync( string engineId, string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - string overallTargetQuoteConventionAnalysis = - ParallelCorpusService.AnalyzeTargetQuoteConvention(parallelCorpora)?.BestQuoteConvention?.Name - ?? string.Empty; + string overallTargetQuoteConventionAnalysis = ParallelCorpusService.AnalyzeTargetQuoteConvention( + parallelCorpora + ); await PlatformService.UpdateTargetQuoteConventionAsync( engineId, @@ -56,7 +56,7 @@ protected override async Task UpdateBuildExecutionData( int pretranslateCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { @@ -119,7 +119,7 @@ protected override IReadOnlyList GetWarnings( int inferenceCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora + IReadOnlyList parallelCorpora ) { List warnings = diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NotFoundInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/NotFoundInterceptor.cs deleted file mode 100644 index 063166b86..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/NotFoundInterceptor.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class NotFoundInterceptor : Interceptor -{ - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (EngineNotFoundException e) - { - throw new RpcException(new Status(StatusCode.NotFound, e.Message, e)); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index d3488e8d3..fce1c1e7d 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -10,7 +10,7 @@ public abstract class PreprocessBuildJob( IParallelCorpusService parallelCorpusService, IOptionsMonitor options ) - : HangfireBuildJob>( + : HangfireBuildJob>( platformService, engines, dataAccessContext, @@ -36,7 +36,7 @@ IOptionsMonitor options protected override async Task DoWorkAsync( string engineId, string buildId, - IReadOnlyList data, + IReadOnlyList data, string? buildOptions, CancellationToken cancellationToken ) @@ -94,20 +94,20 @@ protected abstract Task UpdateBuildExecutionData( int inferenceCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ); protected virtual Task UpdateTargetQuoteConventionAsync( string engineId, string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) => Task.CompletedTask; protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, string? buildOptions, CancellationToken cancellationToken ); @@ -115,7 +115,7 @@ CancellationToken cancellationToken protected override async Task CleanupAsync( string engineId, string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList data, JobCompletionStatus completionStatus ) { @@ -137,7 +137,7 @@ protected virtual IReadOnlyList GetWarnings( int inferenceCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora + IReadOnlyList parallelCorpora ) { List warnings = []; @@ -146,18 +146,18 @@ IReadOnlyList parallelCorpora ( string parallelCorpusId, string monolingualCorpusId, - IReadOnlyList errors + IReadOnlyList errors ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) ) { - foreach (UsfmVersificationError error in errors) + foreach (UsfmVersificationErrorContract error in errors) { warnings.Add( error.Type switch { - UsfmVersificationErrorType.InvalidChapterNumber => + Serval.Shared.Contracts.UsfmVersificationErrorType.InvalidChapterNumber => $"Invalid chapter number error in project {error.ProjectName} at “{error.ActualVerseRef}” (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})", - UsfmVersificationErrorType.InvalidVerseNumber => + Serval.Shared.Contracts.UsfmVersificationErrorType.InvalidVerseNumber => $"Invalid verse number error in project {error.ProjectName} at “{error.ActualVerseRef}” (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})", _ => $"USFM versification error in project {error.ProjectName}, expected verse “{error.ExpectedVerseRef}”, actual verse “{error.ActualVerseRef}”, mismatch type {error.Type} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})", @@ -170,7 +170,7 @@ IReadOnlyList errors ( string parallelCorpusId, string monolingualCorpusId, - MissingParentProjectError error + MissingParentProjectErrorContract error ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora) ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs deleted file mode 100644 index 57221e6dc..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Health.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalHealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase -{ - private readonly HealthCheckService _healthCheckService = healthCheckService; - - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs deleted file mode 100644 index 44b5d3d6a..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs +++ /dev/null @@ -1,333 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalTranslationEngineServiceV1(IEnumerable engineServices) - : TranslationEngineApi.TranslationEngineApiBase -{ - private static readonly Empty Empty = new(); - - private readonly Dictionary _engineServices = engineServices.ToDictionary( - es => es.Type - ); - - public override async Task Create(CreateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.CreateAsync( - request.EngineId, - request.HasEngineName ? request.EngineName : null, - request.SourceLanguage, - request.TargetLanguage, - request.HasIsModelPersisted ? request.IsModelPersisted : null, - context.CancellationToken - ); - return Empty; - } - - public override async Task Delete(DeleteRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.DeleteAsync(request.EngineId, context.CancellationToken); - return Empty; - } - - public override async Task Update(UpdateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.UpdateAsync( - request.EngineId, - request.HasSourceLanguage ? request.SourceLanguage : null, - request.HasTargetLanguage ? request.TargetLanguage : null, - context.CancellationToken - ); - return Empty; - } - - public override async Task Translate(TranslateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - IEnumerable results = await engineService.TranslateAsync( - request.EngineId, - request.N, - request.Segment, - context.CancellationToken - ); - - return new TranslateResponse { Results = { results.Select(Map) } }; - } - - public override async Task GetWordGraph( - GetWordGraphRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - SIL.Machine.Translation.WordGraph wordGraph = await engineService.GetWordGraphAsync( - request.EngineId, - request.Segment, - context.CancellationToken - ); - return new GetWordGraphResponse { WordGraph = Map(wordGraph) }; - } - - public override async Task TrainSegmentPair(TrainSegmentPairRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.TrainSegmentPairAsync( - request.EngineId, - request.SourceSegment, - request.TargetSegment, - request.SentenceStart, - context.CancellationToken - ); - return Empty; - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - SIL.ServiceToolkit.Models.ParallelCorpus[] corpora = request.Corpora.Select(Map).ToArray(); - await engineService.StartBuildAsync( - request.EngineId, - request.BuildId, - request.HasOptions ? request.Options : null, - corpora, - context.CancellationToken - ); - return Empty; - } - - public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - string? buildId = await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); - if (buildId is null) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "There is no build currently running.")); - return new CancelBuildResponse() { BuildId = buildId }; - } - - public override async Task GetModelDownloadUrl( - GetModelDownloadUrlRequest request, - ServerCallContext context - ) - { - try - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - ModelDownloadUrl modelDownloadUrl = await engineService.GetModelDownloadUrlAsync( - request.EngineId, - context.CancellationToken - ); - return new GetModelDownloadUrlResponse - { - Url = modelDownloadUrl.Url, - ModelRevision = modelDownloadUrl.ModelRevision, - ExpiresAt = modelDownloadUrl.ExpiresAt.ToTimestamp(), - }; - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.FailedPrecondition, e.Message)); - } - catch (FileNotFoundException e) - { - throw new RpcException(new Status(StatusCode.NotFound, e.Message)); - } - } - - public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); - } - - public override Task GetLanguageInfo( - GetLanguageInfoRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - bool isNative = engineService.IsLanguageNativeToModel(request.Language, out string internalCode); - return Task.FromResult(new GetLanguageInfoResponse { InternalCode = internalCode, IsNative = isNative }); - } - - private ITranslationEngineService GetEngineService(string engineTypeStr) - { - if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) - return service; - throw new RpcException( - new Status(StatusCode.InvalidArgument, $"The engine type {engineTypeStr} is not supported.") - ); - } - - private static EngineType GetEngineType(string engineTypeStr) - { - engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; - if (System.Enum.TryParse(engineTypeStr, out EngineType engineType)) - return engineType; - throw new RpcException( - new Status(StatusCode.InvalidArgument, $"The engine type {engineTypeStr} is not supported.") - ); - } - - private static Translation.V1.TranslationResult Map(SIL.Machine.Translation.TranslationResult source) - { - return new Translation.V1.TranslationResult - { - Translation = source.Translation, - SourceTokens = { source.SourceTokens }, - TargetTokens = { source.TargetTokens }, - Confidences = { source.Confidences }, - Sources = { source.Sources.Select(Map) }, - Alignment = { Map(source.Alignment) }, - Phrases = { source.Phrases.Select(Map) }, - }; - } - - private static Translation.V1.WordGraph Map(SIL.Machine.Translation.WordGraph source) - { - return new Translation.V1.WordGraph - { - SourceTokens = { source.SourceTokens }, - InitialStateScore = source.InitialStateScore, - FinalStates = { source.FinalStates }, - Arcs = { source.Arcs.Select(Map) }, - }; - } - - private static Translation.V1.WordGraphArc Map(SIL.Machine.Translation.WordGraphArc source) - { - return new Translation.V1.WordGraphArc - { - PrevState = source.PrevState, - NextState = source.NextState, - Score = source.Score, - TargetTokens = { source.TargetTokens }, - Alignment = { Map(source.Alignment) }, - Confidences = { source.Confidences }, - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - Sources = { source.Sources.Select(Map) }, - }; - } - - private static Translation.V1.TranslationSources Map(SIL.Machine.Translation.TranslationSources source) - { - return new Translation.V1.TranslationSources - { - Values = - { - System - .Enum.GetValues() - .Where(s => s != SIL.Machine.Translation.TranslationSources.None && source.HasFlag(s)) - .Select(s => - s switch - { - SIL.Machine.Translation.TranslationSources.Smt => TranslationSource.Primary, - SIL.Machine.Translation.TranslationSources.Nmt => TranslationSource.Primary, - SIL.Machine.Translation.TranslationSources.Transfer => TranslationSource.Secondary, - SIL.Machine.Translation.TranslationSources.Prefix => TranslationSource.Human, - _ => TranslationSource.Primary, - } - ), - }, - }; - } - - private static IEnumerable Map(WordAlignmentMatrix source) - { - return source - .ToAlignedWordPairs() - .Select(wp => new Translation.V1.AlignedWordPair - { - SourceIndex = wp.SourceIndex, - TargetIndex = wp.TargetIndex, - }); - } - - private static Translation.V1.Phrase Map(SIL.Machine.Translation.Phrase source) - { - return new Translation.V1.Phrase - { - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - TargetSegmentCut = source.TargetSegmentCut, - }; - } - - private static SIL.ServiceToolkit.Models.ParallelCorpus Map(Translation.V1.ParallelCorpus source) - { - return new SIL.ServiceToolkit.Models.ParallelCorpus - { - Id = source.Id, - SourceCorpora = source.SourceCorpora.Select(Map).ToList(), - TargetCorpora = source.TargetCorpora.Select(Map).ToList(), - }; - } - - private static SIL.ServiceToolkit.Models.MonolingualCorpus Map(Translation.V1.MonolingualCorpus source) - { - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); - FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll); - - var pretranslateChapters = source.PretranslateChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var pretranslateTextIds = source.PretranslateTextIds.ToHashSet(); - FilterChoice pretranslateFilter = GetFilterChoice( - pretranslateChapters, - pretranslateTextIds, - source.PretranslateAll - ); - - return new SIL.ServiceToolkit.Models.MonolingualCorpus - { - Id = source.Id, - Language = source.Language, - Files = source.Files.Select(Map).ToList(), - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? trainOnTextIds : null, - InferenceChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, - InferenceTextIds = pretranslateFilter == FilterChoice.TextIds ? pretranslateTextIds : null, - }; - } - - private static SIL.ServiceToolkit.Models.CorpusFile Map(Translation.V1.CorpusFile source) - { - return new SIL.ServiceToolkit.Models.CorpusFile - { - Location = source.Location, - Format = (SIL.ServiceToolkit.Models.FileFormat)source.Format, - TextId = source.TextId, - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None, - } - - private static FilterChoice GetFilterChoice( - IReadOnlyDictionary> chapters, - HashSet textIds, - bool noFilter - ) - { - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (noFilter || (chapters is null && textIds is null)) - return FilterChoice.None; - if (chapters is null || chapters.Count == 0) - return FilterChoice.TextIds; - return FilterChoice.Chapters; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs deleted file mode 100644 index d7b0034c7..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public static class ServalTranslationPlatformOutboxConstants -{ - public const string OutboxId = "ServalTranslationPlatform"; - - public const string BuildStarted = "BuildStarted"; - public const string BuildCompleted = "BuildCompleted"; - public const string BuildCanceled = "BuildCanceled"; - public const string BuildFaulted = "BuildFaulted"; - public const string BuildRestarting = "BuildRestarting"; - public const string InsertPretranslations = "InsertPretranslations"; - public const string IncrementEngineCorpusSize = "IncrementTrainEngineCorpusSize"; - public const string UpdateBuildExecutionData = "UpdateBuildExecutionData"; - public const string UpdateTargetQuoteConvention = "UpdateTargetQuoteConvention"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index e7bcd7f04..1cd41afa6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -1,84 +1,39 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; -using Phase = Serval.Translation.V1.Phase; +using Serval.Translation.Contracts; namespace Serval.Machine.Shared.Services; -public class ServalTranslationPlatformService( - TranslationPlatformApi.TranslationPlatformApiClient client, - IOutboxService outboxService -) : IPlatformService +public class ServalTranslationPlatformService(ITranslationPlatformService platformService) : IPlatformService { - EngineGroup IPlatformService.EngineGroup => EngineGroup.Translation; - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private readonly IOutboxService _outboxService = outboxService; + public EngineGroup EngineGroup => EngineGroup.Translation; - public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) + private static readonly JsonSerializerOptions JsonSerializerOptions = new() { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.BuildStarted, - groupId: buildId, - content: new BuildStartedRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new PretranslationConverter() }, + }; + + private readonly ITranslationPlatformService _platformService = platformService; + + public Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildStartedAsync(buildId, cancellationToken); - public async Task BuildCompletedAsync( + public Task BuildCompletedAsync( string buildId, int trainSize, double confidence, CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.BuildCompleted, - groupId: buildId, - content: new BuildCompletedRequest - { - BuildId = buildId, - CorpusSize = trainSize, - Confidence = confidence, - }, - cancellationToken: cancellationToken - ); - } + ) => _platformService.BuildCompletedAsync(buildId, trainSize, confidence, cancellationToken); - public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.BuildCanceled, - groupId: buildId, - content: new BuildCanceledRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + public Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildCanceledAsync(buildId, cancellationToken); - public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.BuildFaulted, - groupId: buildId, - content: new BuildFaultedRequest { BuildId = buildId, Message = message }, - cancellationToken: cancellationToken - ); - } + public Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) => + _platformService.BuildFaultedAsync(buildId, message, cancellationToken); - public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.BuildRestarting, - groupId: buildId, - content: new BuildRestartingRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + public Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildRestartingAsync(buildId, cancellationToken); - public async Task UpdateBuildStatusAsync( + public Task UpdateBuildStatusAsync( string buildId, ProgressStatus progressStatus, int? queueDepth = null, @@ -86,44 +41,32 @@ public async Task UpdateBuildStatusAsync( DateTime? started = null, DateTime? completed = null, CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; - if (progressStatus.PercentCompleted.HasValue) - request.Progress = progressStatus.PercentCompleted.Value; - if (progressStatus.Message is not null) - request.Message = progressStatus.Message; - if (queueDepth is not null) - request.QueueDepth = queueDepth.Value; - foreach (BuildPhase buildPhase in phases ?? []) - { - var phase = new Phase { Stage = (PhaseStage)buildPhase.Stage }; - if (buildPhase.Step is not null) - phase.Step = buildPhase.Step.Value; - if (buildPhase.StepCount is not null) - phase.StepCount = buildPhase.StepCount.Value; - if (buildPhase.Started is not null) - phase.Started = buildPhase.Started.Value.ToTimestamp(); - request.Phases.Add(phase); - } - - if (started is not null) - request.Started = started.Value.ToTimestamp(); - if (completed is not null) - request.Completed = completed.Value.ToTimestamp(); - - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); - } - - public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) - { - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync( - new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, - cancellationToken: cancellationToken + ) => + _platformService.UpdateBuildStatusAsync( + buildId, + new BuildProgressStatusContract + { + Step = progressStatus.Step, + PercentCompleted = progressStatus.PercentCompleted, + Message = progressStatus.Message, + }, + queueDepth, + phases + ?.Select(p => new PhaseContract + { + Stage = (PhaseStage)p.Stage, + Step = p.Step, + StepCount = p.StepCount, + Started = p.Started, + }) + .ToList(), + started, + completed, + cancellationToken ); - } + + public Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) => + _platformService.UpdateBuildStatusAsync(buildId, step, cancellationToken); public async Task InsertInferenceResultsAsync( string engineId, @@ -131,83 +74,177 @@ public async Task InsertInferenceResultsAsync( CancellationToken cancellationToken = default ) { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.InsertPretranslations, - groupId: engineId, - content: engineId, - stream: pretranslationsStream, - cancellationToken: cancellationToken + await _platformService.InsertPretranslationsAsync( + engineId, + ReadPretranslationsAsync(pretranslationsStream, cancellationToken), + cancellationToken ); } - public async Task IncrementTrainSizeAsync( + public Task IncrementTrainSizeAsync( string engineId, int count = 1, CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.IncrementEngineCorpusSize, - groupId: engineId, - content: new IncrementEngineCorpusSizeRequest { EngineId = engineId, Count = count }, - cancellationToken: cancellationToken - ); - } + ) => _platformService.IncrementEngineCorpusSizeAsync(engineId, count, cancellationToken); - public async Task UpdateBuildExecutionDataAsync( + public Task UpdateBuildExecutionDataAsync( string engineId, string buildId, BuildExecutionData executionData, CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildExecutionDataRequest - { - EngineId = engineId, - BuildId = buildId, - ExecutionData = new ExecutionData + ) => + _platformService.UpdateBuildExecutionDataAsync( + engineId, + buildId, + new ExecutionDataContract { - TrainCount = executionData.TrainCount ?? 0, - PretranslateCount = executionData.PretranslateCount ?? 0, + TrainCount = executionData.TrainCount, + PretranslateCount = executionData.PretranslateCount, + Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, - ResolvedSourceLanguage = executionData.ResolvedSourceLanguage ?? string.Empty, - ResolvedTargetLanguage = executionData.ResolvedTargetLanguage ?? string.Empty, + ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, + ResolvedTargetLanguage = executionData.ResolvedTargetLanguage, }, - }; - foreach (string warning in executionData.Warnings ?? []) - request.ExecutionData.Warnings.Add(warning); - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.UpdateBuildExecutionData, - groupId: engineId, - content: request, - cancellationToken: cancellationToken + cancellationToken ); - } - public async Task UpdateTargetQuoteConventionAsync( + public Task UpdateTargetQuoteConventionAsync( string engineId, string buildId, string quoteConvention, CancellationToken cancellationToken = default + ) => _platformService.UpdateTargetQuoteConventionAsync(engineId, buildId, quoteConvention, cancellationToken); + + private static async IAsyncEnumerable ReadPretranslationsAsync( + Stream stream, + [EnumeratorCancellation] CancellationToken cancellationToken ) { - var content = new UpdateTargetQuoteConventionRequest + await foreach ( + Pretranslation? pretranslation in JsonSerializer + .DeserializeAsyncEnumerable(stream, JsonSerializerOptions, cancellationToken) + .WithCancellation(cancellationToken) + ) { - EngineId = engineId, - BuildId = buildId, - TargetQuoteConvention = quoteConvention, - }; - - await _outboxService.EnqueueMessageAsync( - outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, - method: ServalTranslationPlatformOutboxConstants.UpdateTargetQuoteConvention, - groupId: engineId, - content, - cancellationToken: cancellationToken - ); + if (pretranslation is null) + continue; + + yield return new PretranslationContract + { + CorpusId = pretranslation.CorpusId, + TextId = pretranslation.TextId, + SourceRefs = pretranslation.SourceRefs, + TargetRefs = pretranslation.TargetRefs, + Translation = pretranslation.Translation, + SourceTokens = pretranslation.SourceTokens?.ToList(), + TranslationTokens = pretranslation.TranslationTokens?.ToList(), + Alignment = pretranslation + .Alignment?.Select(a => new AlignedWordPairContract + { + SourceIndex = a.SourceIndex, + TargetIndex = a.TargetIndex, + }) + .ToList(), + Confidence = pretranslation.Confidence, + }; + } + } + + private sealed class PretranslationConverter : JsonConverter + { + public override Pretranslation Read( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options + ) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException( + $"Expected StartObject token at the beginning of Pretranslation object but instead encountered {reader.TokenType}" + ); + } + string corpusId = "", + textId = "", + translation = ""; + double confidence = 0.0; + IReadOnlyList sourceRefs = [], + targetRefs = [], + sourceTokens = [], + translationTokens = []; + IReadOnlyList alignedWordPairs = []; + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string s = reader.GetString()!; + switch (s) + { + case "corpusId": + reader.Read(); + corpusId = reader.GetString()!; + break; + case "textId": + reader.Read(); + textId = reader.GetString()!; + break; + case "refs": + reader.Read(); + targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "sourceRefs": + reader.Read(); + sourceRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "targetRefs": + reader.Read(); + targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "translation": + reader.Read(); + translation = reader.GetString()!; + break; + case "sourceTokens": + reader.Read(); + sourceTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "translationTokens": + reader.Read(); + translationTokens = JsonSerializer + .Deserialize>(ref reader, options)! + .ToArray(); + break; + case "alignment": + reader.Read(); + alignedWordPairs = AlignedWordPair.Parse(reader.GetString()).ToArray(); + break; + case "sequenceConfidence": + reader.Read(); + confidence = reader.GetDouble(); + break; + default: + throw new JsonException( + $"Unexpected property name {s} when deserializing Pretranslation object" + ); + } + } + } + return new Pretranslation + { + CorpusId = corpusId, + TextId = textId, + SourceRefs = sourceRefs, + TargetRefs = targetRefs, + Translation = translation, + Alignment = alignedWordPairs, + SourceTokens = sourceTokens, + TranslationTokens = translationTokens, + Confidence = confidence, + }; + } + + public override void Write(Utf8JsonWriter writer, Pretranslation value, JsonSerializerOptions options) => + throw new NotSupportedException(); } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs deleted file mode 100644 index e3dec2b2d..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs +++ /dev/null @@ -1,172 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalWordAlignmentEngineServiceV1(IEnumerable engineServices) - : WordAlignmentEngineApi.WordAlignmentEngineApiBase -{ - private static readonly Empty Empty = new(); - - private readonly Dictionary _engineServices = engineServices.ToDictionary( - es => es.Type - ); - - public override async Task Create(CreateRequest request, ServerCallContext context) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - await engineService.CreateAsync( - request.EngineId, - request.HasEngineName ? request.EngineName : null, - request.SourceLanguage, - request.TargetLanguage, - cancellationToken: context.CancellationToken - ); - return Empty; - } - - public override async Task Delete(DeleteRequest request, ServerCallContext context) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - await engineService.DeleteAsync(request.EngineId, context.CancellationToken); - return Empty; - } - - public override async Task GetWordAlignment( - GetWordAlignmentRequest request, - ServerCallContext context - ) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - WordAlignmentResult result = await engineService.AlignAsync( - request.EngineId, - request.SourceSegment, - request.TargetSegment, - context.CancellationToken - ); - - return new GetWordAlignmentResponse { Result = result }; - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - SIL.ServiceToolkit.Models.ParallelCorpus[] corpora = request.Corpora.Select(Map).ToArray(); - await engineService.StartBuildAsync( - request.EngineId, - request.BuildId, - request.HasOptions ? request.Options : null, - corpora, - context.CancellationToken - ); - return Empty; - } - - public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - string? buildId = await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); - if (buildId is null) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "There is no build currently running.")); - return new CancelBuildResponse() { BuildId = buildId }; - } - - public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) - { - IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); - return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); - } - - private IWordAlignmentEngineService GetEngineService(string engineTypeStr) - { - if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out IWordAlignmentEngineService? service)) - return service; - throw new RpcException( - new Status(StatusCode.InvalidArgument, $"The engine type {engineTypeStr} is not supported.") - ); - } - - private static EngineType GetEngineType(string engineTypeStr) - { - engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; - if (System.Enum.TryParse(engineTypeStr, out EngineType engineType)) - return engineType; - throw new RpcException( - new Status(StatusCode.InvalidArgument, $"The engine type {engineTypeStr} is not supported.") - ); - } - - private static SIL.ServiceToolkit.Models.ParallelCorpus Map(WordAlignment.V1.ParallelCorpus source) - { - return new SIL.ServiceToolkit.Models.ParallelCorpus - { - Id = source.Id, - SourceCorpora = source.SourceCorpora.Select(Map).ToList(), - TargetCorpora = source.TargetCorpora.Select(Map).ToList(), - }; - } - - private static SIL.ServiceToolkit.Models.MonolingualCorpus Map(WordAlignment.V1.MonolingualCorpus source) - { - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); - FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll); - - var wordAlignOnChapters = source.WordAlignOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - var wordAlignOnTextIds = source.WordAlignOnTextIds.ToHashSet(); - FilterChoice wordAlignOnFilter = GetFilterChoice( - wordAlignOnChapters, - wordAlignOnTextIds, - source.WordAlignOnAll - ); - - return new SIL.ServiceToolkit.Models.MonolingualCorpus - { - Id = source.Id, - Language = source.Language, - Files = source.Files.Select(Map).ToList(), - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? trainOnTextIds : null, - InferenceChapters = wordAlignOnFilter == FilterChoice.Chapters ? wordAlignOnChapters : null, - InferenceTextIds = wordAlignOnFilter == FilterChoice.TextIds ? wordAlignOnTextIds : null, - }; - } - - private static SIL.ServiceToolkit.Models.CorpusFile Map(WordAlignment.V1.CorpusFile source) - { - return new SIL.ServiceToolkit.Models.CorpusFile - { - Location = source.Location, - Format = (SIL.ServiceToolkit.Models.FileFormat)source.Format, - TextId = source.TextId, - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None, - } - - private static FilterChoice GetFilterChoice( - IReadOnlyDictionary> chapters, - HashSet textIds, - bool noFilter - ) - { - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (noFilter || (chapters is null && textIds is null)) - return FilterChoice.None; - if (chapters is null || chapters.Count == 0) - return FilterChoice.TextIds; - return FilterChoice.Chapters; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformOutboxConstants.cs deleted file mode 100644 index 9490dceb3..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformOutboxConstants.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public static class ServalWordAlignmentPlatformOutboxConstants -{ - public const string OutboxId = "ServalWordAlignmentPlatform"; - - public const string BuildStarted = "BuildStarted"; - public const string BuildCompleted = "BuildCompleted"; - public const string BuildCanceled = "BuildCanceled"; - public const string BuildFaulted = "BuildFaulted"; - public const string BuildRestarting = "BuildRestarting"; - public const string IncrementTrainEngineCorpusSize = "IncrementTrainEngineCorpusSize"; - public const string InsertWordAlignments = "InsertWordAlignments"; - public const string UpdateBuildExecutionData = "UpdateBuildExecutionData"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index a7e4f48c0..72364dcfc 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -1,84 +1,39 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.WordAlignment.V1; -using Phase = Serval.WordAlignment.V1.Phase; +using Serval.WordAlignment.Contracts; namespace Serval.Machine.Shared.Services; -public class ServalWordAlignmentPlatformService( - WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client, - IOutboxService outboxService -) : IPlatformService +public class ServalWordAlignmentPlatformService(IWordAlignmentPlatformService platformService) : IPlatformService { - EngineGroup IPlatformService.EngineGroup => EngineGroup.WordAlignment; - private readonly WordAlignmentPlatformApi.WordAlignmentPlatformApiClient _client = client; - private readonly IOutboxService _outboxService = outboxService; + public EngineGroup EngineGroup => EngineGroup.WordAlignment; - public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) + private static readonly JsonSerializerOptions JsonSerializerOptions = new() { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.BuildStarted, - groupId: buildId, - content: new BuildStartedRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new WordAlignmentConverter() }, + }; + + private readonly IWordAlignmentPlatformService _platformService = platformService; + + public Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildStartedAsync(buildId, cancellationToken); - public async Task BuildCompletedAsync( + public Task BuildCompletedAsync( string buildId, int trainSize, double confidence, CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.BuildCompleted, - groupId: buildId, - content: new BuildCompletedRequest - { - BuildId = buildId, - CorpusSize = trainSize, - Confidence = confidence, - }, - cancellationToken: cancellationToken - ); - } + ) => _platformService.BuildCompletedAsync(buildId, trainSize, confidence, cancellationToken); - public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.BuildCanceled, - groupId: buildId, - content: new BuildCanceledRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + public Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildCanceledAsync(buildId, cancellationToken); - public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.BuildFaulted, - groupId: buildId, - content: new BuildFaultedRequest { BuildId = buildId, Message = message }, - cancellationToken: cancellationToken - ); - } + public Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) => + _platformService.BuildFaultedAsync(buildId, message, cancellationToken); - public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.BuildRestarting, - groupId: buildId, - content: new BuildRestartingRequest { BuildId = buildId }, - cancellationToken: cancellationToken - ); - } + public Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) => + _platformService.BuildRestartingAsync(buildId, cancellationToken); - public async Task UpdateBuildStatusAsync( + public Task UpdateBuildStatusAsync( string buildId, ProgressStatus progressStatus, int? queueDepth = null, @@ -86,44 +41,32 @@ public async Task UpdateBuildStatusAsync( DateTime? started = null, DateTime? completed = null, CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; - if (progressStatus.PercentCompleted.HasValue) - request.Progress = progressStatus.PercentCompleted.Value; - if (progressStatus.Message is not null) - request.Message = progressStatus.Message; - if (queueDepth is not null) - request.QueueDepth = queueDepth.Value; - foreach (BuildPhase buildPhase in phases ?? []) - { - var phase = new Phase { Stage = (PhaseStage)buildPhase.Stage }; - if (buildPhase.Step is not null) - phase.Step = buildPhase.Step.Value; - if (buildPhase.StepCount is not null) - phase.StepCount = buildPhase.StepCount.Value; - if (buildPhase.Started is not null) - phase.Started = buildPhase.Started.Value.ToTimestamp(); - request.Phases.Add(phase); - } - - if (started is not null) - request.Started = started.Value.ToTimestamp(); - if (completed is not null) - request.Completed = completed.Value.ToTimestamp(); - - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); - } - - public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) - { - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync( - new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, - cancellationToken: cancellationToken + ) => + _platformService.UpdateBuildStatusAsync( + buildId, + new BuildProgressStatusContract + { + Step = progressStatus.Step, + PercentCompleted = progressStatus.PercentCompleted, + Message = progressStatus.Message, + }, + queueDepth, + phases + ?.Select(p => new PhaseContract + { + Stage = (PhaseStage)p.Stage, + Step = p.Step, + StepCount = p.StepCount, + Started = p.Started, + }) + .ToList(), + started, + completed, + cancellationToken ); - } + + public Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) => + _platformService.UpdateBuildStatusAsync(buildId, step, cancellationToken); public async Task InsertInferenceResultsAsync( string engineId, @@ -131,60 +74,38 @@ public async Task InsertInferenceResultsAsync( CancellationToken cancellationToken = default ) { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.InsertWordAlignments, - groupId: engineId, - content: engineId, - stream: wordAlignmentsStream, - cancellationToken: cancellationToken + await _platformService.InsertWordAlignmentsAsync( + engineId, + ReadWordAlignmentsAsync(wordAlignmentsStream, cancellationToken), + cancellationToken ); } - public async Task IncrementTrainSizeAsync( + public Task IncrementTrainSizeAsync( string engineId, int count = 1, CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.IncrementTrainEngineCorpusSize, - groupId: engineId, - content: new IncrementEngineCorpusSizeRequest { EngineId = engineId, Count = count }, - cancellationToken: cancellationToken - ); - } + ) => _platformService.IncrementEngineCorpusSizeAsync(engineId, count, cancellationToken); - public async Task UpdateBuildExecutionDataAsync( + public Task UpdateBuildExecutionDataAsync( string engineId, string buildId, BuildExecutionData executionData, CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildExecutionDataRequest - { - EngineId = engineId, - BuildId = buildId, - ExecutionData = new ExecutionData + ) => + _platformService.UpdateBuildExecutionDataAsync( + engineId, + buildId, + new ExecutionDataContract { - TrainCount = executionData.TrainCount ?? 0, - WordAlignCount = executionData.WordAlignCount ?? 0, + TrainCount = executionData.TrainCount, + WordAlignCount = executionData.WordAlignCount, + Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, }, - }; - foreach (string warning in executionData.Warnings ?? []) - request.ExecutionData.Warnings.Add(warning); - await _outboxService.EnqueueMessageAsync( - outboxId: ServalWordAlignmentPlatformOutboxConstants.OutboxId, - method: ServalWordAlignmentPlatformOutboxConstants.UpdateBuildExecutionData, - groupId: engineId, - content: request, - cancellationToken: cancellationToken + cancellationToken ); - } public Task UpdateTargetQuoteConventionAsync( string engineId, @@ -196,4 +117,119 @@ public Task UpdateTargetQuoteConventionAsync( // Word alignment does not support quote convention analysis return Task.CompletedTask; } + + private static async IAsyncEnumerable ReadWordAlignmentsAsync( + Stream stream, + [EnumeratorCancellation] CancellationToken cancellationToken + ) + { + await foreach ( + Models.WordAlignment? record in JsonSerializer + .DeserializeAsyncEnumerable(stream, JsonSerializerOptions, cancellationToken) + .WithCancellation(cancellationToken) + ) + { + if (record is null) + continue; + + yield return new WordAlignmentContract + { + CorpusId = record.CorpusId, + TextId = record.TextId, + SourceRefs = record.SourceRefs, + TargetRefs = record.TargetRefs, + SourceTokens = record.SourceTokens, + TargetTokens = record.TargetTokens, + Alignment = record + .Alignment.Select(a => new AlignedWordPairContract + { + SourceIndex = a.SourceIndex, + TargetIndex = a.TargetIndex, + Score = a.TranslationScore, + }) + .ToList(), + }; + } + } + + private sealed class WordAlignmentConverter : JsonConverter + { + public override Models.WordAlignment Read( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options + ) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"Expected StartObject token but instead encountered {reader.TokenType}"); + } + string corpusId = "", + textId = ""; + IReadOnlyList sourceRefs = [], + targetRefs = [], + sourceTokens = [], + targetTokens = []; + IReadOnlyList alignedWordPairs = []; + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string s = reader.GetString()!; + switch (s) + { + case "corpusId": + reader.Read(); + corpusId = reader.GetString()!; + break; + case "textId": + reader.Read(); + textId = reader.GetString()!; + break; + case "refs": + reader.Read(); + targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "sourceRefs": + reader.Read(); + sourceRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "targetRefs": + reader.Read(); + targetRefs = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "sourceTokens": + reader.Read(); + sourceTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "targetTokens": + reader.Read(); + targetTokens = JsonSerializer.Deserialize>(ref reader, options)!.ToArray(); + break; + case "alignment": + reader.Read(); + alignedWordPairs = AlignedWordPair.Parse(reader.GetString()).ToArray(); + break; + default: + throw new JsonException( + $"Unexpected property name {s} when deserializing WordAlignmentRecord object" + ); + } + } + } + return new Models.WordAlignment + { + CorpusId = corpusId, + TextId = textId, + SourceRefs = sourceRefs, + TargetRefs = targetRefs, + SourceTokens = sourceTokens, + TargetTokens = targetTokens, + Alignment = alignedWordPairs, + }; + } + + public override void Write(Utf8JsonWriter writer, Models.WordAlignment value, JsonSerializerOptions options) => + throw new NotSupportedException(); + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs index c7e05dcb6..e142984f4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs @@ -1,9 +1,10 @@ -namespace Serval.Machine.Shared.Services; +using Serval.Translation.Contracts; + +namespace Serval.Machine.Shared.Services; public class SmtTransferEngineService( IDistributedReaderWriterLockFactory lockFactory, [FromKeyedServices(EngineGroup.Translation)] IPlatformService platformService, - IDataAccessContext dataAccessContext, IRepository engines, IRepository trainSegmentPairs, SmtTransferEngineStateService stateService, @@ -13,20 +14,17 @@ IClearMLQueueService clearMLQueueService { private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IRepository _engines = engines; private readonly IRepository _trainSegmentPairs = trainSegmentPairs; private readonly SmtTransferEngineStateService _stateService = stateService; private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - public EngineType Type => EngineType.SmtTransfer; - public async Task CreateAsync( string engineId, - string? engineName, string sourceLanguage, string targetLanguage, + string? engineName = null, bool? isModelPersisted = null, CancellationToken cancellationToken = default ) @@ -54,21 +52,15 @@ public async Task CreateAsync( public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) { - // there is no way to cancel this call SmtTransferEngineState state = _stateService.Get(engineId); state.IsMarkedForDeletion = true; - await CancelBuildJobAsync(engineId, CancellationToken.None); + await CancelBuildJobAsync(engineId, cancellationToken); + await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, cancellationToken); + await _buildJobService.DeleteEngineAsync(engineId, cancellationToken); - await _dataAccessContext.WithTransactionAsync( - async ct => - { - await _engines.DeleteAsync(e => e.EngineId == engineId, ct); - await _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, ct); - }, - cancellationToken: CancellationToken.None - ); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); + // after this point, we cannot cancel _stateService.Remove(engineId); state.DeleteData(); state.Dispose(); @@ -95,7 +87,7 @@ await _engines.UpdateAsync( ); } - public async Task> TranslateAsync( + public async Task> TranslateAsync( string engineId, int n, string segment, @@ -119,10 +111,10 @@ public async Task> TranslateAsync( ); state.Touch(); - return results; + return results.Select(Map).ToList(); } - public async Task GetWordGraphAsync( + public async Task GetWordGraphAsync( string engineId, string segment, CancellationToken cancellationToken = default @@ -145,7 +137,7 @@ public async Task GetWordGraphAsync( ); state.Touch(); - return result; + return Map(result); } public async Task TrainSegmentPairAsync( @@ -196,8 +188,8 @@ await _trainSegmentPairs.InsertAsync( public async Task StartBuildAsync( string engineId, string buildId, - string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, + string? options = null, CancellationToken cancellationToken = default ) { @@ -208,12 +200,12 @@ public async Task StartBuildAsync( buildId, BuildStage.Preprocess, corpora, - buildOptions, + options, cancellationToken ); // If there is a pending/running build, then no need to start a new one. if (building) - throw new InvalidOperationException("The engine is already building or in the process of canceling."); + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); SmtTransferEngineState state = _stateService.Get(engineId); state.Touch(); @@ -230,33 +222,31 @@ public async Task StartBuildAsync( return buildId; } - public int GetQueueSize() + public Task GetQueueSizeAsync(CancellationToken cancellationToken = default) { - return _clearMLQueueService.GetQueueSize(Type); + return Task.FromResult(_clearMLQueueService.GetQueueSize(EngineType.SmtTransfer)); } - public bool IsLanguageNativeToModel(string language, out string internalCode) + public Task GetLanguageInfoAsync( + string language, + CancellationToken cancellationToken = default + ) { - internalCode = language; - return true; + return Task.FromResult(new LanguageInfoContract { IsNative = true, InternalCode = language }); } private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async ct => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken + (string? buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync( + engineId, + cancellationToken ); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); return buildId; } - public Task GetModelDownloadUrlAsync( + public Task GetModelDownloadUrlAsync( string engineId, CancellationToken cancellationToken = default ) @@ -279,4 +269,86 @@ private async Task GetBuiltEngineAsync(string engineId, Cance throw new EngineNotBuiltException($"The engine {engineId} must be built first."); return engine; } + + private static TranslationResultContract Map(TranslationResult source) + { + return new TranslationResultContract + { + Translation = source.Translation, + SourceTokens = source.SourceTokens.ToArray(), + TargetTokens = source.TargetTokens.ToArray(), + Confidences = source.Confidences.ToArray(), + Sources = source.Sources.Select(Map).ToList(), + Alignment = MapAlignment(source.Alignment).ToList(), + Phrases = source.Phrases.Select(Map).ToList(), + }; + } + + private static WordGraphContract Map(WordGraph source) + { + return new WordGraphContract + { + SourceTokens = source.SourceTokens.ToArray(), + InitialStateScore = source.InitialStateScore, + FinalStates = source.FinalStates.ToHashSet(), + Arcs = source.Arcs.Select(Map).ToList(), + }; + } + + private static WordGraphArcContract Map(WordGraphArc source) + { + return new WordGraphArcContract + { + PrevState = source.PrevState, + NextState = source.NextState, + Score = source.Score, + TargetTokens = source.TargetTokens.ToArray(), + Confidences = source.Confidences.ToArray(), + SourceSegmentStart = source.SourceSegmentRange.Start, + SourceSegmentEnd = source.SourceSegmentRange.End, + Sources = source.Sources.Select(Map).ToList(), + Alignment = MapAlignment(source.Alignment).ToList(), + }; + } + + private static IReadOnlySet Map(TranslationSources source) + { + return Enum.GetValues() + .Where(s => s != TranslationSources.None && source.HasFlag(s)) + .Select(s => + s switch + { + TranslationSources.Smt => TranslationSource.Primary, + TranslationSources.Nmt => TranslationSource.Primary, + TranslationSources.Transfer => TranslationSource.Secondary, + TranslationSources.Prefix => TranslationSource.Human, + _ => TranslationSource.Primary, + } + ) + .ToHashSet(); + } + + private static IEnumerable MapAlignment(WordAlignmentMatrix source) + { + for (int i = 0; i < source.RowCount; i++) + { + for (int j = 0; j < source.ColumnCount; j++) + { + if (source[i, j]) + { + yield return new AlignedWordPairContract { SourceIndex = i, TargetIndex = j }; + } + } + } + } + + private static PhraseContract Map(Phrase source) + { + return new PhraseContract + { + SourceSegmentStart = source.SourceSegmentRange.Start, + SourceSegmentEnd = source.SourceSegmentRange.End, + TargetSegmentCut = source.TargetSegmentCut, + }; + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs index f38b5c4e9..eb2880c90 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs @@ -13,19 +13,19 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? BuildStage.Preprocess => CreateJob< TranslationEngine, SmtTransferPreprocessBuildJob, - IReadOnlyList - >(engineId, buildId, "smt_transfer", data, buildOptions), + IReadOnlyList + >(engineId, buildId, BuildJobQueues.SmtTransfer, data, buildOptions), BuildStage.Postprocess => CreateJob( engineId, buildId, - "smt_transfer", + BuildJobQueues.SmtTransfer, data, buildOptions ), BuildStage.Train => CreateJob( engineId, buildId, - "smt_transfer", + BuildJobQueues.SmtTransfer, buildOptions ), _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs index cdd618312..eef6282f9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs @@ -29,7 +29,7 @@ IOptionsMonitor options protected override async Task InitializeAsync( string engineId, string buildId, - IReadOnlyList corpora, + IReadOnlyList data, CancellationToken cancellationToken ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs index 36a1fabb0..7d04f5262 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs @@ -1,11 +1,10 @@ -using Serval.WordAlignment.V1; +using Serval.WordAlignment.Contracts; namespace Serval.Machine.Shared.Services; public class StatisticalEngineService( IDistributedReaderWriterLockFactory lockFactory, [FromKeyedServices(EngineGroup.WordAlignment)] IPlatformService platformService, - IDataAccessContext dataAccessContext, IRepository engines, StatisticalEngineStateService stateService, IBuildJobService buildJobService, @@ -14,19 +13,16 @@ IClearMLQueueService clearMLQueueService { private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IRepository _engines = engines; private readonly StatisticalEngineStateService _stateService = stateService; private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - public EngineType Type => EngineType.Statistical; - public async Task CreateAsync( string engineId, - string? engineName, string sourceLanguage, string targetLanguage, + string? engineName = null, CancellationToken cancellationToken = default ) { @@ -50,7 +46,7 @@ public async Task CreateAsync( state.InitNew(); } - public async Task AlignAsync( + public async Task AlignAsync( string engineId, string sourceSegment, string targetSegment, @@ -63,7 +59,7 @@ public async Task AlignAsync( throw new InvalidOperationException("Engine is marked for deletion."); IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - WordAlignmentResult result = await @lock.ReaderLockAsync( + WordAlignmentResultContract result = await @lock.ReaderLockAsync( async ct => { IWordAlignmentModel wordAlignmentModel = await state.GetEngineAsync(engine.BuildRevision, ct); @@ -72,13 +68,22 @@ public async Task AlignAsync( // there is no way to cancel this call IReadOnlyList sourceTokens = tokenizer.Tokenize(sourceSegment).ToList(); IReadOnlyList targetTokens = tokenizer.Tokenize(targetSegment).ToList(); - IReadOnlyCollection wordPairs = - wordAlignmentModel.GetBestAlignedWordPairs(sourceTokens, targetTokens); - return new WordAlignmentResult() + IReadOnlyCollection wordPairs = wordAlignmentModel.GetBestAlignedWordPairs( + sourceTokens, + targetTokens + ); + return new WordAlignmentResultContract { - SourceTokens = { sourceTokens }, - TargetTokens = { targetTokens }, - Alignment = { wordPairs.Select(Map) }, + SourceTokens = sourceTokens, + TargetTokens = targetTokens, + Alignment = wordPairs + .Select(wp => new AlignedWordPairContract + { + SourceIndex = wp.SourceIndex, + TargetIndex = wp.TargetIndex, + Score = wp.TranslationScore, + }) + .ToList(), }; }, cancellationToken: cancellationToken @@ -95,16 +100,10 @@ public async Task DeleteAsync(string engineId, CancellationToken cancellationTok state.IsMarkedForDeletion = true; await CancelBuildJobAsync(engineId, cancellationToken); + await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await _buildJobService.DeleteEngineAsync(engineId, cancellationToken); - await _dataAccessContext.WithTransactionAsync( - async ct => - { - await _engines.DeleteAsync(e => e.EngineId == engineId, ct); - }, - cancellationToken: CancellationToken.None - ); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); - + // after this point, we cannot cancel _stateService.Remove(engineId); state.DeleteData(); state.Dispose(); @@ -114,30 +113,24 @@ await _dataAccessContext.WithTransactionAsync( public async Task StartBuildAsync( string engineId, string buildId, - string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, + string? options = null, CancellationToken cancellationToken = default ) { - await _dataAccessContext.WithTransactionAsync( - async ct => - { - bool building = !await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - EngineType.Statistical, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - ct - ); - // If there is a pending/running build, then no need to start a new one. - if (building) - await _platformService.BuildCanceledAsync(buildId, ct); - }, + bool building = !await _buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + EngineType.Statistical, + engineId, + buildId, + BuildStage.Preprocess, + corpora, + options, cancellationToken ); + // If there is a pending/running build, then no need to start a new one. + if (building) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); StatisticalEngineState state = _stateService.Get(engineId); state.Touch(); @@ -154,23 +147,19 @@ await _dataAccessContext.WithTransactionAsync( return buildId; } - public int GetQueueSize() + public Task GetQueueSizeAsync(CancellationToken cancellationToken = default) { - return _clearMLQueueService.GetQueueSize(Type); + return Task.FromResult(_clearMLQueueService.GetQueueSize(EngineType.Statistical)); } private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async ct => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken + (string? buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync( + engineId, + cancellationToken ); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); return buildId; } @@ -189,14 +178,4 @@ private async Task GetBuiltEngineAsync(string engineId, Can throw new EngineNotBuiltException("The engine must be built first."); return engine; } - - private static WordAlignment.V1.AlignedWordPair Map(SIL.Machine.Corpora.AlignedWordPair alignedWordPair) - { - return new WordAlignment.V1.AlignedWordPair - { - SourceIndex = alignedWordPair.SourceIndex, - TargetIndex = alignedWordPair.TargetIndex, - Score = alignedWordPair.TranslationScore, - }; - } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalHangfireBuildJobFactory.cs index da473ccf4..712b79552 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalHangfireBuildJobFactory.cs @@ -13,19 +13,19 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? BuildStage.Preprocess => CreateJob< WordAlignmentEngine, WordAlignmentPreprocessBuildJob, - IReadOnlyList - >(engineId, buildId, "statistical", data, buildOptions), + IReadOnlyList + >(engineId, buildId, BuildJobQueues.Statistical, data, buildOptions), BuildStage.Postprocess => CreateJob( engineId, buildId, - "statistical", + BuildJobQueues.Statistical, data, buildOptions ), BuildStage.Train => CreateJob( engineId, buildId, - "statistical", + BuildJobQueues.Statistical, buildOptions ), _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TimeoutInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/TimeoutInterceptor.cs deleted file mode 100644 index 8f33674df..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/TimeoutInterceptor.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class TimeoutInterceptor(ILogger logger) : Interceptor -{ - private readonly ILogger _logger = logger; - - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (TimeoutException te) - { - _logger.LogError(te, "The method {Method} took too long to complete.", context.Method); - throw new RpcException(new Status(StatusCode.Unavailable, "The method took too long to complete.")); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 9715afb92..3e9e2ccaf 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -23,7 +23,7 @@ IOptionsMonitor options { protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, string? buildOptions, CancellationToken cancellationToken ) @@ -109,7 +109,7 @@ protected override async Task UpdateBuildExecutionData( int pretranslateCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs deleted file mode 100644 index c75812d62..000000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class UnimplementedInterceptor : Interceptor -{ - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (NotSupportedException) - { - throw new RpcException( - new Status(StatusCode.Unimplemented, "The call is not supported by the specified engine.") - ); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index 8459232aa..a782dbb6a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -23,7 +23,7 @@ IOptionsMonitor options { protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, string? buildOptions, CancellationToken cancellationToken ) @@ -109,7 +109,7 @@ protected override async Task UpdateBuildExecutionData( int wordAlignCount, string sourceLanguageTag, string targetLanguageTag, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { @@ -148,7 +148,7 @@ CancellationToken cancellationToken protected override Task UpdateTargetQuoteConventionAsync( string engineId, string buildId, - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs index 405e8a4dc..3de67a4f3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Usings.cs +++ b/src/Machine/src/Serval.Machine.Shared/Usings.cs @@ -2,6 +2,7 @@ global using System.Collections.Immutable; global using System.ComponentModel; global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; global using System.Formats.Tar; global using System.Globalization; global using System.IO.Compression; @@ -16,22 +17,15 @@ global using System.Text.Json; global using System.Text.Json.Nodes; global using System.Text.Json.Serialization; +global using System.Text.RegularExpressions; global using System.Text.Unicode; global using Amazon; global using Amazon.Runtime; global using Amazon.S3; global using Amazon.S3.Model; -global using Bugsnag.AspNet.Core; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; global using Hangfire; global using Hangfire.Common; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; global using Hangfire.States; -global using Microsoft.AspNetCore.Routing; global using Microsoft.Extensions.Caching.Memory; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; @@ -45,10 +39,10 @@ global using Polly; global using Polly.Extensions.Http; global using Serval.Machine.Shared.Configuration; -global using Serval.Machine.Shared.Consumers; global using Serval.Machine.Shared.Models; global using Serval.Machine.Shared.Services; global using Serval.Machine.Shared.Utils; +global using Serval.Shared.Contracts; global using SIL.DataAccess; global using SIL.Machine.Corpora; global using SIL.Machine.Morphology.HermitCrab; @@ -56,8 +50,5 @@ global using SIL.Machine.Translation; global using SIL.Machine.Translation.Thot; global using SIL.Machine.Utils; -global using SIL.ServiceToolkit.Models; -global using SIL.ServiceToolkit.Services; -global using SIL.ServiceToolkit.Utils; global using SIL.WritingSystems; global using YamlDotNet.RepresentationModel; diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs b/src/Machine/src/Serval.Machine.Shared/Utils/LanguageTagParser.cs similarity index 99% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs rename to src/Machine/src/Serval.Machine.Shared/Utils/LanguageTagParser.cs index 53e4ed6b4..af149f11e 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs +++ b/src/Machine/src/Serval.Machine.Shared/Utils/LanguageTagParser.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Utils; +namespace Serval.Machine.Shared.Utils; public partial class LanguageTagParser { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs b/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs similarity index 96% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs rename to src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs index b220e9a40..2e2f91ec1 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs +++ b/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Machine.Shared.Utils; public abstract class RecurrentTask( string serviceName, diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/TaskEx.cs b/src/Machine/src/Serval.Machine.Shared/Utils/TaskEx.cs new file mode 100644 index 000000000..723f5ceb7 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/TaskEx.cs @@ -0,0 +1,55 @@ +namespace Serval.Machine.Shared.Utils; + +public static class TaskEx +{ + public static async Task<(bool, T?)> Timeout( + Func> action, + TimeSpan timeout, + CancellationToken cancellationToken = default + ) + { + if (timeout == System.Threading.Timeout.InfiniteTimeSpan) + return (true, await action(cancellationToken)); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Task task = action(cts.Token); + Task completedTask = await Task.WhenAny(task as Task, Delay(timeout, cancellationToken)); + T? result = await completedTask; + if (completedTask == task) + return (true, result); + + cts.Cancel(); + return (false, result); + } + + public static async Task Timeout( + Func action, + TimeSpan timeout, + CancellationToken cancellationToken = default + ) + { + if (timeout == System.Threading.Timeout.InfiniteTimeSpan) + { + await action(cancellationToken); + return true; + } + else + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Task task = action(cts.Token); + Task completedTask = await Task.WhenAny(task, Task.Delay(timeout, cancellationToken)); + await completedTask; + if (completedTask == task) + return true; + + cts.Cancel(); + return false; + } + } + + private static async Task Delay(TimeSpan timeout, CancellationToken cancellationToken = default) + { + await Task.Delay(timeout, cancellationToken); + return default; + } +} diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj b/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj index c227f7185..420ba6cbb 100644 --- a/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj +++ b/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj @@ -32,6 +32,7 @@ + diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs b/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs index 31edab2ad..40d15dfa4 100644 --- a/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs +++ b/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs @@ -16,8 +16,7 @@ public void SetUp() public async Task InitializesRepositories() { // Setup - IMachineBuilder machineBuilder = new MachineBuilder(_env.Services, _env.Configuration); - machineBuilder.AddMongoDataAccess(); + _env.Services.AddServal(_env.Configuration, c => c.AddMachineDataAccess()); // SUT await _env.InitializeDatabaseAsync(); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/TranslationInsertPretranslationsConsumerTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/TranslationInsertPretranslationsConsumerTests.cs deleted file mode 100644 index 8b45f10dc..000000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/TranslationInsertPretranslationsConsumerTests.cs +++ /dev/null @@ -1,134 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Consumers; - -[TestFixture] -public class TranslationInsertPretranslationsConsumerTests -{ - [Test] - public async Task HandleMessageAsync_Refs() - { - TestEnvironment env = new(); - - await using (MemoryStream stream = new()) - { - var obj = new JsonObject(); - await JsonSerializer.SerializeAsync( - stream, - new JsonArray - { - new JsonObject - { - ["corpusId"] = "corpus1", - ["textId"] = "MAT", - ["refs"] = new JsonArray { "MAT 1:1" }, - ["translation"] = "translation", - ["sequenceConfidence"] = 0.5, - }, - } - ); - - stream.Seek(0, SeekOrigin.Begin); - await env.Consumer.HandleMessageAsync("engine1", stream); - } - - _ = env.Client.Received(1).InsertPretranslations(); - _ = env - .PretranslationWriter.Received(1) - .WriteAsync( - new InsertPretranslationsRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - SourceRefs = { }, - TargetRefs = { "MAT 1:1" }, - Translation = "translation", - Confidence = 0.5, - }, - Arg.Any() - ); - } - - [Test] - public async Task HandleMessageAsync_SourceAndTargetRefs() - { - TestEnvironment env = new(); - - await using (MemoryStream stream = new()) - { - var obj = new JsonObject(); - await JsonSerializer.SerializeAsync( - stream, - new JsonArray - { - new JsonObject - { - ["corpusId"] = "corpus1", - ["textId"] = "MAT", - ["sourceRefs"] = new JsonArray { "MAT 1:1" }, - ["targetRefs"] = new JsonArray { "MAT 1:1" }, - ["sourceTokens"] = new JsonArray { "translation" }, - ["translationTokens"] = new JsonArray { "translation" }, - ["translation"] = "translation", - ["alignment"] = "0-0", - }, - } - ); - - stream.Seek(0, SeekOrigin.Begin); - await env.Consumer.HandleMessageAsync("engine1", stream); - } - - _ = env.Client.Received(1).InsertPretranslations(); - _ = env - .PretranslationWriter.Received(1) - .WriteAsync( - new InsertPretranslationsRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - SourceRefs = { "MAT 1:1" }, - TargetRefs = { "MAT 1:1" }, - Translation = "translation", - SourceTokens = { "translation" }, - TranslationTokens = { "translation" }, - Alignment = - { - new Translation.V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - }, - Confidence = 0.0, - }, - Arg.Any() - ); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Client = Substitute.For(); - PretranslationWriter = Substitute.For>(); - Client - .InsertPretranslations(cancellationToken: Arg.Any()) - .Returns( - TestCalls.AsyncClientStreamingCall( - PretranslationWriter, - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => [], - () => { } - ) - ); - - Consumer = new TranslationInsertPretranslationsConsumer(Client); - } - - public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } - public TranslationInsertPretranslationsConsumer Consumer { get; } - public IClientStreamWriter PretranslationWriter { get; } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/WordAlignmentInsertWordAlignmentsConsumerTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/WordAlignmentInsertWordAlignmentsConsumerTests.cs deleted file mode 100644 index ea39eda77..000000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Consumers/WordAlignmentInsertWordAlignmentsConsumerTests.cs +++ /dev/null @@ -1,163 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Consumers; - -[TestFixture] -public class WordAlignmentInsertWordAlignmentsConsumerTests -{ - [Test] - public async Task HandleMessageAsync_Refs() - { - TestEnvironment env = new(); - - await using (MemoryStream stream = new()) - { - await JsonSerializer.SerializeAsync( - stream, - new JsonArray - { - new JsonObject - { - { "corpusId", "corpus1" }, - { "textId", "MAT" }, - { - "refs", - new JsonArray { "MAT 1:1" } - }, - { - "sourceTokens", - new JsonArray { "sourceToken1" } - }, - { - "targetTokens", - new JsonArray { "targetToken1" } - }, - { "alignment", "0-0:1.0:1.0" }, - }, - } - ); - stream.Seek(0, SeekOrigin.Begin); - await env.Handler.HandleMessageAsync("engine1", stream); - } - - _ = env.Client.Received(1).InsertWordAlignments(); - _ = env - .WordAlignmentsWriter.Received(1) - .WriteAsync( - new InsertWordAlignmentsRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - SourceRefs = { }, - TargetRefs = { "MAT 1:1" }, - SourceTokens = { "sourceToken1" }, - TargetTokens = { "targetToken1" }, - Alignment = - { - new WordAlignment.V1.AlignedWordPair() - { - SourceIndex = 0, - TargetIndex = 0, - Score = 1.0, - }, - }, - }, - Arg.Any() - ); - } - - [Test] - public async Task HandleMessageAsync_SourceAndTargetRefs() - { - TestEnvironment env = new(); - - await using (MemoryStream stream = new()) - { - await JsonSerializer.SerializeAsync( - stream, - new JsonArray - { - new JsonObject - { - { "corpusId", "corpus1" }, - { "textId", "MAT" }, - { - "sourceRefs", - new JsonArray { "MAT 1:1" } - }, - { - "targetRefs", - new JsonArray { "MAT 1:1" } - }, - { - "sourceTokens", - new JsonArray { "sourceToken1" } - }, - { - "targetTokens", - new JsonArray { "targetToken1" } - }, - { "alignment", "0-0:1.0:1.0" }, - }, - } - ); - stream.Seek(0, SeekOrigin.Begin); - await env.Handler.HandleMessageAsync("engine1", stream); - } - - _ = env.Client.Received(1).InsertWordAlignments(); - _ = env - .WordAlignmentsWriter.Received(1) - .WriteAsync( - new InsertWordAlignmentsRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - SourceRefs = { "MAT 1:1" }, - TargetRefs = { "MAT 1:1" }, - SourceTokens = { "sourceToken1" }, - TargetTokens = { "targetToken1" }, - Alignment = - { - new WordAlignment.V1.AlignedWordPair() - { - SourceIndex = 0, - TargetIndex = 0, - Score = 1.0, - }, - }, - }, - Arg.Any() - ); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Client = Substitute.For(); - WordAlignmentsWriter = Substitute.For>(); - Client - .InsertWordAlignments(cancellationToken: Arg.Any()) - .Returns( - TestCalls.AsyncClientStreamingCall( - WordAlignmentsWriter, - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => [], - () => { } - ) - ); - - Handler = new WordAlignmentInsertWordAlignmentsConsumer(Client); - } - - public WordAlignmentPlatformApi.WordAlignmentPlatformApiClient Client { get; } - public WordAlignmentInsertWordAlignmentsConsumer Handler { get; } - public IClientStreamWriter WordAlignmentsWriter { get; } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs index fb17ad159..e7b5395af 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs @@ -136,7 +136,7 @@ public async Task ReaderLockAsync_WriterLockAcquiredAndExpired() } [Test] - public async Task ReaderLockAsync_Cancelled() + public async Task ReaderLockAsync_Canceled() { TestEnvironment env = new(); IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); @@ -377,7 +377,7 @@ public async Task WriterLockAsync_WriterLockAcquiredAndExpired() } [Test] - public async Task WriterLockAsync_Cancelled() + public async Task WriterLockAsync_Canceled() { var env = new TestEnvironment(); IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs index 3b2bd5574..a88910861 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs @@ -1,4 +1,6 @@ -namespace Serval.Machine.Shared.Services; +using Serval.Translation.Contracts; + +namespace Serval.Machine.Shared.Services; [TestFixture] public class NmtEngineServiceTests @@ -10,7 +12,7 @@ public async Task StartBuildAsync() env.PersistModel(); TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty(), "{}"); await env.WaitForBuildToFinishAsync(); engine = env.Engines.Get("engine1"); Assert.Multiple(() => @@ -30,7 +32,7 @@ public async Task CancelBuildAsync_Building() TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty(), "{}"); await env.WaitForBuildToStartAsync(); engine = env.Engines.Get("engine1"); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -58,7 +60,7 @@ public async Task DeleteAsync_WhileBuilding() TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty(), "{}"); await env.WaitForBuildToStartAsync(); engine = env.Engines.Get("engine1"); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -80,11 +82,11 @@ public async Task UpdateAsync() } [Test] - public void GetLanguageInfo() + public async Task GetLanguageInfoAsync() { using var env = new TestEnvironment(); - env.Service.IsLanguageNativeToModel("en", out string internalCode); - Assert.That(internalCode, Is.EqualTo("eng_Latn")); + LanguageInfoContract info = await env.Service.GetLanguageInfoAsync("en"); + Assert.That(info.InternalCode, Is.EqualTo("eng_Latn")); } private class TestEnvironment : DisposableBase @@ -120,6 +122,7 @@ public TestEnvironment() _jobClient = new BackgroundJobClient(_memoryStorage); PlatformService = Substitute.For(); PlatformService.EngineGroup.Returns(EngineGroup.Translation); + TranslationPlatformService = Substitute.For(); _lockFactory = new DistributedReaderWriterLockFactory( new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), new OptionsWrapper(new DistributedReaderWriterLockOptions()), @@ -204,6 +207,7 @@ public TestEnvironment() public IClearMLQueueService ClearMLQueueService { get; } public MemoryRepository Engines { get; } public IPlatformService PlatformService { get; } + public ITranslationPlatformService TranslationPlatformService { get; } public IClearMLService ClearMLService { get; } public ISharedFileService SharedFileService { get; } public IBuildJobService BuildJobService { get; } @@ -230,7 +234,7 @@ private BackgroundJobServer CreateJobServer() var jobServerOptions = new BackgroundJobServerOptions { Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, + Queues = new[] { BuildJobQueues.Nmt }, CancellationCheckInterval = TimeSpan.FromMilliseconds(50), }; return new BackgroundJobServer(jobServerOptions, _memoryStorage); @@ -239,8 +243,7 @@ private BackgroundJobServer CreateJobServer() private NmtEngineService CreateService() { return new NmtEngineService( - PlatformService, - new MemoryDataAccessContext(), + TranslationPlatformService, Engines, BuildJobService, new LanguageTagService(), @@ -330,7 +333,7 @@ public override object ActivateJob(Type jobType) _env.BuildJobService, _env.SharedFileService, new LanguageTagService(), - new ParallelCorpusService(), + Substitute.For(), _env.BuildJobOptions ); } diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index c62ecaae4..50c802717 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -3,68 +3,11 @@ namespace Serval.Machine.Shared.Services; [TestFixture] public class PreprocessBuildJobTests { - [Test] - public async Task RunAsync_FilterOutEverything() - { - using TestEnvironment env = new(); - env.PersistModel(); - ParallelCorpus corpus1 = env.DefaultTextFileCorpus with { }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnAll() - { - using TestEnvironment env = new(); - env.PersistModel(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: []); - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnTextIds() - { - using TestEnvironment env = new(); - env.PersistModel(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: ["textId1"], inferenceTextIds: []); - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - [Test] public void RunAsync_NothingToInference() { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: []); + TestEnvironment env = new(); + ParallelCorpusContract corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: []); Assert.ThrowsAsync(async () => { @@ -72,225 +15,70 @@ public void RunAsync_NothingToInference() }); } - [Test] - public async Task RunAsync_TrainAndPretranslateAll() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: null); - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_PretranslateAll() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(trainOnTextIds: [], inferenceTextIds: null); - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); - } - - [Test] - public async Task RunAsync_InferenceTextIds() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(inferenceTextIds: ["textId1"], trainOnTextIds: null); - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_InferenceTextIdsOverlapWithTrainOnTextIds() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus( - inferenceTextIds: ["textId1"], - trainOnTextIds: ["textId1"] - ); - - await env.RunBuildJobAsync(corpus1); - Assert.Multiple(async () => - { - Assert.That((await env.GetTrainCountAsync()).Source1Count, Is.EqualTo(4)); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - }); - } - [Test] public async Task RunAsync_BuildWarnings() { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultParatextCorpus; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(8)); - - env.BuildJobOptions.CurrentValue.Returns(new BuildJobOptions() { MaxWarnings = 2 }); - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - // Two warnings after truncation + one warning mentioning that warnings were truncated - Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(3)); - } - - [Test] - public async Task RunAsync_EnableKeyTerms() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultParatextCorpus; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(14)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(3652)); - }); - } - - [Test] - public async Task RunAsync_EnableKeyTermsNoTrainingData() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultParatextCorpus; - corpus1.SourceCorpora[0].TrainOnTextIds = []; - corpus1.TargetCorpora[0].TrainOnTextIds = []; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_DisableKeyTerms() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultParatextCorpus; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(14)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_InferenceChapters() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.ParatextCorpus( - trainOnChapters: [], - inferenceChapters: new Dictionary> { { "1CH", [12] } } - ); - - await env.RunBuildJobAsync(corpus1); - - Assert.That( - await env.GetPretranslateCountAsync(), - Is.EqualTo(4), - JsonSerializer.Serialize(await env.GetPretranslationsAsync()) - ); - } - - [Test] - public async Task RunAsync_DoNotPretranslateRemark() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultParatextCorpus; - - await env.RunBuildJobAsync(corpus1); - - Assert.That( - await env.GetPretranslateCountAsync(), - Is.EqualTo(20), - JsonSerializer.Serialize(await env.GetPretranslationsAsync()) - ); - } - - [Test] - public async Task RunAsync_TrainOnChapters() - { - using TestEnvironment env = new(); - env.PersistModel(); - ParallelCorpus corpus1 = env.ParatextCorpus( - trainOnChapters: new Dictionary> { { "MAT", [1] } }, - inferenceChapters: [] - ); - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => + TestEnvironment env = new(); + ParallelCorpusContract corpus1 = new() { - Assert.That(src1Count, Is.EqualTo(5)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_MixedSource_Paratext() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultMixedSourceParatextCorpus; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(7)); - Assert.That(src2Count, Is.EqualTo(14)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(21)); - } - - [Test] - public async Task RunAsync_MixedSource_Text() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.DefaultMixedSourceTextFileCorpus; + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [TestEnvironment.ParatextFile("pt-source1")], + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [TestEnvironment.ParatextFile("pt-target1")], + }, + ], + }; + env.ParallelCorpusService.AnalyzeUsfmVersification(Arg.Any>()) + .Returns([ + ( + "corpusId1", + "src_1", + [ + new() + { + ActualVerseRef = "MAT 1:1", + ExpectedVerseRef = "MAT 1:1", + ProjectName = "pt-source1", + Type = Serval.Shared.Contracts.UsfmVersificationErrorType.MissingVerse, + }, + new() + { + ActualVerseRef = "MAT 1:2", + ExpectedVerseRef = "MAT 1:2", + ProjectName = "pt-source1", + Type = Serval.Shared.Contracts.UsfmVersificationErrorType.ExtraVerse, + }, + ] + ), + ]); - await env.RunBuildJobAsync(corpus1); + await env.RunBuildJobAsync(corpus1, engineId: "engine4"); + Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(2)); - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(1)); - Assert.That(src2Count, Is.EqualTo(4)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(3)); + env.BuildJobOptions.CurrentValue.Returns(new BuildJobOptions() { MaxWarnings = 1 }); + await env.RunBuildJobAsync(corpus1, engineId: "engine4"); + // Two warnings after truncation + one warning mentioning that warnings were truncated + Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(2)); } [Test] public void RunAsync_UnknownLanguageTagsNoData() { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); + TestEnvironment env = new(); + ParallelCorpusContract corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); Assert.ThrowsAsync(async () => { @@ -301,303 +89,23 @@ public void RunAsync_UnknownLanguageTagsNoData() [Test] public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); + TestEnvironment env = new(); + ParallelCorpusContract corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); await env.RunBuildJobAsync(corpus1, engineId: "engine3", engineType: EngineType.SmtTransfer); } - [Test] - public async Task RunAsync_RemoveFreestandingEllipses() - { - using TestEnvironment env = new(); - ParallelCorpus corpus1 = env.ParatextCorpus( - trainOnChapters: new Dictionary> { { "MAT", [2] } }, - inferenceChapters: new Dictionary> { { "MAT", [2] } } - ); - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - string sourceExtract = await env.GetSourceExtractAsync(); - Assert.That( - sourceExtract, - Is.EqualTo( - "Source one, chapter two, verse one.\nSource one, chapter two, verse two. \u201ca quotation\u201d\n\n" - ), - sourceExtract - ); - string targetExtract = await env.GetTargetExtractAsync(); - Assert.That( - targetExtract, - Is.EqualTo( - "Target one, chapter two, verse one.\n\nTarget one, chapter two, verse three. \"a quotation\"\n" - ), - targetExtract - ); - JsonArray? pretranslations = await env.GetPretranslationsAsync(); - Assert.That(pretranslations, Is.Not.Null); - Assert.That(pretranslations!.Count, Is.EqualTo(1)); - } - - [Test] - public void RunAsync_OnlyParseSelectedBooks_NoBadBooks() - { - using TestEnvironment env = new(); - env.PersistModel(); // MRK does not contain verse data, so there is no inferencing - ParallelCorpus corpus = env.ParatextCorpus(trainOnTextIds: ["LEV"], inferenceTextIds: ["MRK"]); - var parallelCorpusService = new ParallelCorpusService(); - env.ParallelCorpusService = Substitute.For(); - env.ParallelCorpusService.When(s => - s.PreprocessAsync( - Arg.Any>(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any?>() - ) - ) - .Do(async callInfo => - { - CorpusBundle corpusBundle = new(callInfo.ArgAt>(0)); - DummyCorpusBundle dummyCorpusBundle = new DummyCorpusBundle( - corpusBundle, - ["LEV", "MRK", "MAT"], - ["MAT"] - ); - - await parallelCorpusService.PreprocessAsync( - dummyCorpusBundle, - callInfo.ArgAt>(1), - callInfo.ArgAt>(2), - callInfo.ArgAt(3), - callInfo.ArgAt?>(4) - ); - }); - Assert.DoesNotThrowAsync(async () => - { - await env.RunBuildJobAsync(corpus); - }); - } - - [Test] - public async Task RunAsync_OnlyParseSelectedBooks_TrainOnBadBook() - { - using TestEnvironment env = new(); - ParallelCorpus corpus = env.ParatextCorpus(trainOnTextIds: ["MAT"], inferenceTextIds: ["MRK"]); - var parallelCorpusService = new ParallelCorpusService(); - env.ParallelCorpusService = Substitute.For(); - ArgumentException? ex = null; - env.ParallelCorpusService.When(s => - s.PreprocessAsync( - Arg.Any>(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any?>() - ) - ) - .Do(async callInfo => - { - CorpusBundle corpusBundle = new(callInfo.ArgAt>(0)); - DummyCorpusBundle dummyCorpusBundle = new DummyCorpusBundle( - corpusBundle, - ["LEV", "MRK", "MAT"], - ["MAT"] - ); - ex = Assert.ThrowsAsync(async () => - { - await parallelCorpusService.PreprocessAsync( - dummyCorpusBundle, - callInfo.ArgAt>(1), - callInfo.ArgAt>(2), - callInfo.ArgAt(3), - callInfo.ArgAt?>(4) - ); - }); - }); - Assert.ThrowsAsync(async () => - { - await env.RunBuildJobAsync(corpus); - }); - - Assert.That(ex, Is.Not.Null); - } - - [Test] - public void RunAsync_OnlyParseSelectedBooks_PretranslateOnBadBook() + private class TestEnvironment { - using TestEnvironment env = new(); - ParallelCorpus corpus = env.ParatextCorpus(trainOnTextIds: ["LEV"], inferenceTextIds: ["MAT"]); - var parallelCorpusService = new ParallelCorpusService(); - env.ParallelCorpusService = Substitute.For(); - ArgumentException? ex = null; - env.ParallelCorpusService.When(s => - s.PreprocessAsync( - Arg.Any>(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any?>() - ) - ) - .Do(async callInfo => - { - CorpusBundle corpusBundle = new(callInfo.ArgAt>(0)); - DummyCorpusBundle dummyCorpusBundle = new DummyCorpusBundle( - corpusBundle, - ["LEV", "MRK", "MAT"], - ["MAT"] - ); - ex = Assert.ThrowsAsync(async () => - { - await parallelCorpusService.PreprocessAsync( - dummyCorpusBundle, - callInfo.ArgAt>(1), - callInfo.ArgAt>(2), - callInfo.ArgAt(3), - callInfo.ArgAt?>(4) - ); - }); - }); - Assert.ThrowsAsync(async () => - { - await env.RunBuildJobAsync(corpus); - }); - - Assert.That(ex, Is.Not.Null); - } - - [Test] - public async Task ParallelCorpusAsync() - { - using TestEnvironment env = new(); - List corpora = - [ - new ParallelCorpus() - { - Id = "1", - SourceCorpora = - [ - new() - { - Id = "_1", - Language = "en", - Files = [env.ParatextFile("pt-source1")], - TrainOnChapters = new() { { "MAT", [1] }, { "LEV", [] } }, - InferenceChapters = new() { { "1CH", [] } }, - }, - new() - { - Id = "_1", - Language = "en", - Files = [env.ParatextFile("pt-source2")], - TrainOnChapters = new() { { "MAT", [1] }, { "MRK", [] } }, - InferenceChapters = new() { { "1CH", [] } }, - }, - ], - TargetCorpora = - [ - new() - { - Id = "_1", - Language = "en", - Files = [env.ParatextFile("pt-target1")], - TrainOnChapters = new() { { "MAT", [1] }, { "MRK", [] } }, - }, - new() - { - Id = "_2", - Language = "en", - Files = [env.ParatextFile("pt-target2")], - TrainOnChapters = new() - { - { "MAT", [1] }, - { "MRK", [] }, - { "LEV", [] }, - }, - }, - ], - }, - ]; - await env.RunBuildJobAsync(corpora, useKeyTerms: false); - JsonArray? pretranslations = await env.GetPretranslationsAsync(); - Assert.Multiple(async () => - { - string src = await env.GetSourceExtractAsync(); - Assert.That( - src, - Is.EqualTo( - @"Source one, chapter fourteen, verse fifty-five. Segment b. -Source one, chapter fourteen, verse fifty-six. -Source one, chapter one, verse one. -Source one, chapter one, verse two and three. -Source one, chapter one, verse four. -Source one, chapter one, verse five. Source two, chapter one, verse six. -Source two, chapter one, verse seven. Source two, chapter one, verse eight. -Source two, chapter one, verse nine. Source one, chapter one, verse ten. -Source two, chapter one, verse one. -" - ) - .IgnoreLineEndings(), - src - ); - string trg = await env.GetTargetExtractAsync(); - Assert.That( - trg, - Is.EqualTo( - @"Target two, chapter fourteen, verse fifty-five. -Target two, chapter fourteen, verse fifty-six. -Target one, chapter one, verse one. -Target one, chapter one, verse two. Target one, chapter one, verse three. - -Target one, chapter one, verse five and six. -Target one, chapter one, verse seven and eight. -Target one, chapter one, verse nine and ten. - -" - ) - .IgnoreLineEndings(), - trg - ); - Assert.That(pretranslations, Is.Not.Null); - Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToString()); - Assert.That( - pretranslations[2]!["translation"]!.ToString(), - Is.EqualTo("Source one, chapter twelve, verse one.") - ); - Assert.That( - env.ExecutionData.Warnings, - Has.Count.EqualTo(16), - JsonSerializer.Serialize(env.ExecutionData.Warnings) - ); - }); - } - - private class TestEnvironment : DisposableBase - { - private static readonly string TestDataPath = Path.Combine( - AppContext.BaseDirectory, - "..", - "..", - "..", - "Services", - "data" - ); - - private readonly TempDirectory _tempDir; - public ISharedFileService SharedFileService { get; } public IPlatformService PlatformService { get; } public MemoryRepository Engines { get; } public MemoryRepository TrainSegmentPairs { get; } public IDistributedReaderWriterLockFactory LockFactory { get; } public IBuildJobService BuildJobService { get; } - public IParallelCorpusService ParallelCorpusService { get; set; } public IClearMLService ClearMLService { get; } public IOptionsMonitor BuildJobOptions { get; } - - public ParallelCorpus DefaultTextFileCorpus { get; } - public ParallelCorpus DefaultMixedSourceTextFileCorpus { get; } - public ParallelCorpus DefaultParatextCorpus { get; } - public ParallelCorpus DefaultMixedSourceParatextCorpus { get; } + public IParallelCorpusService ParallelCorpusService { get; } public BuildExecutionData ExecutionData { get; private set; } = new BuildExecutionData(); @@ -606,128 +114,6 @@ public TestEnvironment() if (!Sldr.IsInitialized) Sldr.Initialize(offlineTestMode: true); - _tempDir = new TempDirectory("PreprocessBuildJobTests"); - - ZipParatextProject("pt-source1"); - ZipParatextProject("pt-source2"); - ZipParatextProject("pt-target1"); - ZipParatextProject("pt-target2"); - - DefaultTextFileCorpus = new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [TextFile("source1")], - TrainOnTextIds = [], - InferenceTextIds = [], - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [TextFile("target1")], - TrainOnTextIds = [], - }, - ], - }; - - DefaultMixedSourceTextFileCorpus = new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [TextFile("source1"), TextFile("source2")], - TrainOnTextIds = null, - TrainOnChapters = null, - InferenceTextIds = null, - InferenceChapters = null, - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [TextFile("target1")], - TrainOnChapters = null, - TrainOnTextIds = null, - }, - ], - }; - - DefaultParatextCorpus = new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [ParatextFile("pt-source1")], - TrainOnTextIds = null, - InferenceTextIds = null, - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [ParatextFile("pt-target1")], - TrainOnTextIds = null, - }, - ], - }; - - DefaultMixedSourceParatextCorpus = new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [ParatextFile("pt-source1")], - TrainOnTextIds = null, - InferenceTextIds = null, - }, - new() - { - Id = "src_1", - Language = "es", - Files = [ParatextFile("pt-source2")], - TrainOnTextIds = null, - InferenceTextIds = null, - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [ParatextFile("pt-target1")], - TrainOnTextIds = null, - }, - ], - }; - Engines = new MemoryRepository(); Engines.Add( new TranslationEngine @@ -759,7 +145,7 @@ public TestEnvironment() SourceLanguage = "xxx", TargetLanguage = "zzz", BuildRevision = 1, - IsModelPersisted = false, + IsModelPersisted = true, CurrentBuild = new() { BuildId = "build1", @@ -792,6 +178,27 @@ public TestEnvironment() }, } ); + Engines.Add( + new TranslationEngine + { + Id = "engine4", + EngineId = "engine4", + Type = EngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = true, + CurrentBuild = new() + { + BuildId = "build1", + JobId = "job1", + JobState = BuildJobState.Pending, + BuildJobRunner = BuildJobRunnerType.Hangfire, + Stage = BuildStage.Preprocess, + ExecutionData = new BuildExecutionData(), + }, + } + ); TrainSegmentPairs = new MemoryRepository(); PlatformService = Substitute.For(); PlatformService.EngineGroup.Returns(EngineGroup.Translation); @@ -871,7 +278,7 @@ public TestEnvironment() ], Engines ); - ParallelCorpusService = new ParallelCorpusService(); + ParallelCorpusService = Substitute.For(); } public PreprocessBuildJob GetBuildJob(EngineType engineType) @@ -912,7 +319,10 @@ public PreprocessBuildJob GetBuildJob(EngineType engineType) } } - public static ParallelCorpus TextFileCorpus(HashSet? trainOnTextIds, HashSet? inferenceTextIds) + public static ParallelCorpusContract TextFileCorpus( + HashSet? trainOnTextIds, + HashSet? inferenceTextIds + ) { return new() { @@ -941,7 +351,7 @@ public static ParallelCorpus TextFileCorpus(HashSet? trainOnTextIds, Has }; } - public static ParallelCorpus TextFileCorpus(string sourceLanguage, string targetLanguage) + public static ParallelCorpusContract TextFileCorpus(string sourceLanguage, string targetLanguage) { return new() { @@ -970,69 +380,8 @@ public static ParallelCorpus TextFileCorpus(string sourceLanguage, string target }; } - public ParallelCorpus ParatextCorpus( - Dictionary>? trainOnChapters, - Dictionary>? inferenceChapters - ) - { - return new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [ParatextFile("pt-source1")], - TrainOnChapters = trainOnChapters, - InferenceChapters = inferenceChapters, - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [ParatextFile("pt-target1")], - TrainOnChapters = trainOnChapters, - }, - ], - }; - } - - public ParallelCorpus ParatextCorpus(HashSet? trainOnTextIds, HashSet? inferenceTextIds) - { - return new() - { - Id = "corpusId1", - SourceCorpora = - [ - new() - { - Id = "src_1", - Language = "es", - Files = [ParatextFile("pt-source1")], - TrainOnTextIds = trainOnTextIds, - InferenceTextIds = inferenceTextIds, - }, - ], - TargetCorpora = - [ - new() - { - Id = "trg_1", - Language = "en", - Files = [ParatextFile("pt-target1")], - TrainOnTextIds = trainOnTextIds, - }, - ], - }; - } - public Task RunBuildJobAsync( - ParallelCorpus corpus, + ParallelCorpusContract corpus, bool useKeyTerms = true, string engineId = "engine1", EngineType engineType = EngineType.Nmt @@ -1042,7 +391,7 @@ public Task RunBuildJobAsync( } public Task RunBuildJobAsync( - IEnumerable corpora, + IEnumerable corpora, bool useKeyTerms = true, string engineId = "engine1", EngineType engineType = EngineType.Nmt @@ -1058,172 +407,24 @@ public Task RunBuildJobAsync( ); } - public async Task GetSourceExtractAsync() - { - using StreamReader srcReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.src.txt")); - return await srcReader.ReadToEndAsync(); - } - - public async Task GetTargetExtractAsync() - { - using StreamReader trgReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.trg.txt")); - return await trgReader.ReadToEndAsync(); - } - - public async Task<(int Source1Count, int Source2Count, int TargetCount, int TermCount)> GetTrainCountAsync() - { - using StreamReader srcReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.src.txt")); - using StreamReader trgReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.trg.txt")); - using StreamReader srcTermReader = new( - await SharedFileService.OpenReadAsync("builds/build1/train.key-terms.src.txt") - ); - using StreamReader trgTermReader = new( - await SharedFileService.OpenReadAsync("builds/build1/train.key-terms.trg.txt") - ); - int src1Count = 0; - int src2Count = 0; - int trgCount = 0; - int termCount = 0; - string? srcLine; - string? trgLine; - while ( - (srcLine = await srcReader.ReadLineAsync()) is not null - && (trgLine = await trgReader.ReadLineAsync()) is not null - ) - { - srcLine = srcLine.Trim(); - trgLine = trgLine.Trim(); - if (srcLine.StartsWith("Source one")) - src1Count++; - else if (srcLine.StartsWith("Source two")) - src2Count++; - else if (srcLine.Length == 0) - trgCount++; - else - throw new ArgumentException("Unexpected line in test output"); - } - - while ( - (srcLine = await srcTermReader.ReadLineAsync()) is not null - && (trgLine = await trgTermReader.ReadLineAsync()) is not null - ) - { - termCount++; - } - - return (src1Count, src2Count, trgCount, termCount); - } - - public async Task GetPretranslationsAsync() - { - using StreamReader reader = new( - await SharedFileService.OpenReadAsync("builds/build1/pretranslate.src.json") - ); - return JsonSerializer.Deserialize(await reader.ReadToEndAsync()); - } - - public async Task GetPretranslateCountAsync() - { - var pretranslations = await GetPretranslationsAsync(); - return pretranslations?.Count ?? 0; - } - - public void PersistModel(string engineId = "engine1") - { - Engines.Replace(Engines.Get(engineId) with { IsModelPersisted = true }); - } - - private void ZipParatextProject(string name) - { - ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), Path.Combine(_tempDir.Path, $"{name}.zip")); - } - - public CorpusFile ParatextFile(string name) + public static CorpusFileContract ParatextFile(string name) { return new() { TextId = name, Format = FileFormat.Paratext, - Location = Path.Combine(_tempDir.Path, $"{name}.zip"), + Location = $"{name}.zip", }; } - private static CorpusFile TextFile(string name) + private static CorpusFileContract TextFile(string name) { return new() { TextId = "textId1", Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, $"{name}.txt"), + Location = $"{name}.txt", }; } - - protected override void DisposeManagedResources() - { - _tempDir.Dispose(); - } - } - - private class DummyCorpus(IEnumerable books, IEnumerable failsOn) : ITextCorpus - { - private IEnumerable FailsOn { get; } = failsOn; - - public IEnumerable Texts => - books.Select(b => new MemoryText( - b, - [new TextRow(b, new ScriptureRef(new VerseRef("MAT", "1", "1", ScrVers.English)))] - )); - - public bool IsTokenized => false; - - public ScrVers Versification => ScrVers.English; - - public int Count(bool includeEmpty = true, IEnumerable? textIds = null) - { - throw new NotImplementedException(); - } - - public int Count(bool includeEmpty = true) - { - throw new NotImplementedException(); - } - - public IEnumerator GetEnumerator() - { - throw new NotImplementedException(); - } - - public IEnumerable GetRows(IEnumerable textIds) - { - if (textIds.Intersect(FailsOn).Any()) - { - throw new ArgumentException( - $"Text ids provided ({string.Join(',', textIds)}) include text ids specified to fail on ({string.Join(',', FailsOn)})." - ); - } - return Texts.Where(t => textIds.Contains(t.Id)).SelectMany(t => t.GetRows()); - } - - public IEnumerable GetRows() - { - throw new NotImplementedException(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return Texts.GetEnumerator(); - } - } - - private class DummyCorpusBundle(CorpusBundle corpusBundle, IEnumerable books, IEnumerable failsOn) - : CorpusBundle(corpusBundle.ParallelCorpora) - { - private IEnumerable FailsOn { get; } = failsOn; - private IEnumerable Books { get; } = books; - - protected override IReadOnlyList CreateTextCorpora(IReadOnlyList files) - { - return [new DummyCorpus(Books, FailsOn)]; - } } } diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs index 03eb5482e..4fd088aee 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs @@ -1,4 +1,6 @@ -namespace Serval.Machine.Shared.Services; +using Serval.Translation.Contracts; + +namespace Serval.Machine.Shared.Services; [TestFixture] public class SmtTransferEngineServiceTests @@ -12,7 +14,7 @@ public class SmtTransferEngineServiceTests public async Task CreateAsync() { using var env = new TestEnvironment(); - await env.Service.CreateAsync(EngineId2, "Engine 2", "es", "en"); + await env.Service.CreateAsync(EngineId2, "es", "en", "Engine 2"); TranslationEngine? engine = await env.Engines.GetAsync(e => e.EngineId == EngineId2); Assert.Multiple(() => { @@ -38,12 +40,11 @@ public async Task StartBuildAsync(BuildJobRunnerType trainJobRunnerType) await env.Service.StartBuildAsync( EngineId1, BuildId1, - null, [ - new ParallelCorpus() + new ParallelCorpusContract() { Id = CorpusId1, - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -54,7 +55,7 @@ await env.Service.StartBuildAsync( InferenceTextIds = null, }, }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -94,7 +95,7 @@ public async Task CancelBuildAsync_Building(BuildJobRunnerType trainJobRunnerTyp using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForTrainingToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -120,7 +121,7 @@ public async Task StartBuildAsync_RestartUnfinishedBuild() using var env = new TestEnvironment(BuildJobRunnerType.Hangfire); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForTrainingToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -145,7 +146,7 @@ public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerTyp using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForTrainingToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -175,7 +176,7 @@ public async Task TrainSegmentPairAsync(BuildJobRunnerType trainJobRunnerType) using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForBuildToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -214,7 +215,9 @@ public async Task CommitAsync_LoadedActive() public async Task TranslateAsync() { using var env = new TestEnvironment(); - TranslationResult result = (await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."))[0]; + TranslationResultContract result = (await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."))[ + 0 + ]; Assert.That(result.Translation, Is.EqualTo("this is a TEST.")); } @@ -222,7 +225,7 @@ public async Task TranslateAsync() public async Task GetWordGraphAsync() { using var env = new TestEnvironment(); - WordGraph result = await env.Service.GetWordGraphAsync(EngineId1, "esto es una prueba."); + WordGraphContract result = await env.Service.GetWordGraphAsync(EngineId1, "esto es una prueba."); Assert.That( result.Arcs.Select(a => string.Join(' ', a.TargetTokens)), Is.EqualTo(new[] { "this is", "a test", "." }) @@ -230,11 +233,11 @@ public async Task GetWordGraphAsync() } [Test] - public void GetLanguageInfo() + public async Task GetLanguageInfoAsync() { using var env = new TestEnvironment(); - env.Service.IsLanguageNativeToModel("en", out string internalCode); - Assert.That(internalCode, Is.EqualTo("en")); + LanguageInfoContract info = await env.Service.GetLanguageInfoAsync("en"); + Assert.That(info.InternalCode, Is.EqualTo("en")); } private class TestEnvironment : DisposableBase @@ -419,7 +422,7 @@ private BackgroundJobServer CreateJobServer() var jobServerOptions = new BackgroundJobServerOptions { Activator = new EnvActivator(this), - Queues = new[] { "smt_transfer" }, + Queues = new[] { BuildJobQueues.SmtTransfer }, CancellationCheckInterval = TimeSpan.FromMilliseconds(50), }; return new BackgroundJobServer(jobServerOptions, _memoryStorage); @@ -443,7 +446,6 @@ private SmtTransferEngineService CreateService() return new SmtTransferEngineService( _lockFactory, PlatformService, - new MemoryDataAccessContext(), Engines, TrainSegmentPairs, StateService, @@ -603,7 +605,7 @@ private static TranslationSources[] GetSources(int count, bool isUnknown) public async Task WaitForAllHangfireJobsToFinishAsync() { IMonitoringApi monitoringApi = _memoryStorage.GetMonitoringApi(); - while (monitoringApi.EnqueuedCount("smt_transfer") > 0 || monitoringApi.ProcessingCount() > 0) + while (monitoringApi.EnqueuedCount(BuildJobQueues.SmtTransfer) > 0 || monitoringApi.ProcessingCount() > 0) await Task.Delay(50); } @@ -713,7 +715,7 @@ public override object ActivateJob(Type jobType) _env.SharedFileService, _env._lockFactory, _env.TrainSegmentPairs, - new ParallelCorpusService(), + Substitute.For(), _env.BuildJobOptions ) { diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/StatisticalEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/StatisticalEngineServiceTests.cs index 4722b4b88..c6d6ea0d5 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/StatisticalEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/StatisticalEngineServiceTests.cs @@ -1,4 +1,4 @@ -using WordAlignmentResult = Serval.WordAlignment.V1.WordAlignmentResult; +using Serval.WordAlignment.Contracts; namespace Serval.Machine.Shared.Services; @@ -14,7 +14,7 @@ public class StatisticalEngineServiceTests public async Task CreateAsync() { using var env = new TestEnvironment(); - await env.Service.CreateAsync(EngineId2, "Engine 2", "es", "en"); + await env.Service.CreateAsync(EngineId2, "es", "en", "Engine 2"); WordAlignmentEngine? engine = await env.Engines.GetAsync(e => e.EngineId == EngineId2); Assert.Multiple(() => { @@ -38,12 +38,11 @@ public async Task StartBuildAsync(BuildJobRunnerType trainJobRunnerType) await env.Service.StartBuildAsync( EngineId1, BuildId1, - null, [ - new ParallelCorpus() + new ParallelCorpusContract() { Id = CorpusId1, - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -54,7 +53,7 @@ await env.Service.StartBuildAsync( InferenceTextIds = null, }, }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -88,7 +87,7 @@ public async Task CancelBuildAsync_Building(BuildJobRunnerType trainJobRunnerTyp using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForTrainingToStartAsync(); WordAlignmentEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -115,7 +114,7 @@ public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerTyp using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, Array.Empty(), "{}"); await env.WaitForTrainingToStartAsync(); WordAlignmentEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -131,10 +130,14 @@ public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerTyp public async Task AlignAsync() { using var env = new TestEnvironment(); - WordAlignmentResult result = await env.Service.AlignAsync(EngineId1, "esto es una prueba.", "this is a test."); + WordAlignmentResultContract result = await env.Service.AlignAsync( + EngineId1, + "esto es una prueba.", + "this is a test." + ); Assert.That(string.Join(' ', result.TargetTokens), Is.EqualTo("this is a test .")); - Assert.That(result.Alignment.First().SourceIndex, Is.EqualTo(0)); - Assert.That(result.Alignment.First().TargetIndex, Is.EqualTo(0)); + Assert.That(result.Alignment[0].SourceIndex, Is.EqualTo(0)); + Assert.That(result.Alignment[0].TargetIndex, Is.EqualTo(0)); } private class TestEnvironment : DisposableBase @@ -298,7 +301,7 @@ private BackgroundJobServer CreateJobServer() var jobServerOptions = new BackgroundJobServerOptions { Activator = new EnvActivator(this), - Queues = new[] { "statistical" }, + Queues = new[] { BuildJobQueues.Statistical }, CancellationCheckInterval = TimeSpan.FromMilliseconds(50), }; return new BackgroundJobServer(jobServerOptions, _memoryStorage); @@ -320,7 +323,6 @@ private StatisticalEngineService CreateService() return new StatisticalEngineService( _lockFactory, PlatformService, - new MemoryDataAccessContext(), Engines, StateService, BuildJobService, @@ -350,7 +352,7 @@ private IWordAlignmentModelFactory CreateWordAlignmentModelFactory() public async Task WaitForAllHangfireJobsToFinishAsync() { IMonitoringApi monitoringApi = _memoryStorage.GetMonitoringApi(); - while (monitoringApi.EnqueuedCount("statistical") > 0 || monitoringApi.ProcessingCount() > 0) + while (monitoringApi.EnqueuedCount(BuildJobQueues.Statistical) > 0 || monitoringApi.ProcessingCount() > 0) await Task.Delay(50); } @@ -457,7 +459,7 @@ public override object ActivateJob(Type jobType) Substitute.For>(), _env.BuildJobService, _env.SharedFileService, - new ParallelCorpusService(), + Substitute.For(), _env.BuildJobOptions ) { diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml deleted file mode 100644 index 37d2772a0..000000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml +++ /dev/null @@ -1,33 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test2 - 65001 - T - - NFC - Te2 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Ten.SFM - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs index 969d244e3..8c24bf417 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs @@ -1,10 +1,4 @@ -global using System.Collections; -global using System.IO.Compression; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Grpc.Core; -global using Grpc.Core.Testing; -global using Hangfire; +global using Hangfire; global using Hangfire.Storage; global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Hosting.Internal; @@ -19,7 +13,7 @@ global using RichardSzalay.MockHttp; global using Serval.Machine.Shared.Configuration; global using Serval.Machine.Shared.Models; -global using Serval.Machine.Shared.Utils; +global using Serval.Shared.Contracts; global using SIL.DataAccess; global using SIL.Machine.Annotations; global using SIL.Machine.Corpora; @@ -27,8 +21,4 @@ global using SIL.Machine.Translation; global using SIL.Machine.Utils; global using SIL.ObjectModel; -global using SIL.Scripture; -global using SIL.ServiceToolkit.Models; -global using SIL.ServiceToolkit.Services; -global using SIL.ServiceToolkit.Utils; global using SIL.WritingSystems; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs new file mode 100644 index 000000000..15cf01273 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Utils; + +public sealed class IgnoreLineEndingsStringComparer : StringComparer +{ + public override int Compare(string? x, string? y) + { + return string.Compare(x?.ReplaceLineEndings(), y?.ReplaceLineEndings(), StringComparison.InvariantCulture); + } + + public override bool Equals(string? x, string? y) => + string.Equals(x?.ReplaceLineEndings(), y?.ReplaceLineEndings(), StringComparison.InvariantCulture); + + public override int GetHashCode(string obj) => obj.ReplaceLineEndings().GetHashCode(); +} diff --git a/src/Serval/src/Serval.ApiServer/Contracts/DeploymentInfoDto.cs b/src/Serval/src/Serval.ApiServer/Contracts/DeploymentInfoDto.cs deleted file mode 100644 index 41aa16b4d..000000000 --- a/src/Serval/src/Serval.ApiServer/Contracts/DeploymentInfoDto.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.ApiServer.Contracts; - -public class DeploymentInfoDto -{ - public string DeploymentVersion { get; set; } = "Unknown"; - public string AspNetCoreEnvironment { get; set; } = "Unknown"; -} diff --git a/src/Serval/src/Serval.ApiServer/Contracts/HealthReportDto.cs b/src/Serval/src/Serval.ApiServer/Contracts/HealthReportDto.cs deleted file mode 100644 index 3c8a46ac1..000000000 --- a/src/Serval/src/Serval.ApiServer/Contracts/HealthReportDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.ApiServer.Contracts; - -public class HealthReportDto -{ - public string Status { get; set; } = default!; - public string TotalDuration { get; set; } = default!; - public IDictionary Results { get; set; } = default!; -} diff --git a/src/Serval/src/Serval.ApiServer/Contracts/HealthReportEntryDto.cs b/src/Serval/src/Serval.ApiServer/Contracts/HealthReportEntryDto.cs deleted file mode 100644 index 38053e0c2..000000000 --- a/src/Serval/src/Serval.ApiServer/Contracts/HealthReportEntryDto.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Serval.ApiServer.Contracts; - -public class HealthReportEntryDto -{ - public string Status { get; set; } = default!; - public string Duration { get; set; } = default!; - public string? Description { get; set; } = default!; - public string? Exception { get; set; } = default!; - public IDictionary? Data { get; set; } = default!; -} diff --git a/src/Serval/src/Serval.ApiServer/Controllers/StatusController.cs b/src/Serval/src/Serval.ApiServer/Controllers/StatusController.cs index 60463c43b..5779a751b 100644 --- a/src/Serval/src/Serval.ApiServer/Controllers/StatusController.cs +++ b/src/Serval/src/Serval.ApiServer/Controllers/StatusController.cs @@ -36,7 +36,7 @@ public async Task> GetHealthAsync() { _logger.LogWarning("Health check failed: {Report}", Newtonsoft.Json.JsonConvert.SerializeObject(report)); } - return Ok(Map(report)); + return Ok(Map(report, includeResults: true)); } /// @@ -55,10 +55,8 @@ public async Task> GetPingAsync() _logger.LogWarning("Health check failed: {Report}", Newtonsoft.Json.JsonConvert.SerializeObject(report)); } - HealthReportDto reportDto = Map(report); - // remove results as this is a public endpoint - reportDto.Results = new Dictionary(); + HealthReportDto reportDto = Map(report, includeResults: false); return Ok(reportDto); } @@ -84,12 +82,12 @@ public ActionResult GetDeploymentInfo() ); } - private static HealthReportDto Map(HealthReport healthReport) + private static HealthReportDto Map(HealthReport healthReport, bool includeResults) { return new HealthReportDto { Status = healthReport.Status.ToString(), - Results = healthReport.Entries.ToDictionary(f => f.Key, f => Map(f.Value)), + Results = includeResults ? healthReport.Entries.ToDictionary(f => f.Key, f => Map(f.Value)) : [], TotalDuration = healthReport.TotalDuration.ToString(), }; } diff --git a/src/Serval/src/Serval.ApiServer/Dtos/DeploymentInfoDto.cs b/src/Serval/src/Serval.ApiServer/Dtos/DeploymentInfoDto.cs new file mode 100644 index 000000000..10bc64092 --- /dev/null +++ b/src/Serval/src/Serval.ApiServer/Dtos/DeploymentInfoDto.cs @@ -0,0 +1,7 @@ +namespace Serval.ApiServer.Dtos; + +public record DeploymentInfoDto +{ + public required string DeploymentVersion { get; init; } + public required string AspNetCoreEnvironment { get; init; } +} diff --git a/src/Serval/src/Serval.ApiServer/Dtos/HealthReportDto.cs b/src/Serval/src/Serval.ApiServer/Dtos/HealthReportDto.cs new file mode 100644 index 000000000..753e22e19 --- /dev/null +++ b/src/Serval/src/Serval.ApiServer/Dtos/HealthReportDto.cs @@ -0,0 +1,8 @@ +namespace Serval.ApiServer.Dtos; + +public record HealthReportDto +{ + public required string Status { get; init; } + public required string TotalDuration { get; init; } + public required IDictionary Results { get; init; } +} diff --git a/src/Serval/src/Serval.ApiServer/Dtos/HealthReportEntryDto.cs b/src/Serval/src/Serval.ApiServer/Dtos/HealthReportEntryDto.cs new file mode 100644 index 000000000..b93dd7aea --- /dev/null +++ b/src/Serval/src/Serval.ApiServer/Dtos/HealthReportEntryDto.cs @@ -0,0 +1,10 @@ +namespace Serval.ApiServer.Dtos; + +public record HealthReportEntryDto +{ + public required string Status { get; init; } + public required string Duration { get; init; } + public string? Description { get; init; } + public string? Exception { get; init; } + public IDictionary? Data { get; init; } +} diff --git a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj index 397ebc3c6..a40c2f67d 100644 --- a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj +++ b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj @@ -23,9 +23,8 @@ - - + all @@ -36,7 +35,6 @@ - @@ -46,6 +44,8 @@ + + diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/DiagnosticService.cs b/src/Serval/src/Serval.ApiServer/Services/DiagnosticService.cs similarity index 97% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/DiagnosticService.cs rename to src/Serval/src/Serval.ApiServer/Services/DiagnosticService.cs index 6a6163e7d..386b846fd 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/DiagnosticService.cs +++ b/src/Serval/src/Serval.ApiServer/Services/DiagnosticService.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.ApiServer.Services; /// /// Diagnostic information service. diff --git a/src/Serval/src/Serval.ApiServer/UrlService.cs b/src/Serval/src/Serval.ApiServer/Services/UrlService.cs similarity index 92% rename from src/Serval/src/Serval.ApiServer/UrlService.cs rename to src/Serval/src/Serval.ApiServer/Services/UrlService.cs index c0bccc0d3..746f8628b 100644 --- a/src/Serval/src/Serval.ApiServer/UrlService.cs +++ b/src/Serval/src/Serval.ApiServer/Services/UrlService.cs @@ -1,6 +1,4 @@ -using System.ComponentModel; - -namespace Serval.ApiServer; +namespace Serval.ApiServer.Services; public class UrlService(LinkGenerator linkGenerator) : IUrlService { diff --git a/src/Serval/src/Serval.ApiServer/Startup.cs b/src/Serval/src/Serval.ApiServer/Startup.cs index 5cf49fcc2..1a79fd25a 100644 --- a/src/Serval/src/Serval.ApiServer/Startup.cs +++ b/src/Serval/src/Serval.ApiServer/Startup.cs @@ -8,7 +8,6 @@ public class Startup(IConfiguration configuration, IWebHostEnvironment environme public void ConfigureServices(IServiceCollection services) { - services.AddFeatureManagement(); services.AddMemoryCache(); services.AddRouting(o => o.LowercaseUrls = true); @@ -72,55 +71,20 @@ public void ConfigureServices(IServiceCollection services) }); services.AddSingleton(); services.AddSingleton(); - - services.AddGrpc(); - - services - .AddServal(Configuration) - .AddMongoDataAccess(cfg => - { - cfg.AddTranslationRepositories(); - cfg.AddWordAlignmentRepositories(); - cfg.AddDataFilesRepositories(); - cfg.AddWebhooksRepositories(); - }) - .AddMongoOutbox() - .AddOutboxDeliveryService() - .AddTranslation() - .AddWordAlignment() - .AddDataFiles() - .AddWebhooks(); services.AddTransient(); - services.AddHangfire(c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy(), - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) + services.AddServal( + Configuration, + c => + { + c.AddTranslation(); + c.AddWordAlignment(); + c.AddDataFiles(); + c.AddWebhooks(); + c.AddMachineEngines(); + c.AddEchoEngines(); + } ); - services.AddHangfireServer(); - - services.AddMediator(cfg => - { - cfg.AddTranslationConsumers(); - cfg.AddWordAlignmentConsumers(); - cfg.AddDataFilesConsumers(); - cfg.AddWebhooksConsumers(); - }); - services.AddScoped(sp => sp.GetRequiredService()); - services.AddScoped(sp => sp.GetRequiredService().CreateRequestClient()); services .AddApiVersioning(o => @@ -152,8 +116,6 @@ public void ConfigureServices(IServiceCollection services) o.ApiGroupNames = ["v" + version.Major]; o.Version = version.Major + "." + version.Minor; - var featureManager = sp.GetRequiredService(); - o.SchemaSettings.SchemaNameGenerator = new ServalSchemaNameGenerator(); o.UseControllerSummaryAsTagDescription = true; o.AddSecurity( @@ -199,7 +161,6 @@ public void ConfigureServices(IServiceCollection services) builder .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") .AddConsoleExporter(); }); @@ -212,7 +173,7 @@ public void ConfigureServices(IServiceCollection services) } services.Configure(Configuration.GetSection("Bugsnag")); services.AddBugsnag(); - services.AddDiagnostics(); + services.AddHostedService(); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) @@ -228,8 +189,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseEndpoints(x => { x.MapControllers(); - x.MapServalTranslationServices(); - x.MapServalWordAlignmentServices(); x.MapHangfireDashboard(); }); diff --git a/src/Serval/src/Serval.ApiServer/Usings.cs b/src/Serval/src/Serval.ApiServer/Usings.cs index 941afefa2..175232ba0 100644 --- a/src/Serval/src/Serval.ApiServer/Usings.cs +++ b/src/Serval/src/Serval.ApiServer/Usings.cs @@ -1,21 +1,17 @@ -global using System.Net; +global using System.ComponentModel; +global using System.Diagnostics; +global using System.Net; global using System.Security.Claims; global using System.Text.Json.Serialization; global using Asp.Versioning; global using Bugsnag.AspNet.Core; global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using MassTransit; -global using MassTransit.Mediator; global using Microsoft.AspNetCore.Authentication.JwtBearer; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http.Timeouts; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.OutputCaching; global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.FeatureManagement; global using Microsoft.IdentityModel.Tokens; global using NJsonSchema; global using NJsonSchema.Generation; @@ -25,9 +21,9 @@ global using NSwag.Generation.Processors.Security; global using OpenTelemetry.Metrics; global using OpenTelemetry.Trace; -global using Serval.ApiServer.Contracts; +global using Serval.ApiServer.Dtos; +global using Serval.ApiServer.Services; global using Serval.Shared.Configuration; -global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; global using Serval.Shared.Models; global using Serval.Shared.Services; diff --git a/src/Serval/src/Serval.ApiServer/appsettings.Development.json b/src/Serval/src/Serval.ApiServer/appsettings.Development.json index 05d945c40..302cabf0e 100644 --- a/src/Serval/src/Serval.ApiServer/appsettings.Development.json +++ b/src/Serval/src/Serval.ApiServer/appsettings.Development.json @@ -4,34 +4,6 @@ "Mongo": "mongodb://localhost:27017/serval", "Hangfire": "mongodb://localhost:27017/serval_jobs" }, - "Translation": { - "Engines": [ - { - "Type": "Echo", - "Address": "https://localhost:8055" - }, - { - "Type": "SmtTransfer", - "Address": "https://localhost:9000" - }, - { - "Type": "Nmt", - "Address": "https://localhost:9000" - } - ] - }, - "WordAlignment": { - "Engines": [ - { - "Type": "EchoWordAlignment", - "Address": "https://localhost:8055" - }, - { - "Type": "Statistical", - "Address": "https://localhost:9000" - } - ] - }, "Logging": { "LogLevel": { "Default": "Information", @@ -40,5 +12,14 @@ }, "Bugsnag": { "ReleaseStage": "Development" + }, + + // Machine configuration + "ClearML": { + "MaxSteps": 1000, + "Project": "dev" + }, + "SharedFile": { + "Uri": "s3://silnlp/dev/" } } \ No newline at end of file diff --git a/src/Serval/src/Serval.ApiServer/appsettings.json b/src/Serval/src/Serval.ApiServer/appsettings.json index 8f0ca24dc..11165e7ec 100644 --- a/src/Serval/src/Serval.ApiServer/appsettings.json +++ b/src/Serval/src/Serval.ApiServer/appsettings.json @@ -1,6 +1,8 @@ { + "ConnectionStrings": { + "ClearML": "https://api.sil.hosted.allegro.ai" + }, "AllowedHosts": "*", - "FeatureManagement": {}, "Auth": { "Domain": "sil-appbuilder.auth0.com", "Audience": "https://serval-api.org/" @@ -21,7 +23,37 @@ "Production" ] }, - "MessageOutbox": { - "OutboxDir": "/var/lib/serval/outbox" + + // Machine configuration + "BuildJob": { + "ClearML": [ + { + "EngineType": "Nmt", + "ModelType": "huggingface", + "Queue": "jobs_backlog", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "EngineType": "SmtTransfer", + "ModelType": "thot", + "Queue": "jobs_backlog.cpu_only", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "EngineType": "Statistical", + "ModelType": "thot", + "Queue": "jobs_backlog.cpu_only", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + } + ] + }, + "SmtTransferEngine": { + "EnginesDir": "/var/lib/machine/engines" + }, + "StatisticalEngine": { + "EnginesDir": "/var/lib/machine/engines" + }, + "ClearML": { + "BuildPollingEnabled": true } } \ No newline at end of file diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1ccb305f8..ef5f7c223 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -2179,34 +2179,43 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial interface ITranslationBuildsClient + public partial interface ITranslationEngineTypesClient { /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all builds for your translation engines that are created after the specified date. + /// Get information regarding a language for a given engine type /// - /// The date and time in UTC that the builds were created after (optional). - /// The engines + /// + /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally + ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. + ///
**Base Models available** + ///
* **NLLB-200**: This is the only base NMT translation model currently available. + ///
* The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). + ///
Response format: + ///
* **`engineType`**: See above + ///
* **`isNative`**: Whether the base translation model supports this language without fine-tuning. + ///
* **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt, echo, or smt-transfer + /// The language to retrieve information on. + /// Language information for the specified engine type /// A server side error occurred. - System.Threading.Tasks.Task> GetAllBuildsCreatedAfterAsync(System.DateTimeOffset? createdAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the next build that finished after the specified date and time. - ///
If not build has yet completed after that timestamp, - ///
Serval will wait until a build is finished after that date and time. + /// Get queue information for a given engine type /// - /// The date and time in UTC that the next build should have finished after. - ///
You should use the finished timestamp of the build previously returned when calling this endpoint. - /// The engines + /// A valid engine type: smt-transfer, nmt, or echo + /// Queue information for the specified engine type /// A server side error occurred. - System.Threading.Tasks.Task GetNextFinishedBuildAsync(System.DateTimeOffset? finishedAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationBuildsClient : ITranslationBuildsClient + public partial class TranslationEngineTypesClient : ITranslationEngineTypesClient { #pragma warning disable 8618 private string _baseUrl; @@ -2217,7 +2226,7 @@ public partial class TranslationBuildsClient : ITranslationBuildsClient private Newtonsoft.Json.JsonSerializerSettings _instanceSettings; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - public TranslationBuildsClient(System.Net.Http.HttpClient httpClient) + public TranslationEngineTypesClient(System.Net.Http.HttpClient httpClient) #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. { BaseUrl = "/api/v1"; @@ -2255,13 +2264,31 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all builds for your translation engines that are created after the specified date. + /// Get information regarding a language for a given engine type /// - /// The date and time in UTC that the builds were created after (optional). - /// The engines + /// + /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally + ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. + ///
**Base Models available** + ///
* **NLLB-200**: This is the only base NMT translation model currently available. + ///
* The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). + ///
Response format: + ///
* **`engineType`**: See above + ///
* **`isNative`**: Whether the base translation model supports this language without fine-tuning. + ///
* **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt, echo, or smt-transfer + /// The language to retrieve information on. + /// Language information for the specified engine type /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllBuildsCreatedAfterAsync(System.DateTimeOffset? createdAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { + if (engineType == null) + throw new System.ArgumentNullException("engineType"); + + if (language == null) + throw new System.ArgumentNullException("language"); + var client_ = _httpClient; var disposeClient_ = false; try @@ -2273,14 +2300,11 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/builds" - urlBuilder_.Append("translation/builds"); - urlBuilder_.Append('?'); - if (createdAfter != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("created-after")).Append('=').Append(System.Uri.EscapeDataString(createdAfter.Value.ToString("u", System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + // Operation Path: "translation/engine-types/{engineType}/languages/{language}" + urlBuilder_.Append("translation/engine-types/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/languages/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(language, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -2307,7 +2331,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -2318,19 +2342,19 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); } else - if (status_ == 503) + if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); } else { @@ -2354,16 +2378,16 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the next build that finished after the specified date and time. - ///
If not build has yet completed after that timestamp, - ///
Serval will wait until a build is finished after that date and time. + /// Get queue information for a given engine type /// - /// The date and time in UTC that the next build should have finished after. - ///
You should use the finished timestamp of the build previously returned when calling this endpoint. - /// The engines + /// A valid engine type: smt-transfer, nmt, or echo + /// Queue information for the specified engine type /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetNextFinishedBuildAsync(System.DateTimeOffset? finishedAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { + if (engineType == null) + throw new System.ArgumentNullException("engineType"); + var client_ = _httpClient; var disposeClient_ = false; try @@ -2375,14 +2399,10 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/builds/next-finished" - urlBuilder_.Append("translation/builds/next-finished"); - urlBuilder_.Append('?'); - if (finishedAfter != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("finished-after")).Append('=').Append(System.Uri.EscapeDataString(finishedAfter.Value.ToString("u", System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + // Operation Path: "translation/engine-types/{engineType}/queues" + urlBuilder_.Append("translation/engine-types/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/queues"); PrepareRequest(client_, request_, urlBuilder_); @@ -2409,7 +2429,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -2420,25 +2440,19 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); - } - else - if (status_ == 408) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); } else { @@ -2598,11 +2612,12 @@ public partial interface ITranslationEnginesClient /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all translation engines + /// Cancel the current build job (whether pending or active) for a translation engine /// - /// The engines + /// The translation engine id + /// The build job was cancelled successfully. /// A server side error occurred. - System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// @@ -2646,12 +2661,11 @@ public partial interface ITranslationEnginesClient /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a translation engine by unique id + /// Get all translation engines /// - /// The translation engine id - /// The translation engine + /// The engines /// A server side error occurred. - System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// @@ -2662,6 +2676,15 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a translation engine by unique id + /// + /// The translation engine id + /// The translation engine + /// A server side error occurred. + System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// /// Update the source and/or target languages of a translation engine @@ -2681,34 +2704,85 @@ public partial interface ITranslationEnginesClient /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Translate a segment of text + /// Get a link to download the NMT translation model of the last build that was successfully saved. /// + /// + /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, + ///
then the model from the most recent successful build can be downloaded. + ///
+ ///
The endpoint will return a URL that can be used to download the model for up to 1 hour + ///
after the request is made. If the URL is not used within that time, a new request will need to be made. + ///
+ ///
The download itself is created by g-zipping together the folder containing the fine tuned model + ///
with all necessary supporting files. This zipped folder is then named by the pattern: + ///
* <engine_id>_<model_revision>.tar.gz + /// /// The translation engine id - /// The source segment - /// The translation result + /// The url to download the model. /// A server side error occurred. - System.Threading.Tasks.Task TranslateAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Returns the top N translations of a segment + /// Get the word graph that represents all possible translations of a segment of text /// /// The translation engine id - /// The number of translations to generate /// The source segment - /// The translation results + /// The word graph result /// A server side error occurred. - System.Threading.Tasks.Task> TranslateNAsync(string id, int n, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetWordGraphAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the word graph that represents all possible translations of a segment of text + /// Starts a build job for a translation engine. + /// + /// + /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). + ///
Specifying a corpus: + ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. + ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. + ///
+ ///
Filtering by text id or chapter: + ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). + ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). + ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + ///
+ ///
Filter - train on all or none + ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively + ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. + ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. + ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. + ///
+ ///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, + ///
the following text will be pretranslated: + ///
* Text segments that are in the source but do not exist in the target. + ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. + ///
If the engine does not support pretranslation, these fields have no effect. + ///
Pretranslating uses the same filtering as training. + ///
+ ///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. + ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + ///
+ ///
Note that when using a parallel corpus: + ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. + ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + /// + /// The translation engine id + /// The build config (see remarks) + /// The new build job + /// A server side error occurred. + System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all build jobs for a translation engine /// /// The translation engine id - /// The source segment - /// The word graph result + /// The build jobs /// A server side error occurred. - System.Threading.Tasks.Task GetWordGraphAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// @@ -2725,6 +2799,27 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task TrainSegmentAsync(string id, SegmentPair segmentPair, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Translate a segment of text + /// + /// The translation engine id + /// The source segment + /// The translation result + /// A server side error occurred. + System.Threading.Tasks.Task TranslateAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Returns the top N translations of a segment + /// + /// The translation engine id + /// The number of translations to generate + /// The source segment + /// The translation results + /// A server side error occurred. + System.Threading.Tasks.Task> TranslateNAsync(string id, int n, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// /// Add a corpus to a translation engine (obsolete - use parallel corpora instead) @@ -3048,59 +3143,7 @@ public partial interface ITranslationEnginesClient /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all build jobs for a translation engine - /// - /// The translation engine id - /// The build jobs - /// A server side error occurred. - System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Starts a build job for a translation engine. - /// - /// - /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). - ///
Specifying a corpus: - ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. - ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
- ///
Filtering by text id or chapter: - ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). - ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). - ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
- ///
Filter - train on all or none - ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively - ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. - ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. - ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
- ///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, - ///
the following text will be pretranslated: - ///
* Text segments that are in the source but do not exist in the target. - ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. - ///
If the engine does not support pretranslation, these fields have no effect. - ///
Pretranslating uses the same filtering as training. - ///
- ///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. - ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. - ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. - ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - ///
- ///
Note that when using a parallel corpus: - ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. - ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. - /// - /// The translation engine id - /// The build config (see remarks) - /// The new build job - /// A server side error occurred. - System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get a build job + /// Get a build job /// /// /// If the `minRevision` is not defined, the current build, at whatever state it is, @@ -3132,35 +3175,6 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task GetCurrentBuildAsync(string id, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Cancel the current build job (whether pending or active) for a translation engine - /// - /// The translation engine id - /// The build job was cancelled successfully. - /// A server side error occurred. - System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get a link to download the NMT translation model of the last build that was successfully saved. - /// - /// - /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, - ///
then the model from the most recent successful build can be downloaded. - ///
- ///
The endpoint will return a URL that can be used to download the model for up to 1 hour - ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
- ///
The download itself is created by g-zipping together the folder containing the fine tuned model - ///
with all necessary supporting files. This zipped folder is then named by the pattern: - ///
* <engine_id>_<model_revision>.tar.gz - /// - /// The translation engine id - /// The url to download the model. - /// A server side error occurred. - System.Threading.Tasks.Task GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] @@ -3213,25 +3227,32 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all translation engines + /// Cancel the current build job (whether pending or active) for a translation engine /// - /// The engines + /// The translation engine id + /// The build job was cancelled successfully. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { + if (id == null) + throw new System.ArgumentNullException("id"); + var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Content = new System.Net.Http.StringContent(string.Empty, System.Text.Encoding.UTF8, "application/json"); + request_.Method = new System.Net.Http.HttpMethod("POST"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines" - urlBuilder_.Append("translation/engines"); + // Operation Path: "translation/engines/{id}/current-build/cancel" + urlBuilder_.Append("translation/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/current-build/cancel"); PrepareRequest(client_, request_, urlBuilder_); @@ -3258,7 +3279,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -3266,6 +3287,12 @@ public string BaseUrl return objectResponse_.Object; } else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("There is no active build job.", status_, responseText_, headers_, null); + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -3275,7 +3302,19 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The translation engine does not support cancelling builds.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3442,16 +3481,12 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a translation engine by unique id + /// Get all translation engines /// - /// The translation engine id - /// The translation engine + /// The engines /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { - if (id == null) - throw new System.ArgumentNullException("id"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -3463,9 +3498,8 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}" - urlBuilder_.Append("translation/engines/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + // Operation Path: "translation/engines" + urlBuilder_.Append("translation/engines"); PrepareRequest(client_, request_, urlBuilder_); @@ -3492,7 +3526,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -3509,13 +3543,7 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); - } - else - if (status_ == 404) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3642,38 +3670,24 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Update the source and/or target languages of a translation engine + /// Get a translation engine by unique id /// - /// - /// ## Sample request: - ///
- ///
{ - ///
"sourceLanguage": "en", - ///
"targetLanguage": "en" - ///
} - /// /// The translation engine id - /// The engine language was successfully updated. + /// The translation engine /// A server side error occurred. - public virtual async System.Threading.Tasks.Task UpdateAsync(string id, TranslationEngineUpdateConfig request, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (request == null) - throw new System.ArgumentNullException("request"); - var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(request, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("PATCH"); + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); @@ -3706,7 +3720,12 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - return; + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; } else if (status_ == 401) @@ -3724,7 +3743,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist and therefore cannot be updated.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3754,19 +3773,26 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Translate a segment of text + /// Update the source and/or target languages of a translation engine /// + /// + /// ## Sample request: + ///
+ ///
{ + ///
"sourceLanguage": "en", + ///
"targetLanguage": "en" + ///
} + /// /// The translation engine id - /// The source segment - /// The translation result + /// The engine language was successfully updated. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task TranslateAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task UpdateAsync(string id, TranslationEngineUpdateConfig request, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (segment == null) - throw new System.ArgumentNullException("segment"); + if (request == null) + throw new System.ArgumentNullException("request"); var client_ = _httpClient; var disposeClient_ = false; @@ -3774,19 +3800,17 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segment, JsonSerializerSettings); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(request, JsonSerializerSettings); var content_ = new System.Net.Http.StringContent(json_); content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + request_.Method = new System.Net.Http.HttpMethod("PATCH"); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/translate" + // Operation Path: "translation/engines/{id}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/translate"); PrepareRequest(client_, request_, urlBuilder_); @@ -3813,18 +3837,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; - } - else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + return; } else if (status_ == 401) @@ -3842,19 +3855,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); - } - else - if (status_ == 405) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); - } - else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist and therefore cannot be updated.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3884,44 +3885,42 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Returns the top N translations of a segment + /// Get a link to download the NMT translation model of the last build that was successfully saved. /// + /// + /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, + ///
then the model from the most recent successful build can be downloaded. + ///
+ ///
The endpoint will return a URL that can be used to download the model for up to 1 hour + ///
after the request is made. If the URL is not used within that time, a new request will need to be made. + ///
+ ///
The download itself is created by g-zipping together the folder containing the fine tuned model + ///
with all necessary supporting files. This zipped folder is then named by the pattern: + ///
* <engine_id>_<model_revision>.tar.gz + /// /// The translation engine id - /// The number of translations to generate - /// The source segment - /// The translation results + /// The url to download the model. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> TranslateNAsync(string id, int n, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (n == null) - throw new System.ArgumentNullException("n"); - - if (segment == null) - throw new System.ArgumentNullException("segment"); - var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segment, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Method = new System.Net.Http.HttpMethod("GET"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/translate/{n}" + // Operation Path: "translation/engines/{id}/model-download-url" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/translate/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(n, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/model-download-url"); PrepareRequest(client_, request_, urlBuilder_); @@ -3948,7 +3947,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -3956,12 +3955,6 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -3971,25 +3964,19 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist or there is no saved model.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); - } - else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); + throw new ServalApiException("The translation engine does not support downloading builds.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -4149,24 +4136,52 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Incrementally train a translation engine with a segment pair + /// Starts a build job for a translation engine. /// /// - /// A segment pair consists of a source and target segment as well as a boolean flag `sentenceStart` - ///
that should be set to `true` if this segment pair forms the beginning of a sentence. (This information - ///
will be used to reconstruct proper capitalization when training/inferencing). + /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). + ///
Specifying a corpus: + ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. + ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. + ///
+ ///
Filtering by text id or chapter: + ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). + ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). + ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + ///
+ ///
Filter - train on all or none + ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively + ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. + ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. + ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. + ///
+ ///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, + ///
the following text will be pretranslated: + ///
* Text segments that are in the source but do not exist in the target. + ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. + ///
If the engine does not support pretranslation, these fields have no effect. + ///
Pretranslating uses the same filtering as training. + ///
+ ///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. + ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + ///
+ ///
Note that when using a parallel corpus: + ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. + ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. /// /// The translation engine id - /// The segment pair - /// The engine was trained successfully. + /// The build config (see remarks) + /// The new build job /// A server side error occurred. - public virtual async System.Threading.Tasks.Task TrainSegmentAsync(string id, SegmentPair segmentPair, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (segmentPair == null) - throw new System.ArgumentNullException("segmentPair"); + if (buildConfig == null) + throw new System.ArgumentNullException("buildConfig"); var client_ = _httpClient; var disposeClient_ = false; @@ -4174,18 +4189,19 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segmentPair, JsonSerializerSettings); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(buildConfig, JsonSerializerSettings); var content_ = new System.Net.Http.StringContent(json_); content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); request_.Content = content_; request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/train-segment" + // Operation Path: "translation/engines/{id}/builds" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/train-segment"); + urlBuilder_.Append("/builds"); PrepareRequest(client_, request_, urlBuilder_); @@ -4210,15 +4226,20 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 200) + if (status_ == 201) { - return; + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; } else if (status_ == 400) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + throw new ServalApiException("The build configuration was invalid.", status_, responseText_, headers_, null); } else if (status_ == 401) @@ -4230,7 +4251,7 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) @@ -4239,16 +4260,10 @@ public string BaseUrl throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 405) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); - } - else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + throw new ServalApiException("There is already an active/pending build or a build in the process of being canceled.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -4278,58 +4293,31 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Add a corpus to a translation engine (obsolete - use parallel corpora instead) + /// Get all build jobs for a translation engine /// - /// - /// ## Parameters - ///
* **name**: A name to help identify and distinguish the corpus from other corpora - ///
* The name does not have to be unique since the corpus is uniquely identified by an auto-generated id - ///
* **`sourceLanguage`**: The source language code (See documentation on endpoint /translation/engines/ - "Create a new translation engine" for details on language codes). - ///
* Normally, this is the same as the engine's `sourceLanguage`. This may change for future engines as a means of transfer learning. - ///
* **`targetLanguage`**: The target language code (See documentation on endpoint /translation/engines/ - "Create a new translation engine" for details on language codes). - ///
* **`sourceFiles`**: The source files associated with the corpus - ///
* **`fileId`**: The unique id referencing the uploaded file - ///
* **`textId`**: The client-defined name to associate source and target files. - ///
* If the text ids in the source files and target files match, they will be used to train the engine. - ///
* If selected for pretranslation when building, all source files that have no target file, or lines of text in a source file that have missing or blank lines in the target file will be pretranslated. - ///
* If a text id is used more than once in source files, the sources will be randomly and evenly mixed for training. - ///
* For pretranslating, multiple sources with the same text id will be combined, but the first source will always take precedence (no random mixing). - ///
* For Paratext projects, text id will be ignored - multiple Paratext source projects will always be mixed (as if they have the same text id). - ///
* **`targetFiles`**: The target files associated with the corpus - ///
* Same as `sourceFiles`, except only a single instance of a text id or a single Paratext project is supported. There is no mixing or combining of multiple targets. - /// /// The translation engine id - /// The corpus configuration (see remarks) - /// The added corpus + /// The build jobs /// A server side error occurred. - [System.Obsolete] - public virtual async System.Threading.Tasks.Task AddCorpusAsync(string id, TranslationCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusConfig == null) - throw new System.ArgumentNullException("corpusConfig"); - var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Method = new System.Net.Http.HttpMethod("GET"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora" + // Operation Path: "translation/engines/{id}/builds" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora"); + urlBuilder_.Append("/builds"); PrepareRequest(client_, request_, urlBuilder_); @@ -4354,9 +4342,9 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 201) + if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -4364,12 +4352,6 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -4415,32 +4397,43 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all corpora for a translation engine (obsolete - use parallel corpora instead) + /// Incrementally train a translation engine with a segment pair /// + /// + /// A segment pair consists of a source and target segment as well as a boolean flag `sentenceStart` + ///
that should be set to `true` if this segment pair forms the beginning of a sentence. (This information + ///
will be used to reconstruct proper capitalization when training/inferencing). + /// /// The translation engine id - /// The corpora + /// The segment pair + /// The engine was trained successfully. /// A server side error occurred. - [System.Obsolete] - public virtual async System.Threading.Tasks.Task> GetAllCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task TrainSegmentAsync(string id, SegmentPair segmentPair, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); + if (segmentPair == null) + throw new System.ArgumentNullException("segmentPair"); + var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segmentPair, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora" + // Operation Path: "translation/engines/{id}/train-segment" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora"); + urlBuilder_.Append("/train-segment"); PrepareRequest(client_, request_, urlBuilder_); @@ -4467,36 +4460,49 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; + return; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -4520,28 +4526,19 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Update a corpus with a new set of files (obsolete - use parallel corpora instead) + /// Translate a segment of text /// - /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. - /// /// The translation engine id - /// The corpus id - /// The corpus configuration - /// The corpus was updated successfully + /// The source segment + /// The translation result /// A server side error occurred. - [System.Obsolete] - public virtual async System.Threading.Tasks.Task UpdateCorpusAsync(string id, string corpusId, TranslationCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task TranslateAsync(string id, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); - - if (corpusConfig == null) - throw new System.ArgumentNullException("corpusConfig"); + if (segment == null) + throw new System.ArgumentNullException("segment"); var client_ = _httpClient; var disposeClient_ = false; @@ -4549,20 +4546,19 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segment, JsonSerializerSettings); var content_ = new System.Net.Http.StringContent(json_); content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("PATCH"); + request_.Method = new System.Net.Http.HttpMethod("POST"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}" + // Operation Path: "translation/engines/{id}/translate" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/translate"); PrepareRequest(client_, request_, urlBuilder_); @@ -4589,7 +4585,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -4618,7 +4614,19 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -4648,20 +4656,23 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the configuration of a corpus for a translation engine (obsolete - use parallel corpora instead) + /// Returns the top N translations of a segment /// /// The translation engine id - /// The corpus id - /// The corpus configuration + /// The number of translations to generate + /// The source segment + /// The translation results /// A server side error occurred. - [System.Obsolete] - public virtual async System.Threading.Tasks.Task GetCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> TranslateNAsync(string id, int n, string segment, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); + if (n == null) + throw new System.ArgumentNullException("n"); + + if (segment == null) + throw new System.ArgumentNullException("segment"); var client_ = _httpClient; var disposeClient_ = false; @@ -4669,16 +4680,20 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(segment, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}" + // Operation Path: "translation/engines/{id}/translate/{n}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/translate/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(n, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -4705,7 +4720,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -4713,6 +4728,12 @@ public string BaseUrl return objectResponse_.Object; } else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -4728,7 +4749,19 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -4758,24 +4791,38 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Remove a corpus from a translation engine (obsolete - use parallel corpora instead) + /// Add a corpus to a translation engine (obsolete - use parallel corpora instead) /// /// - /// Removing a corpus will remove all pretranslations associated with that corpus. + /// ## Parameters + ///
* **name**: A name to help identify and distinguish the corpus from other corpora + ///
* The name does not have to be unique since the corpus is uniquely identified by an auto-generated id + ///
* **`sourceLanguage`**: The source language code (See documentation on endpoint /translation/engines/ - "Create a new translation engine" for details on language codes). + ///
* Normally, this is the same as the engine's `sourceLanguage`. This may change for future engines as a means of transfer learning. + ///
* **`targetLanguage`**: The target language code (See documentation on endpoint /translation/engines/ - "Create a new translation engine" for details on language codes). + ///
* **`sourceFiles`**: The source files associated with the corpus + ///
* **`fileId`**: The unique id referencing the uploaded file + ///
* **`textId`**: The client-defined name to associate source and target files. + ///
* If the text ids in the source files and target files match, they will be used to train the engine. + ///
* If selected for pretranslation when building, all source files that have no target file, or lines of text in a source file that have missing or blank lines in the target file will be pretranslated. + ///
* If a text id is used more than once in source files, the sources will be randomly and evenly mixed for training. + ///
* For pretranslating, multiple sources with the same text id will be combined, but the first source will always take precedence (no random mixing). + ///
* For Paratext projects, text id will be ignored - multiple Paratext source projects will always be mixed (as if they have the same text id). + ///
* **`targetFiles`**: The target files associated with the corpus + ///
* Same as `sourceFiles`, except only a single instance of a text id or a single Paratext project is supported. There is no mixing or combining of multiple targets. /// /// The translation engine id - /// The corpus id - /// If `true`, all files associated with the corpus will be deleted as well (even if they are associated with other corpora). If false, no files will be deleted. - /// The corpus was deleted successfully. + /// The corpus configuration (see remarks) + /// The added corpus /// A server side error occurred. [System.Obsolete] - public virtual async System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, bool? deleteFiles = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task AddCorpusAsync(string id, TranslationCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); var client_ = _httpClient; var disposeClient_ = false; @@ -4783,21 +4830,19 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("DELETE"); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}" + // Operation Path: "translation/engines/{id}/corpora" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append('?'); - if (deleteFiles != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("delete-files")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(deleteFiles, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + urlBuilder_.Append("/corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -4822,9 +4867,20 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 200) + if (status_ == 201) { - return; + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); } else if (status_ == 401) @@ -4842,7 +4898,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -4872,33 +4928,17 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all pretranslations in a corpus or parallel corpus of a translation engine + /// Get all corpora for a translation engine (obsolete - use parallel corpora instead) /// - /// - /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: - ///
* **`textId`**: The text id of the source file defined when the corpus was created. - ///
* **`refs`** (a list of strings): A list of references including: - ///
* The references defined in the source file per line, if any. - ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. - ///
* **`translation`**: the text of the pretranslation - ///
- ///
Pretranslations can be filtered by text id if provided. - ///
Only pretranslations for the most recent successful build of the engine are returned. - /// /// The translation engine id - /// The corpus id or parallel corpus id - /// The text id (optional) - /// The pretranslations + /// The corpora /// A server side error occurred. [System.Obsolete] - public virtual async System.Threading.Tasks.Task> GetAllCorpusPretranslationsAsync(string id, string corpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -4910,18 +4950,10 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations" + // Operation Path: "translation/engines/{id}/corpora" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations"); - urlBuilder_.Append('?'); - if (textId != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("text-id")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + urlBuilder_.Append("/corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -4948,7 +4980,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -4959,31 +4991,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); - } - else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); } else { @@ -5007,25 +5033,19 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all pretranslations for the specified text in a corpus or parallel corpus of a translation engine + /// Update a corpus with a new set of files (obsolete - use parallel corpora instead) /// /// - /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: - ///
* **`textId`**: The text id of the source file defined when the corpus was created. - ///
* **`refs`** (a list of strings): A list of references including: - ///
* The references defined in the source file per line, if any. - ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. - ///
* **`translation`**: the text of the pretranslation - ///
- ///
Only pretranslations for the most recent successful build of the engine are returned. + /// See posting a new corpus for details of use. Will completely replace corpus' file associations. + ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. /// /// The translation engine id - /// The corpus id or parallel corpus id - /// The text id - /// The pretranslations + /// The corpus id + /// The corpus configuration + /// The corpus was updated successfully /// A server side error occurred. [System.Obsolete] - public virtual async System.Threading.Tasks.Task> GetCorpusPretranslationsByTextIdAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task UpdateCorpusAsync(string id, string corpusId, TranslationCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -5033,8 +5053,8 @@ public string BaseUrl if (corpusId == null) throw new System.ArgumentNullException("corpusId"); - if (textId == null) - throw new System.ArgumentNullException("textId"); + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); var client_ = _httpClient; var disposeClient_ = false; @@ -5042,18 +5062,20 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("PATCH"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); urlBuilder_.Append("/corpora/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -5080,7 +5102,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -5088,6 +5110,12 @@ public string BaseUrl return objectResponse_.Object; } else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -5106,12 +5134,6 @@ public string BaseUrl throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); - } - else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -5139,49 +5161,14 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a pretranslated Scripture book in USFM format. + /// Get the configuration of a corpus for a translation engine (obsolete - use parallel corpora instead) /// - /// - /// The text that populates the USFM structure can be controlled by the `text-origin` parameter: - ///
* `PreferExisting`: The existing and pretranslated texts are merged into the USFM, preferring existing text. **This is the default**. - ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. - ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). - ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
- ///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: - ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. - ///
* `Source`: The source book is used as the template. - ///
* `Target`: The target book is used as the template. - ///
- ///
The intra-segment USFM markers are handled in the following way: - ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. - ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
- ///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. - ///
* `PushToEnd`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers**. - ///
* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. - ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for embeds and style markers**. - ///
- ///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: - ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. - ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
- ///
Only pretranslations for the most recent successful build of the engine are returned. - ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). - /// /// The translation engine id - /// The corpus id or parallel corpus id - /// The text id - /// The source[s] of the data to populate the USFM file with. - /// The source or target book to use as the USFM template. - /// The behavior of paragraph markers. - /// The behavior of embed markers. - /// The behavior of style markers. - /// The normalization behavior of quotes. - /// The book in USFM format + /// The corpus id + /// The corpus configuration /// A server side error occurred. [System.Obsolete] - public virtual async System.Threading.Tasks.Task GetCorpusPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -5189,9 +5176,6 @@ public string BaseUrl if (corpusId == null) throw new System.ArgumentNullException("corpusId"); - if (textId == null) - throw new System.ArgumentNullException("textId"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -5199,44 +5183,15 @@ public string BaseUrl using (var request_ = new System.Net.Http.HttpRequestMessage()) { request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("text/plain")); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}/usfm" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); urlBuilder_.Append("/corpora/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/usfm"); - urlBuilder_.Append('?'); - if (textOrigin != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("text-origin")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textOrigin, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (template != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("template")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(template, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (paragraphMarkerBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("paragraph-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(paragraphMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (embedBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("embed-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(embedBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (styleMarkerBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("style-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(styleMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (quoteNormalizationBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("quotation-mark-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(quoteNormalizationBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -5263,27 +5218,18 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - var result_ = (string)System.Convert.ChangeType(responseData_, typeof(string)); - return result_; - } - else - if (status_ == 204) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The specified book does not exist in the source or target corpus.", status_, responseText_, headers_, null); - } - else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The corpus is not a valid Scripture corpus.", status_, responseText_, headers_, null); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) @@ -5298,12 +5244,6 @@ public string BaseUrl throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); - } - else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -5331,24 +5271,24 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Add a parallel corpus to a translation engine + /// Remove a corpus from a translation engine (obsolete - use parallel corpora instead) /// /// - /// ## Parameters - ///
* **`sourceCorpusIds`**: The source corpora associated with the parallel corpus - ///
* **`targetCorpusIds`**: The target corpora associated with the parallel corpus + /// Removing a corpus will remove all pretranslations associated with that corpus. /// /// The translation engine id - /// The corpus configuration (see remarks) - /// The added corpus + /// The corpus id + /// If `true`, all files associated with the corpus will be deleted as well (even if they are associated with other corpora). If false, no files will be deleted. + /// The corpus was deleted successfully. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task AddParallelCorpusAsync(string id, TranslationParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + [System.Obsolete] + public virtual async System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, bool? deleteFiles = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusConfig == null) - throw new System.ArgumentNullException("corpusConfig"); + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); var client_ = _httpClient; var disposeClient_ = false; @@ -5356,19 +5296,21 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + request_.Method = new System.Net.Http.HttpMethod("DELETE"); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora"); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append('?'); + if (deleteFiles != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("delete-files")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(deleteFiles, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -5393,20 +5335,9 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 201) - { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; - } - else - if (status_ == 400) + if (status_ == 200) { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + return; } else if (status_ == 401) @@ -5424,7 +5355,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -5454,16 +5385,33 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all parallel corpora for a translation engine + /// Get all pretranslations in a corpus or parallel corpus of a translation engine /// + /// + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **`textId`**: The text id of the source file defined when the corpus was created. + ///
* **`refs`** (a list of strings): A list of references including: + ///
* The references defined in the source file per line, if any. + ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. + ///
* **`translation`**: the text of the pretranslation + ///
+ ///
Pretranslations can be filtered by text id if provided. + ///
Only pretranslations for the most recent successful build of the engine are returned. + /// /// The translation engine id - /// The parallel corpora + /// The corpus id or parallel corpus id + /// The text id (optional) + /// The pretranslations /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + [System.Obsolete] + public virtual async System.Threading.Tasks.Task> GetAllCorpusPretranslationsAsync(string id, string corpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); + var client_ = _httpClient; var disposeClient_ = false; try @@ -5475,10 +5423,18 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora"); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations"); + urlBuilder_.Append('?'); + if (textId != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("text-id")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -5505,7 +5461,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -5516,25 +5472,31 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -5558,26 +5520,34 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Update a parallel corpus with a new set of corpora + /// Get all pretranslations for the specified text in a corpus or parallel corpus of a translation engine /// /// - /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **`textId`**: The text id of the source file defined when the corpus was created. + ///
* **`refs`** (a list of strings): A list of references including: + ///
* The references defined in the source file per line, if any. + ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. + ///
* **`translation`**: the text of the pretranslation + ///
+ ///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id - /// The parallel corpus id - /// The corpus configuration - /// The corpus was updated successfully + /// The corpus id or parallel corpus id + /// The text id + /// The pretranslations /// A server side error occurred. - public virtual async System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, TranslationParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + [System.Obsolete] + public virtual async System.Threading.Tasks.Task> GetCorpusPretranslationsByTextIdAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (parallelCorpusId == null) - throw new System.ArgumentNullException("parallelCorpusId"); + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); - if (corpusConfig == null) - throw new System.ArgumentNullException("corpusConfig"); + if (textId == null) + throw new System.ArgumentNullException("textId"); var client_ = _httpClient; var disposeClient_ = false; @@ -5585,20 +5555,18 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("PATCH"); + request_.Method = new System.Net.Http.HttpMethod("GET"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -5625,7 +5593,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -5633,12 +5601,6 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -5657,6 +5619,12 @@ public string BaseUrl throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -5684,19 +5652,58 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the configuration of a parallel corpus for a translation engine + /// Get a pretranslated Scripture book in USFM format. /// + /// + /// The text that populates the USFM structure can be controlled by the `text-origin` parameter: + ///
* `PreferExisting`: The existing and pretranslated texts are merged into the USFM, preferring existing text. **This is the default**. + ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. + ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). + ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. + ///
+ ///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: + ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. + ///
* `Source`: The source book is used as the template. + ///
* `Target`: The target book is used as the template. + ///
+ ///
The intra-segment USFM markers are handled in the following way: + ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. + ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. + ///
+ ///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. + ///
* `PushToEnd`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers**. + ///
* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. + ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for embeds and style markers**. + ///
+ ///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: + ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. + ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. + ///
+ ///
Only pretranslations for the most recent successful build of the engine are returned. + ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). + /// /// The translation engine id - /// The parallel corpus id - /// The parallel corpus configuration + /// The corpus id or parallel corpus id + /// The text id + /// The source[s] of the data to populate the USFM file with. + /// The source or target book to use as the USFM template. + /// The behavior of paragraph markers. + /// The behavior of embed markers. + /// The behavior of style markers. + /// The normalization behavior of quotes. + /// The book in USFM format /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + [System.Obsolete] + public virtual async System.Threading.Tasks.Task GetCorpusPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (parallelCorpusId == null) - throw new System.ArgumentNullException("parallelCorpusId"); + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); var client_ = _httpClient; var disposeClient_ = false; @@ -5705,15 +5712,44 @@ public string BaseUrl using (var request_ = new System.Net.Http.HttpRequestMessage()) { request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("text/plain")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" + // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}/usfm" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/usfm"); + urlBuilder_.Append('?'); + if (textOrigin != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("text-origin")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textOrigin, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (template != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("template")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(template, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (paragraphMarkerBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("paragraph-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(paragraphMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (embedBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("embed-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(embedBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (styleMarkerBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("style-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(styleMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (quoteNormalizationBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("quotation-mark-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(quoteNormalizationBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -5740,18 +5776,27 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + var result_ = (string)System.Convert.ChangeType(responseData_, typeof(string)); + return result_; + } + else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The specified book does not exist in the source or target corpus.", status_, responseText_, headers_, null); + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The corpus is not a valid Scripture corpus.", status_, responseText_, headers_, null); } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) @@ -5763,7 +5808,13 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -5793,22 +5844,24 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Remove a parallel corpus from a translation engine + /// Add a parallel corpus to a translation engine /// /// - /// Removing a parallel corpus will remove all pretranslations associated with that corpus. + /// ## Parameters + ///
* **`sourceCorpusIds`**: The source corpora associated with the parallel corpus + ///
* **`targetCorpusIds`**: The target corpora associated with the parallel corpus /// /// The translation engine id - /// The parallel corpus id - /// The parallel corpus was deleted successfully. + /// The corpus configuration (see remarks) + /// The added corpus /// A server side error occurred. - public virtual async System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task AddParallelCorpusAsync(string id, TranslationParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (parallelCorpusId == null) - throw new System.ArgumentNullException("parallelCorpusId"); + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); var client_ = _httpClient; var disposeClient_ = false; @@ -5816,15 +5869,19 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("DELETE"); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" + // Operation Path: "translation/engines/{id}/parallel-corpora" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -5849,9 +5906,20 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 200) + if (status_ == 201) { - return; + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); } else if (status_ == 401) @@ -5869,7 +5937,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -5899,32 +5967,16 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all pretranslations in a parallel corpus of a translation engine + /// Get all parallel corpora for a translation engine /// - /// - /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: - ///
* **`textId`**: The text id of the source file defined when the corpus was created. - ///
* **`refs`** (a list of strings): A list of references including: - ///
* The references defined in the source file per line, if any. - ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. - ///
* **`translation`**: the text of the pretranslation - ///
- ///
Pretranslations can be filtered by text id if provided. - ///
Only pretranslations for the most recent successful build of the engine are returned. - /// /// The translation engine id - /// The parallel corpus id - /// The text id (optional) - /// The pretranslations + /// The parallel corpora /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllPretranslationsAsync(string id, string parallelCorpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (parallelCorpusId == null) - throw new System.ArgumentNullException("parallelCorpusId"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -5936,18 +5988,10 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations" + // Operation Path: "translation/engines/{id}/parallel-corpora" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/parallel-corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations"); - urlBuilder_.Append('?'); - if (textId != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("text-id")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + urlBuilder_.Append("/parallel-corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -5974,7 +6018,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -5985,31 +6029,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); - } - else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); } else { @@ -6033,24 +6071,17 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all pretranslations for the specified text in a parallel corpus of a translation engine + /// Update a parallel corpus with a new set of corpora /// /// - /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: - ///
* **`textId`**: The text id of the source file defined when the corpus was created. - ///
* **`refs`** (a list of strings): A list of references including: - ///
* The references defined in the source file per line, if any. - ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. - ///
* **`translation`**: the text of the pretranslation - ///
- ///
Only pretranslations for the most recent successful build of the engine are returned. + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. /// /// The translation engine id /// The parallel corpus id - /// The text id - /// The pretranslations + /// The corpus configuration + /// The corpus was updated successfully /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetPretranslationsByTextIdAsync(string id, string parallelCorpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, TranslationParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -6058,8 +6089,8 @@ public string BaseUrl if (parallelCorpusId == null) throw new System.ArgumentNullException("parallelCorpusId"); - if (textId == null) - throw new System.ArgumentNullException("textId"); + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); var client_ = _httpClient; var disposeClient_ = false; @@ -6067,18 +6098,20 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("PATCH"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations/{textId}" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); urlBuilder_.Append("/parallel-corpora/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -6105,7 +6138,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -6113,6 +6146,12 @@ public string BaseUrl return objectResponse_.Object; } else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -6128,13 +6167,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); - } - else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6164,48 +6197,13 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a pretranslated Scripture book in USFM format. + /// Get the configuration of a parallel corpus for a translation engine /// - /// - /// The text that populates the USFM structure can be controlled by the `text-origin` parameter: - ///
* `PreferExisting`: The existing and pretranslated texts are merged into the USFM, preferring existing text. **This is the default**. - ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. - ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). - ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
- ///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: - ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. - ///
* `Source`: The source book is used as the template. - ///
* `Target`: The target book is used as the template. - ///
- ///
The intra-segment USFM markers are handled in the following way: - ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. - ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
- ///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. - ///
* `PushToEnd`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. - ///
* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language.**. - ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
- ///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: - ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. - ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
- ///
Only pretranslations for the most recent successful build of the engine are returned. - ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). - /// /// The translation engine id /// The parallel corpus id - /// The text id - /// The source[s] of the data to populate the USFM file with. - /// The source or target book to use as the USFM template. - /// The behavior of paragraph markers. - /// The behavior of embed markers. - /// The behavior of style markers. - /// The normalization behavior of quotes. - /// The book in USFM format + /// The parallel corpus configuration /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetPretranslatedUsfmAsync(string id, string parallelCorpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -6213,9 +6211,6 @@ public string BaseUrl if (parallelCorpusId == null) throw new System.ArgumentNullException("parallelCorpusId"); - if (textId == null) - throw new System.ArgumentNullException("textId"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -6223,44 +6218,15 @@ public string BaseUrl using (var request_ = new System.Net.Http.HttpRequestMessage()) { request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("text/plain")); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations/{textId}/usfm" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); urlBuilder_.Append("/parallel-corpora/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/pretranslations/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/usfm"); - urlBuilder_.Append('?'); - if (textOrigin != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("text-origin")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textOrigin, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (template != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("template")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(template, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (paragraphMarkerBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("paragraph-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(paragraphMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (embedBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("embed-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(embedBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (styleMarkerBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("style-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(styleMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - if (quoteNormalizationBehavior != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("quotation-mark-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(quoteNormalizationBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -6287,27 +6253,18 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - var result_ = (string)System.Convert.ChangeType(responseData_, typeof(string)); - return result_; - } - else - if (status_ == 204) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The specified book does not exist in the source or target corpus.", status_, responseText_, headers_, null); - } - else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The parallel corpus does not contain a valid Scripture corpus, or the USFM is invalid.", status_, responseText_, headers_, null); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) @@ -6322,12 +6279,6 @@ public string BaseUrl throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 409) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); - } - else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -6355,31 +6306,38 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all build jobs for a translation engine + /// Remove a parallel corpus from a translation engine /// + /// + /// Removing a parallel corpus will remove all pretranslations associated with that corpus. + /// /// The translation engine id - /// The build jobs + /// The parallel corpus id + /// The parallel corpus was deleted successfully. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + request_.Method = new System.Net.Http.HttpMethod("DELETE"); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/builds" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/builds"); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -6406,12 +6364,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; + return; } else if (status_ == 401) @@ -6429,7 +6382,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6459,52 +6412,31 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Starts a build job for a translation engine. + /// Get all pretranslations in a parallel corpus of a translation engine /// /// - /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). - ///
Specifying a corpus: - ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. - ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
- ///
Filtering by text id or chapter: - ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). - ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). - ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
- ///
Filter - train on all or none - ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively - ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. - ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. - ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
- ///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, - ///
the following text will be pretranslated: - ///
* Text segments that are in the source but do not exist in the target. - ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. - ///
If the engine does not support pretranslation, these fields have no effect. - ///
Pretranslating uses the same filtering as training. - ///
- ///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. - ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. - ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. - ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **`textId`**: The text id of the source file defined when the corpus was created. + ///
* **`refs`** (a list of strings): A list of references including: + ///
* The references defined in the source file per line, if any. + ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. + ///
* **`translation`**: the text of the pretranslation ///
- ///
Note that when using a parallel corpus: - ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. - ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + ///
Pretranslations can be filtered by text id if provided. + ///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id - /// The build config (see remarks) - /// The new build job + /// The parallel corpus id + /// The text id (optional) + /// The pretranslations /// A server side error occurred. - public virtual async System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllPretranslationsAsync(string id, string parallelCorpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (buildConfig == null) - throw new System.ArgumentNullException("buildConfig"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); var client_ = _httpClient; var disposeClient_ = false; @@ -6512,19 +6444,23 @@ public string BaseUrl { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(buildConfig, JsonSerializerSettings); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Method = new System.Net.Http.HttpMethod("GET"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/builds" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/builds"); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations"); + urlBuilder_.Append('?'); + if (textId != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("text-id")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -6549,9 +6485,9 @@ public string BaseUrl ProcessResponse(client_, response_); var status_ = (int)response_.StatusCode; - if (status_ == 201) + if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -6559,12 +6495,6 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The build configuration was invalid.", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -6574,19 +6504,19 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("There is already an active/pending build or a build in the process of being canceled.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6616,30 +6546,33 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a build job + /// Get all pretranslations for the specified text in a parallel corpus of a translation engine /// /// - /// If the `minRevision` is not defined, the current build, at whatever state it is, - ///
will be immediately returned. If `minRevision` is defined, Serval will wait for - ///
up to 40 seconds for the engine to build to the `minRevision` specified, else - ///
will timeout. - ///
A use case is to actively query the state of the current build, where the subsequent - ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. - ///
This method should use request throttling. - ///
Note: Within the returned build, progress is a value between 0 and 1. + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **`textId`**: The text id of the source file defined when the corpus was created. + ///
* **`refs`** (a list of strings): A list of references including: + ///
* The references defined in the source file per line, if any. + ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. + ///
* **`translation`**: the text of the pretranslation + ///
+ ///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id - /// The build job id - /// The minimum revision - /// The build job + /// The parallel corpus id + /// The text id + /// The pretranslations /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetBuildAsync(string id, string buildId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetPretranslationsByTextIdAsync(string id, string parallelCorpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (buildId == null) - throw new System.ArgumentNullException("buildId"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); var client_ = _httpClient; var disposeClient_ = false; @@ -6652,17 +6585,13 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/builds/{buildId}" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations/{textId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/builds/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append('?'); - if (minRevision != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("min-revision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -6689,7 +6618,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -6706,19 +6635,19 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 408) + if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6748,20 +6677,58 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the currently running build job for a translation engine + /// Get a pretranslated Scripture book in USFM format. /// /// - /// See documentation on endpoint /translation/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. - /// - /// The translation engine id - /// The minimum revision - /// The build job + /// The text that populates the USFM structure can be controlled by the `text-origin` parameter: + ///
* `PreferExisting`: The existing and pretranslated texts are merged into the USFM, preferring existing text. **This is the default**. + ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. + ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). + ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. + ///
+ ///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: + ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. + ///
* `Source`: The source book is used as the template. + ///
* `Target`: The target book is used as the template. + ///
+ ///
The intra-segment USFM markers are handled in the following way: + ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. + ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. + ///
+ ///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. + ///
* `PushToEnd`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. + ///
* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language.**. + ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. + ///
+ ///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: + ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. + ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. + ///
+ ///
Only pretranslations for the most recent successful build of the engine are returned. + ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). + /// + /// The translation engine id + /// The parallel corpus id + /// The text id + /// The source[s] of the data to populate the USFM file with. + /// The source or target book to use as the USFM template. + /// The behavior of paragraph markers. + /// The behavior of embed markers. + /// The behavior of style markers. + /// The normalization behavior of quotes. + /// The book in USFM format /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetCurrentBuildAsync(string id, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetPretranslatedUsfmAsync(string id, string parallelCorpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); + var client_ = _httpClient; var disposeClient_ = false; try @@ -6769,18 +6736,42 @@ public string BaseUrl using (var request_ = new System.Net.Http.HttpRequestMessage()) { request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("text/plain")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/current-build" + // Operation Path: "translation/engines/{id}/parallel-corpora/{parallelCorpusId}/pretranslations/{textId}/usfm" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/current-build"); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/usfm"); urlBuilder_.Append('?'); - if (minRevision != null) + if (textOrigin != null) { - urlBuilder_.Append(System.Uri.EscapeDataString("min-revision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + urlBuilder_.Append(System.Uri.EscapeDataString("text-origin")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textOrigin, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (template != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("template")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(template, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (paragraphMarkerBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("paragraph-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(paragraphMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (embedBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("embed-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(embedBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (styleMarkerBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("style-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(styleMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + if (quoteNormalizationBehavior != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("quotation-mark-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(quoteNormalizationBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); } urlBuilder_.Length--; @@ -6809,48 +6800,45 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + var result_ = (string)System.Convert.ChangeType(responseData_, typeof(string)); + return result_; } else if (status_ == 204) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("There is no build currently running.", status_, responseText_, headers_, null); + throw new ServalApiException("The specified book does not exist in the source or target corpus.", status_, responseText_, headers_, null); } else if (status_ == 400) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + throw new ServalApiException("The parallel corpus does not contain a valid Scripture corpus, or the USFM is invalid.", status_, responseText_, headers_, null); } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 408) + if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6880,32 +6868,53 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Cancel the current build job (whether pending or active) for a translation engine + /// Get a build job /// + /// + /// If the `minRevision` is not defined, the current build, at whatever state it is, + ///
will be immediately returned. If `minRevision` is defined, Serval will wait for + ///
up to 40 seconds for the engine to build to the `minRevision` specified, else + ///
will timeout. + ///
A use case is to actively query the state of the current build, where the subsequent + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
This method should use request throttling. + ///
Note: Within the returned build, progress is a value between 0 and 1. + /// /// The translation engine id - /// The build job was cancelled successfully. + /// The build job id + /// The minimum revision + /// The build job /// A server side error occurred. - public virtual async System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetBuildAsync(string id, string buildId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); + if (buildId == null) + throw new System.ArgumentNullException("buildId"); + var client_ = _httpClient; var disposeClient_ = false; try { using (var request_ = new System.Net.Http.HttpRequestMessage()) { - request_.Content = new System.Net.Http.StringContent(string.Empty, System.Text.Encoding.UTF8, "application/json"); - request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Method = new System.Net.Http.HttpMethod("GET"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/current-build/cancel" + // Operation Path: "translation/engines/{id}/builds/{buildId}" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/current-build/cancel"); + urlBuilder_.Append("/builds/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append('?'); + if (minRevision != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("min-revision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -6940,12 +6949,6 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 204) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("There is no active build job.", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -6961,13 +6964,13 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 405) + if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The translation engine does not support cancelling builds.", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -6997,23 +7000,16 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get a link to download the NMT translation model of the last build that was successfully saved. + /// Get the currently running build job for a translation engine /// /// - /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, - ///
then the model from the most recent successful build can be downloaded. - ///
- ///
The endpoint will return a URL that can be used to download the model for up to 1 hour - ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
- ///
The download itself is created by g-zipping together the folder containing the fine tuned model - ///
with all necessary supporting files. This zipped folder is then named by the pattern: - ///
* <engine_id>_<model_revision>.tar.gz + /// See documentation on endpoint /translation/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. /// /// The translation engine id - /// The url to download the model. + /// The minimum revision + /// The build job /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetCurrentBuildAsync(string id, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -7029,10 +7025,16 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/model-download-url" + // Operation Path: "translation/engines/{id}/current-build" urlBuilder_.Append("translation/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/model-download-url"); + urlBuilder_.Append("/current-build"); + urlBuilder_.Append('?'); + if (minRevision != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("min-revision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -7059,7 +7061,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -7067,6 +7069,18 @@ public string BaseUrl return objectResponse_.Object; } else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("There is no build currently running.", status_, responseText_, headers_, null); + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -7082,13 +7096,13 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist or there is no saved model.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else - if (status_ == 405) + if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The translation engine does not support downloading builds.", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -7249,43 +7263,34 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial interface ITranslationEngineTypesClient + public partial interface ITranslationBuildsClient { /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get queue information for a given engine type + /// Get all builds for your translation engines that are created after the specified date. /// - /// A valid engine type: smt-transfer, nmt, or echo - /// Queue information for the specified engine type + /// The date and time in UTC that the builds were created after (optional). + /// The engines /// A server side error occurred. - System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task> GetAllBuildsCreatedAfterAsync(System.DateTimeOffset? createdAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get information regarding a language for a given engine type + /// Get the next build that finished after the specified date and time. + ///
If not build has yet completed after that timestamp, + ///
Serval will wait until a build is finished after that date and time. /// - /// - /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally - ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. - ///
**Base Models available** - ///
* **NLLB-200**: This is the only base NMT translation model currently available. - ///
* The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). - ///
Response format: - ///
* **`engineType`**: See above - ///
* **`isNative`**: Whether the base translation model supports this language without fine-tuning. - ///
* **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). - /// - /// A valid engine type: nmt, echo, or smt-transfer - /// The language to retrieve information on. - /// Language information for the specified engine type + /// The date and time in UTC that the next build should have finished after. + ///
You should use the finished timestamp of the build previously returned when calling this endpoint. + /// The engines /// A server side error occurred. - System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetNextFinishedBuildAsync(System.DateTimeOffset? finishedAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationEngineTypesClient : ITranslationEngineTypesClient + public partial class TranslationBuildsClient : ITranslationBuildsClient { #pragma warning disable 8618 private string _baseUrl; @@ -7296,7 +7301,7 @@ public partial class TranslationEngineTypesClient : ITranslationEngineTypesClien private Newtonsoft.Json.JsonSerializerSettings _instanceSettings; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - public TranslationEngineTypesClient(System.Net.Http.HttpClient httpClient) + public TranslationBuildsClient(System.Net.Http.HttpClient httpClient) #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. { BaseUrl = "/api/v1"; @@ -7334,16 +7339,13 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get queue information for a given engine type + /// Get all builds for your translation engines that are created after the specified date. /// - /// A valid engine type: smt-transfer, nmt, or echo - /// Queue information for the specified engine type + /// The date and time in UTC that the builds were created after (optional). + /// The engines /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllBuildsCreatedAfterAsync(System.DateTimeOffset? createdAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { - if (engineType == null) - throw new System.ArgumentNullException("engineType"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -7355,16 +7357,20 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engine-types/{engineType}/queues" - urlBuilder_.Append("translation/engine-types/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/queues"); - - PrepareRequest(client_, request_, urlBuilder_); - - var url_ = urlBuilder_.ToString(); - request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); - + // Operation Path: "translation/builds" + urlBuilder_.Append("translation/builds"); + urlBuilder_.Append('?'); + if (createdAfter != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("created-after")).Append('=').Append(System.Uri.EscapeDataString(createdAfter.Value.ToString("u", System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + PrepareRequest(client_, request_, url_); var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); @@ -7385,7 +7391,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -7396,19 +7402,19 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -7432,31 +7438,16 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get information regarding a language for a given engine type + /// Get the next build that finished after the specified date and time. + ///
If not build has yet completed after that timestamp, + ///
Serval will wait until a build is finished after that date and time. /// - /// - /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally - ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. - ///
**Base Models available** - ///
* **NLLB-200**: This is the only base NMT translation model currently available. - ///
* The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). - ///
Response format: - ///
* **`engineType`**: See above - ///
* **`isNative`**: Whether the base translation model supports this language without fine-tuning. - ///
* **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). - /// - /// A valid engine type: nmt, echo, or smt-transfer - /// The language to retrieve information on. - /// Language information for the specified engine type + /// The date and time in UTC that the next build should have finished after. + ///
You should use the finished timestamp of the build previously returned when calling this endpoint. + /// The engines /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetNextFinishedBuildAsync(System.DateTimeOffset? finishedAfter = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { - if (engineType == null) - throw new System.ArgumentNullException("engineType"); - - if (language == null) - throw new System.ArgumentNullException("language"); - var client_ = _httpClient; var disposeClient_ = false; try @@ -7468,11 +7459,14 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engine-types/{engineType}/languages/{language}" - urlBuilder_.Append("translation/engine-types/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/languages/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(language, System.Globalization.CultureInfo.InvariantCulture))); + // Operation Path: "translation/builds/next-finished" + urlBuilder_.Append("translation/builds/next-finished"); + urlBuilder_.Append('?'); + if (finishedAfter != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("finished-after")).Append('=').Append(System.Uri.EscapeDataString(finishedAfter.Value.ToString("u", System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; PrepareRequest(client_, request_, urlBuilder_); @@ -7499,7 +7493,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -7510,19 +7504,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); } else - if (status_ == 405) + if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -11022,6 +11022,35 @@ public enum FileFormat } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class LanguageInfo + { + + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isNative", Required = Newtonsoft.Json.Required.Always)] + public bool IsNative { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("internalCode", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? InternalCode { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Queue + { + + [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] + public int Size { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationBuild { @@ -11367,29 +11396,56 @@ public partial class TranslationEngineConfig } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationEngineUpdateConfig + public partial class ModelDownloadUrl { - [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? SourceLanguage { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TargetLanguage { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] + public int ModelRevision { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("expiresAt", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public System.DateTimeOffset ExpiresAt { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationResult + public partial class WordGraph { - [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Translation { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("initialStateScore", Required = Newtonsoft.Json.Required.Always)] + public float InitialStateScore { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("finalStates", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList FinalStates { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("arcs", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Arcs { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class WordGraphArc + { + + [Newtonsoft.Json.JsonProperty("prevState", Required = Newtonsoft.Json.Required.Always)] + public int PrevState { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("nextState", Required = Newtonsoft.Json.Required.Always)] + public int NextState { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Always)] + public double Score { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); @@ -11398,17 +11454,34 @@ public partial class TranslationResult [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentStart { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentEnd { get; set; } = default!; [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("phrases", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Phrases { get; set; } = new System.Collections.ObjectModel.Collection(); + public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AlignedWordPair + { + + [Newtonsoft.Json.JsonProperty("sourceIndex", Required = Newtonsoft.Json.Required.Always)] + public int SourceIndex { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetIndex", Required = Newtonsoft.Json.Required.Always)] + public int TargetIndex { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? Score { get; set; } = default!; } @@ -11428,68 +11501,118 @@ public enum TranslationSource } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AlignedWordPair + public partial class TranslationBuildConfig { - [Newtonsoft.Json.JsonProperty("sourceIndex", Required = Newtonsoft.Json.Required.Always)] - public int SourceIndex { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetIndex", Required = Newtonsoft.Json.Required.Always)] - public int TargetIndex { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public double? Score { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Phrase + public partial class TrainingCorpusConfig { - [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentStart { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public string? CorpusId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentEnd { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetSegmentCut", Required = Newtonsoft.Json.Required.Always)] - public int TargetSegmentCut { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordGraph + public partial class ParallelCorpusFilterConfig { - [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string CorpusId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("initialStateScore", Required = Newtonsoft.Json.Required.Always)] - public float InitialStateScore { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("finalStates", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList FinalStates { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("arcs", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Arcs { get; set; } = new System.Collections.ObjectModel.Collection(); + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class PretranslateCorpusConfig + { + + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public string? CorpusId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [System.Obsolete] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordGraphArc + public partial class SegmentPair { - [Newtonsoft.Json.JsonProperty("prevState", Required = Newtonsoft.Json.Required.Always)] - public int PrevState { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string SourceSegment { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("nextState", Required = Newtonsoft.Json.Required.Always)] - public int NextState { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetSegment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetSegment { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Always)] - public double Score { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sentenceStart", Required = Newtonsoft.Json.Required.Always)] + public bool SentenceStart { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationResult + { + + [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Translation { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] @@ -11499,36 +11622,32 @@ public partial class WordGraphArc [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentStart { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentEnd { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("phrases", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + public System.Collections.Generic.IList Phrases { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class SegmentPair + public partial class Phrase { - [Newtonsoft.Json.JsonProperty("sourceSegment", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceSegment { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentStart { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetSegment", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetSegment { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentEnd { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sentenceStart", Required = Newtonsoft.Json.Required.Always)] - public bool SentenceStart { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetSegmentCut", Required = Newtonsoft.Json.Required.Always)] + public int TargetSegmentCut { get; set; } = default!; } @@ -11783,133 +11902,14 @@ public partial class TranslationParallelCorpusUpdateConfig } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationBuildConfig - { - - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public object? Options { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TrainingCorpusConfig - { - - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public string? CorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public string? ScriptureRange { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ParallelCorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ParallelCorpusFilterConfig - { - - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string CorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class PretranslateCorpusConfig - { - - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public string? CorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public string? ScriptureRange { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ParallelCorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ModelDownloadUrl - { - - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] - public int ModelRevision { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("expiresAt", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public System.DateTimeOffset ExpiresAt { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Queue - { - - [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] - public int Size { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string EngineType { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class LanguageInfo + public partial class TranslationEngineUpdateConfig { - [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string EngineType { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("isNative", Required = Newtonsoft.Json.Required.Always)] - public bool IsNative { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? SourceLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("internalCode", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? InternalCode { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TargetLanguage { get; set; } = default!; } diff --git a/src/Serval/src/Serval.DataFiles.Contracts/CorpusContract.cs b/src/Serval/src/Serval.DataFiles.Contracts/CorpusContract.cs new file mode 100644 index 000000000..d4fd10e10 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/CorpusContract.cs @@ -0,0 +1,8 @@ +namespace Serval.DataFiles.Contracts; + +public record CorpusContract( + string CorpusId, + string Language, + string? Name, + IReadOnlyList Files +); diff --git a/src/Serval/src/Serval.DataFiles.Contracts/CorpusDataFileContract.cs b/src/Serval/src/Serval.DataFiles.Contracts/CorpusDataFileContract.cs new file mode 100644 index 000000000..4a5de1202 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/CorpusDataFileContract.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record CorpusDataFileContract(DataFileContract File, string TextId); diff --git a/src/Serval/src/Serval.DataFiles.Contracts/CorpusUpdated.cs b/src/Serval/src/Serval.DataFiles.Contracts/CorpusUpdated.cs new file mode 100644 index 000000000..43984517e --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/CorpusUpdated.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record CorpusUpdated(string CorpusId, IReadOnlyList Files) : IEvent; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/DataFileContract.cs b/src/Serval/src/Serval.DataFiles.Contracts/DataFileContract.cs new file mode 100644 index 000000000..3774c259c --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/DataFileContract.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record DataFileContract(string DataFileId, string Name, string Filename, FileFormat Format); diff --git a/src/Serval/src/Serval.DataFiles.Contracts/DataFileDeleted.cs b/src/Serval/src/Serval.DataFiles.Contracts/DataFileDeleted.cs new file mode 100644 index 000000000..9dea5b1ba --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/DataFileDeleted.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record DataFileDeleted(string DataFileId) : IEvent; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/DataFileUpdated.cs b/src/Serval/src/Serval.DataFiles.Contracts/DataFileUpdated.cs new file mode 100644 index 000000000..e0cef3ec6 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/DataFileUpdated.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record DataFileUpdated(string DataFileId, string Filename) : IEvent; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/DeleteDataFile.cs b/src/Serval/src/Serval.DataFiles.Contracts/DeleteDataFile.cs new file mode 100644 index 000000000..79ab64780 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/DeleteDataFile.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record DeleteDataFile(string DataFileId) : IRequest; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/GetCorpus.cs b/src/Serval/src/Serval.DataFiles.Contracts/GetCorpus.cs new file mode 100644 index 000000000..611fa88ab --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/GetCorpus.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record GetCorpus(string CorpusId, string Owner) : IRequest; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/GetCorpusResponse.cs b/src/Serval/src/Serval.DataFiles.Contracts/GetCorpusResponse.cs new file mode 100644 index 000000000..195acd6bd --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/GetCorpusResponse.cs @@ -0,0 +1,8 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Serval.DataFiles.Contracts; + +public record GetCorpusResponse( + [property: MemberNotNullWhen(true, nameof(GetCorpusResponse.Corpus))] bool IsFound, + CorpusContract? Corpus = null +); diff --git a/src/Serval/src/Serval.DataFiles.Contracts/GetDataFile.cs b/src/Serval/src/Serval.DataFiles.Contracts/GetDataFile.cs new file mode 100644 index 000000000..d553996fd --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/GetDataFile.cs @@ -0,0 +1,3 @@ +namespace Serval.DataFiles.Contracts; + +public record GetDataFile(string DataFileId, string Owner) : IRequest; diff --git a/src/Serval/src/Serval.DataFiles.Contracts/GetDataFileResponse.cs b/src/Serval/src/Serval.DataFiles.Contracts/GetDataFileResponse.cs new file mode 100644 index 000000000..b7df07be8 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/GetDataFileResponse.cs @@ -0,0 +1,8 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Serval.DataFiles.Contracts; + +public record GetDataFileResponse( + [property: MemberNotNullWhen(true, nameof(File))] bool IsFound, + DataFileContract? File = null +); diff --git a/src/Serval/src/Serval.DataFiles.Contracts/Serval.DataFiles.Contracts.csproj b/src/Serval/src/Serval.DataFiles.Contracts/Serval.DataFiles.Contracts.csproj new file mode 100644 index 000000000..c0c82186c --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/Serval.DataFiles.Contracts.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + + + + + + + + + diff --git a/src/Serval/src/Serval.DataFiles.Contracts/Usings.cs b/src/Serval/src/Serval.DataFiles.Contracts/Usings.cs new file mode 100644 index 000000000..e713c55cd --- /dev/null +++ b/src/Serval/src/Serval.DataFiles.Contracts/Usings.cs @@ -0,0 +1 @@ +global using Serval.Shared.Contracts; diff --git a/src/Serval/src/Serval.DataFiles/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.DataFiles/Configuration/IMediatorRegistrationConfiguratorExtensions.cs deleted file mode 100644 index ad6bdd57f..000000000 --- a/src/Serval/src/Serval.DataFiles/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMediatorRegistrationConfiguratorExtensions -{ - public static IMediatorRegistrationConfigurator AddDataFilesConsumers( - this IMediatorRegistrationConfigurator configurator - ) - { - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.DataFiles/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.DataFiles/Configuration/IMemoryDataAccessConfiguratorExtensions.cs deleted file mode 100644 index 0b6195572..000000000 --- a/src/Serval/src/Serval.DataFiles/Configuration/IMemoryDataAccessConfiguratorExtensions.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMemoryDataAccessConfiguratorExtensions -{ - public static IMemoryDataAccessConfigurator AddDataFilesRepositories( - this IMemoryDataAccessConfigurator configurator - ) - { - configurator.AddRepository(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.DataFiles/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.DataFiles/Configuration/IServalBuilderExtensions.cs deleted file mode 100644 index 11af65e19..000000000 --- a/src/Serval/src/Serval.DataFiles/Configuration/IServalBuilderExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServalBuilderExtensions -{ - public static IServalBuilder AddDataFiles(this IServalBuilder builder) - { - builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); - - builder.Services.AddScoped(); - builder.Services.AddHostedService(); - - builder.Services.AddScoped(); - - return builder; - } -} diff --git a/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.DataFiles/Configuration/IServalConfiguratorExtensions.cs similarity index 58% rename from src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs rename to src/Serval/src/Serval.DataFiles/Configuration/IServalConfiguratorExtensions.cs index f3fc035de..b5c1b3162 100644 --- a/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.DataFiles/Configuration/IServalConfiguratorExtensions.cs @@ -1,12 +1,24 @@ -using MongoDB.Driver; +namespace Microsoft.Extensions.DependencyInjection; -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMongoDataAccessConfiguratorExtensions +public static class IServalConfiguratorExtensions { - public static IMongoDataAccessConfigurator AddDataFilesRepositories(this IMongoDataAccessConfigurator configurator) + public static IServalConfigurator AddDataFiles(this IServalConfigurator configurator) + { + configurator.Services.AddScoped(); + configurator.Services.AddHostedService(); + + configurator.Services.AddScoped(); + + configurator.AddDataFilesDataAccess(); + + configurator.AddHandlers(Assembly.GetExecutingAssembly()); + + return configurator; + } + + public static IServalConfigurator AddDataFilesDataAccess(this IServalConfigurator configurator) { - configurator.AddRepository( + configurator.DataAccess.AddRepository( "data_files.files", init: [ @@ -17,7 +29,7 @@ public static IMongoDataAccessConfigurator AddDataFilesRepositories(this IMongoD ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "data_files.deleted_files", init: [ @@ -27,7 +39,7 @@ public static IMongoDataAccessConfigurator AddDataFilesRepositories(this IMongoD ), ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "corpora.corpus", init: [ diff --git a/src/Serval/src/Serval.DataFiles/Consumers/DataFileDeletedConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/DataFileDeletedConsumer.cs deleted file mode 100644 index a32b4fc54..000000000 --- a/src/Serval/src/Serval.DataFiles/Consumers/DataFileDeletedConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.DataFiles.Consumers; - -public class DataFileDeletedConsumer(ICorpusService corpusService) : IConsumer -{ - private readonly ICorpusService _corpusService = corpusService; - - public async Task Consume(ConsumeContext context) - { - await _corpusService.DeleteAllCorpusFilesAsync(context.Message.DataFileId, context.CancellationToken); - } -} diff --git a/src/Serval/src/Serval.DataFiles/Consumers/DeleteDataFileConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/DeleteDataFileConsumer.cs deleted file mode 100644 index c14a9fc07..000000000 --- a/src/Serval/src/Serval.DataFiles/Consumers/DeleteDataFileConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.DataFiles.Consumers; - -public class DeleteDataFileConsumer(IDataFileService dataFileService) : IConsumer -{ - private readonly IDataFileService _dataFileService = dataFileService; - - public async Task Consume(ConsumeContext context) - { - await _dataFileService.DeleteAsync(context.Message.DataFileId, context.CancellationToken); - } -} diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs deleted file mode 100644 index 62520ca2d..000000000 --- a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs +++ /dev/null @@ -1,56 +0,0 @@ -namespace Serval.DataFiles.Consumers; - -public class GetCorpusConsumer(ICorpusService corpusService, IDataFileService dataFileService) : IConsumer -{ - private readonly ICorpusService _corpusService = corpusService; - private readonly IDataFileService _dataFileService = dataFileService; - - public async Task Consume(ConsumeContext context) - { - try - { - Corpus corpus = await _corpusService.GetAsync( - context.Message.CorpusId, - context.Message.Owner, - context.CancellationToken - ); - IEnumerable corpusFileIds = corpus.Files.Select(f => f.FileRef); - IDictionary corpusDataFilesDict = ( - await _dataFileService.GetAllAsync(corpusFileIds, context.CancellationToken) - ).ToDictionary(f => f.Id); - - await context.RespondAsync( - new CorpusResult - { - CorpusId = corpus.Id, - Name = corpus.Name, - Language = corpus.Language, - Files = corpus - .Files.Select(f => new CorpusFileResult - { - TextId = f.TextId ?? corpusDataFilesDict[f.FileRef].Name, - File = Map(corpusDataFilesDict[f.FileRef]), - }) - .ToList(), - } - ); - } - catch (EntityNotFoundException) - { - await context.RespondAsync( - new CorpusNotFound { CorpusId = context.Message.CorpusId, Owner = context.Message.Owner } - ); - } - } - - private static DataFileResult Map(DataFile dataFile) - { - return new DataFileResult - { - DataFileId = dataFile.Id, - Name = dataFile.Name, - Filename = dataFile.Filename, - Format = dataFile.Format, - }; - } -} diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs deleted file mode 100644 index 079c762d5..000000000 --- a/src/Serval/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace Serval.DataFiles.Consumers; - -public class GetDataFileConsumer(IDataFileService dataFileService) : IConsumer -{ - private readonly IDataFileService _dataFileService = dataFileService; - - public async Task Consume(ConsumeContext context) - { - try - { - DataFile dataFile = await _dataFileService.GetAsync( - context.Message.DataFileId, - context.Message.Owner, - context.CancellationToken - ); - await context.RespondAsync( - new DataFileResult - { - DataFileId = dataFile.Id, - Name = dataFile.Name, - Filename = dataFile.Filename, - Format = dataFile.Format, - } - ); - } - catch (EntityNotFoundException) - { - await context.RespondAsync( - new DataFileNotFound { DataFileId = context.Message.DataFileId, Owner = context.Message.Owner } - ); - } - } -} diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs index 7a69e1824..c9656b6dc 100644 --- a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs +++ b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs @@ -76,7 +76,6 @@ public async Task> GetAsync([NotNull] string id, Cancell [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> CreateAsync( [FromBody] CorpusConfigDto corpusConfig, - [FromServices] IRequestClient getDataFileClient, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken ) @@ -169,18 +168,18 @@ private async Task MapAsync(CorpusConfigDto corpusConfig, string id, Can }; } - private async Task> MapAsync( + private async Task> MapAsync( IReadOnlyList files, CancellationToken cancellationToken ) { - var dataFiles = new List(); + var dataFiles = new List(); foreach (CorpusFileConfigDto file in files) { DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken); if (dataFile == null) throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist."); - dataFiles.Add(new Models.CorpusFile { FileRef = file.FileId, TextId = file.TextId }); + dataFiles.Add(new CorpusFile { FileRef = file.FileId, TextId = file.TextId }); } return dataFiles; } @@ -198,7 +197,7 @@ private CorpusDto Map(Corpus source) }; } - private CorpusFileDto Map(Models.CorpusFile source) + private CorpusFileDto Map(CorpusFile source) { return new CorpusFileDto { diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusConfigDto.cs b/src/Serval/src/Serval.DataFiles/Dtos/CorpusConfigDto.cs similarity index 84% rename from src/Serval/src/Serval.DataFiles/Contracts/CorpusConfigDto.cs rename to src/Serval/src/Serval.DataFiles/Dtos/CorpusConfigDto.cs index cb744e417..f31a06e96 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusConfigDto.cs +++ b/src/Serval/src/Serval.DataFiles/Dtos/CorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.DataFiles.Contracts; +namespace Serval.DataFiles.Dtos; public record CorpusConfigDto { diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusDto.cs b/src/Serval/src/Serval.DataFiles/Dtos/CorpusDto.cs similarity index 89% rename from src/Serval/src/Serval.DataFiles/Contracts/CorpusDto.cs rename to src/Serval/src/Serval.DataFiles/Dtos/CorpusDto.cs index b7446fa3b..483d2b5a5 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusDto.cs +++ b/src/Serval/src/Serval.DataFiles/Dtos/CorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.DataFiles.Contracts; +namespace Serval.DataFiles.Dtos; public record CorpusDto { diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileConfigDto.cs b/src/Serval/src/Serval.DataFiles/Dtos/CorpusFileConfigDto.cs similarity index 77% rename from src/Serval/src/Serval.DataFiles/Contracts/CorpusFileConfigDto.cs rename to src/Serval/src/Serval.DataFiles/Dtos/CorpusFileConfigDto.cs index 746a9686c..d47c1b5bd 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileConfigDto.cs +++ b/src/Serval/src/Serval.DataFiles/Dtos/CorpusFileConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.DataFiles.Contracts; +namespace Serval.DataFiles.Dtos; public record CorpusFileConfigDto { diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs b/src/Serval/src/Serval.DataFiles/Dtos/CorpusFileDto.cs similarity index 77% rename from src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs rename to src/Serval/src/Serval.DataFiles/Dtos/CorpusFileDto.cs index 9fda28fd5..3a188210f 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs +++ b/src/Serval/src/Serval.DataFiles/Dtos/CorpusFileDto.cs @@ -1,4 +1,4 @@ -namespace Serval.DataFiles.Contracts; +namespace Serval.DataFiles.Dtos; public record CorpusFileDto { diff --git a/src/Serval/src/Serval.DataFiles/Contracts/DataFileDto.cs b/src/Serval/src/Serval.DataFiles/Dtos/DataFileDto.cs similarity index 86% rename from src/Serval/src/Serval.DataFiles/Contracts/DataFileDto.cs rename to src/Serval/src/Serval.DataFiles/Dtos/DataFileDto.cs index 3d6a5a710..741d20d26 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/DataFileDto.cs +++ b/src/Serval/src/Serval.DataFiles/Dtos/DataFileDto.cs @@ -1,4 +1,4 @@ -namespace Serval.DataFiles.Contracts; +namespace Serval.DataFiles.Dtos; public record DataFileDto { diff --git a/src/Serval/src/Serval.DataFiles/Handlers/DataFileDeletedHandler.cs b/src/Serval/src/Serval.DataFiles/Handlers/DataFileDeletedHandler.cs new file mode 100644 index 000000000..5c56beeef --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Handlers/DataFileDeletedHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.DataFiles.Handlers; + +public class DataFileDeletedHandler(ICorpusService corpusService) : IEventHandler +{ + public Task HandleAsync(DataFileDeleted evt, CancellationToken cancellationToken) + { + return corpusService.DeleteAllCorpusFilesAsync(evt.DataFileId, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.DataFiles/Handlers/DeleteDataFileHandler.cs b/src/Serval/src/Serval.DataFiles/Handlers/DeleteDataFileHandler.cs new file mode 100644 index 000000000..a6fc9317b --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Handlers/DeleteDataFileHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.DataFiles.Handlers; + +public class DeleteDataFileHandler(IDataFileService dataFileService) : IRequestHandler +{ + public Task HandleAsync(DeleteDataFile request, CancellationToken cancellationToken) + { + return dataFileService.DeleteAsync(request.DataFileId, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.DataFiles/Handlers/GetCorpusHandler.cs b/src/Serval/src/Serval.DataFiles/Handlers/GetCorpusHandler.cs new file mode 100644 index 000000000..a2d9a5ec5 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Handlers/GetCorpusHandler.cs @@ -0,0 +1,40 @@ +namespace Serval.DataFiles.Handlers; + +public class GetCorpusHandler(ICorpusService corpusService, IDataFileService dataFileService) + : IRequestHandler +{ + public async Task HandleAsync(GetCorpus request, CancellationToken cancellationToken) + { + try + { + Corpus corpus = await corpusService.GetAsync(request.CorpusId, request.Owner, cancellationToken); + IEnumerable corpusFileIds = corpus.Files.Select(f => f.FileRef); + var corpusDataFilesDict = ( + await dataFileService.GetAllAsync(corpusFileIds, cancellationToken) + ).ToDictionary(f => f.Id); + return new GetCorpusResponse( + IsFound: true, + new CorpusContract( + corpus.Id, + corpus.Language, + corpus.Name, + [ + .. corpus.Files.Select(f => new CorpusDataFileContract( + File: Map(corpusDataFilesDict[f.FileRef]), + f.TextId ?? corpusDataFilesDict[f.FileRef].Name + )), + ] + ) + ); + } + catch (EntityNotFoundException) + { + return new GetCorpusResponse(IsFound: false); + } + } + + private static DataFileContract Map(DataFile dataFile) + { + return new DataFileContract(dataFile.Id, dataFile.Name, dataFile.Filename, dataFile.Format); + } +} diff --git a/src/Serval/src/Serval.DataFiles/Handlers/GetDataFileHandler.cs b/src/Serval/src/Serval.DataFiles/Handlers/GetDataFileHandler.cs new file mode 100644 index 000000000..e2f3078de --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Handlers/GetDataFileHandler.cs @@ -0,0 +1,17 @@ +namespace Serval.DataFiles.Handlers; + +public class GetDataFileHandler(IDataFileService dataFileService) : IRequestHandler +{ + public async Task HandleAsync(GetDataFile request, CancellationToken cancellationToken) + { + try + { + DataFile dataFile = await dataFileService.GetAsync(request.DataFileId, request.Owner, cancellationToken); + return new(IsFound: true, new(dataFile.Id, dataFile.Name, dataFile.Filename, dataFile.Format)); + } + catch (EntityNotFoundException) + { + return new(IsFound: false); + } + } +} diff --git a/src/Serval/src/Serval.DataFiles/Serval.DataFiles.csproj b/src/Serval/src/Serval.DataFiles/Serval.DataFiles.csproj index 229966c2c..0adf4a657 100644 --- a/src/Serval/src/Serval.DataFiles/Serval.DataFiles.csproj +++ b/src/Serval/src/Serval.DataFiles/Serval.DataFiles.csproj @@ -10,14 +10,16 @@ $(NoWarn);CS1591;CS1573 + + - + diff --git a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs index 9539151ac..6e379ac2c 100644 --- a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs @@ -4,12 +4,12 @@ public class CorpusService( IRepository corpora, IRepository dataFiles, IDataAccessContext dataAccessContext, - IScopedMediator mediator + IEventRouter eventRouter ) : OwnedEntityServiceBase(corpora), ICorpusService { private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IRepository _dataFiles = dataFiles; - private readonly IScopedMediator _mediator = mediator; + private readonly IEventRouter _eventRouter = eventRouter; public async Task GetAsync(string id, string owner, CancellationToken cancellationToken = default) { @@ -21,7 +21,7 @@ public async Task GetAsync(string id, string owner, CancellationToken ca public async Task UpdateAsync( string id, - IReadOnlyList files, + IReadOnlyList files, CancellationToken cancellationToken = default ) { @@ -39,18 +39,16 @@ public async Task UpdateAsync( IDictionary corpusDataFilesDict = ( await _dataFiles.GetAllAsync(f => corpusFileIds.Contains(f.Id), ct) ).ToDictionary(f => f.Id); - await _mediator.Publish( - new CorpusUpdated - { - CorpusId = corpus.Id, - Files = corpus - .Files.Select(f => new CorpusFileResult - { - TextId = f.TextId ?? corpusDataFilesDict[f.FileRef].Name, - File = Map(corpusDataFilesDict[f.FileRef]), - }) - .ToList(), - }, + await _eventRouter.PublishAsync( + new CorpusUpdated( + corpus.Id, + [ + .. corpus.Files.Select(f => new CorpusDataFileContract( + File: Map(corpusDataFilesDict[f.FileRef]), + f.TextId ?? corpusDataFilesDict[f.FileRef].Name + )), + ] + ), ct ); return corpus; @@ -71,14 +69,8 @@ public Task DeleteAllCorpusFilesAsync(string fileId, CancellationToken cancellat ); } - private static DataFileResult Map(DataFile dataFile) + private static DataFileContract Map(DataFile dataFile) { - return new DataFileResult - { - DataFileId = dataFile.Id, - Name = dataFile.Name, - Filename = dataFile.Filename, - Format = dataFile.Format, - }; + return new DataFileContract(dataFile.Id, dataFile.Name, dataFile.Filename, dataFile.Format); } } diff --git a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs index b9f2eed2f..0196f5ed9 100644 --- a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs @@ -4,7 +4,7 @@ public class DataFileService : OwnedEntityServiceBase, IDataFileServic { private readonly IOptionsMonitor _options; private readonly IDataAccessContext _dataAccessContext; - private readonly IScopedMediator _mediator; + private readonly IEventRouter _eventRouter; private readonly IRepository _deletedFiles; private readonly IFileSystem _fileSystem; @@ -12,7 +12,7 @@ public DataFileService( IRepository dataFiles, IDataAccessContext dataAccessContext, IOptionsMonitor options, - IScopedMediator mediator, + IEventRouter eventRouter, IRepository deletedFiles, IFileSystem fileSystem ) @@ -20,7 +20,7 @@ IFileSystem fileSystem { _dataAccessContext = dataAccessContext; _options = options; - _mediator = mediator; + _eventRouter = eventRouter; _deletedFiles = deletedFiles; _fileSystem = fileSystem; _fileSystem.CreateDirectory(_options.CurrentValue.FilesDirectory); @@ -85,7 +85,7 @@ await _deletedFiles.InsertAsync( new DeletedFile { Filename = originalDataFile.Filename, DeletedAt = DateTime.UtcNow }, cancellationToken: ct ); - await _mediator.Publish(new DataFileUpdated { DataFileId = id, Filename = filename }, ct); + await _eventRouter.PublishAsync(new DataFileUpdated(id, filename), ct); }, cancellationToken: cancellationToken ); @@ -116,7 +116,8 @@ await _deletedFiles.InsertAsync( new DeletedFile { Filename = dataFile.Filename, DeletedAt = DateTime.UtcNow }, ct ); - await _mediator.Publish(new DataFileDeleted { DataFileId = id }, ct); + + await _eventRouter.PublishAsync(new DataFileDeleted(id), ct); }, cancellationToken: cancellationToken ); diff --git a/src/Serval/src/Serval.DataFiles/Usings.cs b/src/Serval/src/Serval.DataFiles/Usings.cs index b4765e0e7..291d07e8f 100644 --- a/src/Serval/src/Serval.DataFiles/Usings.cs +++ b/src/Serval/src/Serval.DataFiles/Usings.cs @@ -1,9 +1,8 @@ global using System.Diagnostics; global using System.Diagnostics.CodeAnalysis; +global using System.Reflection; global using Asp.Versioning; global using Cronos; -global using MassTransit; -global using MassTransit.Mediator; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; @@ -13,16 +12,17 @@ global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; +global using MongoDB.Driver; global using NSwag.Annotations; -global using Serval.DataFiles.Consumers; global using Serval.DataFiles.Contracts; +global using Serval.DataFiles.Dtos; global using Serval.DataFiles.Models; global using Serval.DataFiles.Services; global using Serval.Shared.Configuration; global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; +global using Serval.Shared.Dtos; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; global using SIL.DataAccess; -global using SIL.ServiceToolkit.Services; diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto b/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto deleted file mode 100644 index 68c6974fb..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto +++ /dev/null @@ -1,21 +0,0 @@ -syntax = "proto3"; - -package serval.health.v1; - -import "google/protobuf/empty.proto"; - -service HealthApi { - rpc HealthCheck(google.protobuf.Empty) returns (HealthCheckResponse); -} - -message HealthCheckResponse { - HealthCheckStatus status = 1; - map data = 2; - optional string error = 3; -} - -enum HealthCheckStatus { - UNHEALTHY = 0; - DEGRADED = 1; - HEALTHY = 2; -} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/common.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/common.proto deleted file mode 100644 index 4f9dabb38..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/common.proto +++ /dev/null @@ -1,8 +0,0 @@ -syntax = "proto3"; - -package serval.translation.v1; - -message AlignedWordPair { - int32 source_index = 1; - int32 target_index = 2; -} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto deleted file mode 100644 index 2b95c93f8..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto +++ /dev/null @@ -1,195 +0,0 @@ -syntax = "proto3"; - -package serval.translation.v1; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; -import "serval/translation/v1/common.proto"; - -service TranslationEngineApi { - rpc Create(CreateRequest) returns (google.protobuf.Empty); - rpc Delete(DeleteRequest) returns (google.protobuf.Empty); - rpc Update(UpdateRequest) returns (google.protobuf.Empty); - rpc Translate(TranslateRequest) returns (TranslateResponse); - rpc GetWordGraph(GetWordGraphRequest) returns (GetWordGraphResponse); - rpc TrainSegmentPair(TrainSegmentPairRequest) returns (google.protobuf.Empty); - rpc StartBuild(StartBuildRequest) returns (google.protobuf.Empty); - rpc CancelBuild(CancelBuildRequest) returns (CancelBuildResponse); - rpc GetModelDownloadUrl(GetModelDownloadUrlRequest) returns (GetModelDownloadUrlResponse); - rpc GetQueueSize(GetQueueSizeRequest) returns (GetQueueSizeResponse); - rpc GetLanguageInfo(GetLanguageInfoRequest) returns (GetLanguageInfoResponse); -} - -message CreateRequest { - string engine_type = 1; - string engine_id = 2; - optional string engine_name = 3; - string source_language = 4; - string target_language = 5; - optional bool is_model_persisted = 6; -} - -message DeleteRequest { - string engine_type = 1; - string engine_id = 2; -} - -message UpdateRequest { - string engine_type = 1; - string engine_id = 2; - optional string source_language = 3; - optional string target_language = 4; -} - -message TranslateRequest { - string engine_type = 1; - string engine_id = 2; - string segment = 3; - int32 n = 4; -} - -message TranslateResponse { - repeated TranslationResult results = 1; -} - -message GetWordGraphRequest { - string engine_type = 1; - string engine_id = 2; - string segment = 3; -} - -message GetWordGraphResponse { - WordGraph word_graph = 1; -} - -message TrainSegmentPairRequest { - string engine_type = 1; - string engine_id = 2; - string source_segment = 3; - string target_segment = 4; - bool sentence_start = 5; -} - -message StartBuildRequest { - string engine_type = 1; - string engine_id = 2; - string build_id = 3; - optional string options = 4; - repeated ParallelCorpus corpora = 5; -} - -message CancelBuildRequest { - string engine_type = 1; - string engine_id = 2; -} - -message CancelBuildResponse { - string build_id = 1; -} - -message GetModelDownloadUrlRequest { - string engine_type = 1; - string engine_id = 2; -} - -message GetModelDownloadUrlResponse { - string url = 1; - int32 model_revision = 2; - google.protobuf.Timestamp expires_at = 3; -} - -message GetQueueSizeRequest { - string engine_type = 1; -} - -message GetQueueSizeResponse { - int32 size = 1; -} - -message GetLanguageInfoRequest { - string engine_type = 1; - string language = 2; -} - -message GetLanguageInfoResponse { - bool is_native = 3; - optional string internal_code = 1; -} - -message Phrase { - int32 source_segment_start = 1; - int32 source_segment_end = 2; - int32 target_segment_cut = 3; -} - -message TranslationSources { - repeated TranslationSource values = 1; -} - -message TranslationResult { - string translation = 1; - repeated string source_tokens = 2; - repeated string target_tokens = 3; - repeated double confidences = 4; - repeated TranslationSources sources = 5; - repeated AlignedWordPair alignment = 6; - repeated Phrase phrases = 7; -} - -message WordGraphArc { - int32 prev_state = 1; - int32 next_state = 2; - double score = 3; - repeated string target_tokens = 4; - repeated double confidences = 5; - int32 source_segment_start = 6; - int32 source_segment_end = 7; - repeated AlignedWordPair alignment = 8; - repeated TranslationSources sources = 9; -} - -message WordGraph { - repeated string source_tokens = 1; - double initial_state_score = 2; - repeated int32 final_states = 3; - repeated WordGraphArc arcs = 4; -} - -message ParallelCorpus { - string id = 1; - repeated MonolingualCorpus source_corpora = 2; - repeated MonolingualCorpus target_corpora = 3; -} - -message MonolingualCorpus { - string id = 1; - string language = 2; - bool train_on_all = 3; - bool pretranslate_all = 4; - map train_on_chapters = 5; - map pretranslate_chapters = 6; - repeated string train_on_text_ids = 7; - repeated string pretranslate_text_ids = 8; - repeated CorpusFile files = 9; -} - -message ScriptureChapters { - repeated int32 chapters = 1; -} - -message CorpusFile { - string location = 1; - FileFormat format = 2; - string text_id = 3; -} - -enum FileFormat { - FILE_FORMAT_TEXT = 0; - FILE_FORMAT_PARATEXT = 1; -} - -enum TranslationSource { - TRANSLATION_SOURCE_PRIMARY = 0; - TRANSLATION_SOURCE_SECONDARY = 1; - TRANSLATION_SOURCE_HUMAN = 2; -} \ No newline at end of file diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto deleted file mode 100644 index ed2d40433..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto +++ /dev/null @@ -1,108 +0,0 @@ -syntax = "proto3"; - -package serval.translation.v1; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; -import "serval/translation/v1/common.proto"; - - -service TranslationPlatformApi { - rpc UpdateBuildStatus(UpdateBuildStatusRequest) returns (google.protobuf.Empty); - rpc BuildStarted(BuildStartedRequest) returns (google.protobuf.Empty); - rpc BuildCompleted(BuildCompletedRequest) returns (google.protobuf.Empty); - rpc BuildCanceled(BuildCanceledRequest) returns (google.protobuf.Empty); - rpc BuildFaulted(BuildFaultedRequest) returns (google.protobuf.Empty); - rpc BuildRestarting(BuildRestartingRequest) returns (google.protobuf.Empty); - - rpc IncrementEngineCorpusSize(IncrementEngineCorpusSizeRequest) returns (google.protobuf.Empty); - rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty); - rpc UpdateBuildExecutionData(UpdateBuildExecutionDataRequest) returns (google.protobuf.Empty); - rpc UpdateTargetQuoteConvention(UpdateTargetQuoteConventionRequest) returns (google.protobuf.Empty); -} - -message UpdateBuildStatusRequest { - string build_id = 1; - int32 step = 2; - optional double progress = 3; - optional string message = 4; - optional int32 queue_depth = 5; - repeated Phase phases = 6; - optional google.protobuf.Timestamp started = 7; - optional google.protobuf.Timestamp completed = 8; -} - -message BuildStartedRequest { - string build_id = 1; -} - -message BuildCompletedRequest { - string build_id = 1; - int32 corpus_size = 2; - double confidence = 3; -} - -message BuildCanceledRequest { - string build_id = 1; -} - -message BuildFaultedRequest { - string build_id = 1; - string message = 2; -} - -message BuildRestartingRequest { - string build_id = 1; -} - -message IncrementEngineCorpusSizeRequest { - string engine_id = 1; - int32 count = 2; -} - -message InsertPretranslationsRequest { - string engine_id = 1; - string corpus_id = 2; - string text_id = 3; - repeated string source_refs = 4; - repeated string target_refs = 5; - string translation = 6; - repeated string source_tokens = 7; - repeated string translation_tokens = 8; - repeated AlignedWordPair alignment = 9; - double confidence = 10; -} - -message UpdateBuildExecutionDataRequest { - string engine_id = 1; - string build_id = 2; - ExecutionData execution_data = 3; -} - -message UpdateTargetQuoteConventionRequest { - string engine_id = 1; - string build_id = 2; - optional string target_quote_convention = 3; -} - -message Phase { - PhaseStage stage = 1; - optional int32 step = 2; - optional int32 step_count = 3; - optional google.protobuf.Timestamp started = 4; -} - -enum PhaseStage { - PHASE_STAGE_TRAIN = 0; - PHASE_STAGE_INFERENCE = 1; -} - -message ExecutionData { - int32 train_count = 1; - int32 pretranslate_count = 2; - repeated string warnings = 3; - string engine_source_language_tag = 4; - string engine_target_language_tag = 5; - string resolved_source_language = 6; - string resolved_target_language = 7; -} \ No newline at end of file diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/common.proto b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/common.proto deleted file mode 100644 index df7c53e91..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/common.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; - -package serval.word_alignment.v1; - -message AlignedWordPair { - int32 source_index = 1; - int32 target_index = 2; - double score = 3; -} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto deleted file mode 100644 index 8e4c722d8..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto +++ /dev/null @@ -1,103 +0,0 @@ -syntax = "proto3"; - -package serval.word_alignment.v1; - -import "google/protobuf/empty.proto"; -import "serval/word_alignment/v1/common.proto"; - -service WordAlignmentEngineApi { - rpc Create(CreateRequest) returns (google.protobuf.Empty); - rpc Delete(DeleteRequest) returns (google.protobuf.Empty); - rpc GetWordAlignment(GetWordAlignmentRequest) returns (GetWordAlignmentResponse); - rpc StartBuild(StartBuildRequest) returns (google.protobuf.Empty); - rpc CancelBuild(CancelBuildRequest) returns (CancelBuildResponse); - rpc GetQueueSize(GetQueueSizeRequest) returns (GetQueueSizeResponse); -} - -message CreateRequest { - string engine_type = 1; - string engine_id = 2; - optional string engine_name = 3; - string source_language = 4; - string target_language = 5; -} - -message DeleteRequest { - string engine_type = 1; - string engine_id = 2; -} - -message GetWordAlignmentRequest { - string engine_type = 1; - string engine_id = 2; - string source_segment = 3; - string target_segment = 4; -} - -message GetWordAlignmentResponse { - WordAlignmentResult result = 1; -} - -message StartBuildRequest { - string engine_type = 1; - string engine_id = 2; - string build_id = 3; - optional string options = 4; - repeated ParallelCorpus corpora = 5; -} - -message CancelBuildRequest { - string engine_type = 1; - string engine_id = 2; -} - -message CancelBuildResponse { - string build_id = 1; -} - -message GetQueueSizeRequest { - string engine_type = 1; -} - -message GetQueueSizeResponse { - int32 size = 1; -} - -message WordAlignmentResult { - repeated string source_tokens = 1; - repeated string target_tokens = 2; - repeated AlignedWordPair alignment = 4; -} - -message ParallelCorpus { - string id = 1; - repeated MonolingualCorpus source_corpora = 2; - repeated MonolingualCorpus target_corpora = 3; -} - -message MonolingualCorpus { - string id = 1; - string language = 2; - bool train_on_all = 3; - bool word_align_on_all = 4; - map train_on_chapters = 5; - map word_align_on_chapters = 6; - repeated string train_on_text_ids = 7; - repeated string word_align_on_text_ids = 8; - repeated CorpusFile files = 9; -} - -message ScriptureChapters { - repeated int32 chapters = 1; -} - -message CorpusFile { - string location = 1; - FileFormat format = 2; - string text_id = 3; -} - -enum FileFormat { - FILE_FORMAT_TEXT = 0; - FILE_FORMAT_PARATEXT = 1; -} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto deleted file mode 100644 index 73f705910..000000000 --- a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto +++ /dev/null @@ -1,98 +0,0 @@ -syntax = "proto3"; - -package serval.word_alignment.v1; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; -import "serval/word_alignment/v1/common.proto"; - - -service WordAlignmentPlatformApi { - rpc UpdateBuildStatus(UpdateBuildStatusRequest) returns (google.protobuf.Empty); - rpc BuildStarted(BuildStartedRequest) returns (google.protobuf.Empty); - rpc BuildCompleted(BuildCompletedRequest) returns (google.protobuf.Empty); - rpc BuildCanceled(BuildCanceledRequest) returns (google.protobuf.Empty); - rpc BuildFaulted(BuildFaultedRequest) returns (google.protobuf.Empty); - rpc BuildRestarting(BuildRestartingRequest) returns (google.protobuf.Empty); - - rpc IncrementEngineCorpusSize(IncrementEngineCorpusSizeRequest) returns (google.protobuf.Empty); - rpc InsertWordAlignments(stream InsertWordAlignmentsRequest) returns (google.protobuf.Empty); - rpc UpdateBuildExecutionData(UpdateBuildExecutionDataRequest) returns (google.protobuf.Empty); - -} - -message UpdateBuildStatusRequest { - string build_id = 1; - int32 step = 2; - optional double progress = 3; - optional string message = 4; - optional int32 queue_depth = 5; - repeated Phase phases = 6; - optional google.protobuf.Timestamp started = 7; - optional google.protobuf.Timestamp completed = 8; -} - -message BuildStartedRequest { - string build_id = 1; -} - -message BuildCompletedRequest { - string build_id = 1; - int32 corpus_size = 2; - double confidence = 3; -} - -message BuildCanceledRequest { - string build_id = 1; -} - -message BuildFaultedRequest { - string build_id = 1; - string message = 2; -} - -message BuildRestartingRequest { - string build_id = 1; -} - -message IncrementEngineCorpusSizeRequest { - string engine_id = 1; - int32 count = 2; -} - -message InsertWordAlignmentsRequest { - string engine_id = 1; - string corpus_id = 2; - string text_id = 3; - repeated string source_refs = 4; - repeated string target_refs = 5; - repeated string source_tokens = 6; - repeated string target_tokens = 7; - repeated AlignedWordPair alignment = 8; -} - -message UpdateBuildExecutionDataRequest { - string engine_id = 1; - string build_id = 2; - ExecutionData execution_data = 3; -} - -message Phase { - PhaseStage stage = 1; - optional int32 step = 2; - optional int32 step_count = 3; - optional google.protobuf.Timestamp started = 4; -} - -enum PhaseStage { - PHASE_STAGE_TRAIN = 0; - PHASE_STAGE_INFERENCE = 1; -} - -message ExecutionData { - int32 train_count = 1; - int32 word_align_count = 2; - repeated string warnings = 3; - string engine_source_language_tag = 4; - string engine_target_language_tag = 5; -} diff --git a/src/Serval/src/Serval.Grpc/Serval.Grpc.csproj b/src/Serval/src/Serval.Grpc/Serval.Grpc.csproj deleted file mode 100644 index 302dada63..000000000 --- a/src/Serval/src/Serval.Grpc/Serval.Grpc.csproj +++ /dev/null @@ -1,32 +0,0 @@ - - - - net10.0 - enable - enable - 0.16.0 - The Serval gRPC APIs. - true - true - true - $(NoWarn);CS1591;CS1573 - Serval - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - diff --git a/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs b/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs deleted file mode 100644 index 7e9e6da20..000000000 --- a/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Microsoft.Extensions.Diagnostics.HealthChecks; - -namespace Serval.Health.V1; - -public class WriteGrpcHealthCheckResponse -{ - public static HealthCheckResponse Generate(HealthReport healthReport) - { - Dictionary healthCheckResultData = []; - string? healthCheckResultException = null; - - // Combine data and exceptions from all health checks - foreach (KeyValuePair entry in healthReport.Entries) - { - healthCheckResultData.Add(entry.Key, $"{entry.Value.Status}: {entry.Value.Description ?? ""}"); - if ((entry.Value.Exception?.ToString() ?? "") != "") - { - if (healthCheckResultException is null) - healthCheckResultException = $"{entry.Key}: {entry.Value.Exception}"; - else - healthCheckResultException += $"\n{entry.Key}: {entry.Value.Exception}"; - } - } - // Assemble response - HealthCheckResponse healthCheckResponse = new() - { - Status = (HealthCheckStatus)healthReport.Status, - Error = healthCheckResultException ?? "", - }; - foreach (KeyValuePair entry in healthCheckResultData) - { - healthCheckResponse.Data.Add(entry.Key, entry.Value); - } - return healthCheckResponse; - } -} diff --git a/src/Serval/src/Serval.Shared.Contracts/AlignedWordPairContract.cs b/src/Serval/src/Serval.Shared.Contracts/AlignedWordPairContract.cs new file mode 100644 index 000000000..05f5f318b --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/AlignedWordPairContract.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Contracts; + +public record AlignedWordPairContract +{ + public required int SourceIndex { get; set; } + public required int TargetIndex { get; set; } + public double Score { get; set; } +} diff --git a/src/Serval/src/Serval.Shared.Contracts/BuildProgressStatusContract.cs b/src/Serval/src/Serval.Shared.Contracts/BuildProgressStatusContract.cs new file mode 100644 index 000000000..342efa50b --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/BuildProgressStatusContract.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Contracts; + +public record BuildProgressStatusContract +{ + public int Step { get; set; } + public double? PercentCompleted { get; set; } + public string? Message { get; set; } +} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared.Contracts/CorpusFileContract.cs similarity index 55% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Models/CorpusFile.cs rename to src/Serval/src/Serval.Shared.Contracts/CorpusFileContract.cs index 6e1f9298b..d2cefab36 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.Shared.Contracts/CorpusFileContract.cs @@ -1,12 +1,6 @@ -namespace SIL.ServiceToolkit.Models; +namespace Serval.Shared.Contracts; -public enum FileFormat -{ - Text = 0, - Paratext = 1, -} - -public record CorpusFile +public record CorpusFileContract { public required string Location { get; init; } public required FileFormat Format { get; init; } diff --git a/src/Serval/src/Serval.Shared/Contracts/FileFormat.cs b/src/Serval/src/Serval.Shared.Contracts/FileFormat.cs similarity index 100% rename from src/Serval/src/Serval.Shared/Contracts/FileFormat.cs rename to src/Serval/src/Serval.Shared.Contracts/FileFormat.cs diff --git a/src/Serval/src/Serval.Shared.Contracts/IEvent.cs b/src/Serval/src/Serval.Shared.Contracts/IEvent.cs new file mode 100644 index 000000000..701610533 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IEvent.cs @@ -0,0 +1,3 @@ +namespace Serval.Shared.Contracts; + +public interface IEvent; diff --git a/src/Serval/src/Serval.Shared.Contracts/IEventHandler.cs b/src/Serval/src/Serval.Shared.Contracts/IEventHandler.cs new file mode 100644 index 000000000..6e2cb0342 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IEventHandler.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Contracts; + +public interface IEventHandler + where TEvent : IEvent +{ + Task HandleAsync(TEvent evt, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IEventRouter.cs b/src/Serval/src/Serval.Shared.Contracts/IEventRouter.cs new file mode 100644 index 000000000..fa80ea6f4 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IEventRouter.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Contracts; + +public interface IEventRouter +{ + Task PublishAsync(TEvent evt, CancellationToken cancellationToken = default) + where TEvent : IEvent; +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs new file mode 100644 index 000000000..616d831dc --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs @@ -0,0 +1,32 @@ +namespace Serval.Shared.Contracts; + +public interface IParallelCorpusService +{ + string AnalyzeTargetQuoteConvention(IEnumerable parallelCorpora); + + IReadOnlyList<( + string ParallelCorpusId, + string MonolingualCorpusId, + IReadOnlyList Errors + )> AnalyzeUsfmVersification(IEnumerable parallelCorpora); + + IReadOnlyList<( + string ParallelCorpusId, + string MonolingualCorpusId, + MissingParentProjectErrorContract Error + )> FindMissingParentProjects(IEnumerable parallelCorpora); + + Task PreprocessAsync( + IEnumerable parallelCorpora, + Func train, + Func inference, + bool useKeyTerms = false, + HashSet? ignoreUsfmMarkers = null + ); + + Dictionary> GetChapters( + IReadOnlyList parallelCorpora, + string fileLocation, + string scriptureRange + ); +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IRequest.cs b/src/Serval/src/Serval.Shared.Contracts/IRequest.cs new file mode 100644 index 000000000..c936cb139 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IRequest.cs @@ -0,0 +1,5 @@ +namespace Serval.Shared.Contracts; + +public interface IRequest; + +public interface IRequest; diff --git a/src/Serval/src/Serval.Shared.Contracts/IRequestHandler.cs b/src/Serval/src/Serval.Shared.Contracts/IRequestHandler.cs new file mode 100644 index 000000000..f4b46e3cc --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IRequestHandler.cs @@ -0,0 +1,13 @@ +namespace Serval.Shared.Contracts; + +public interface IRequestHandler + where TRequest : IRequest +{ + Task HandleAsync(TRequest request, CancellationToken cancellationToken = default); +} + +public interface IRequestHandler + where TRequest : IRequest +{ + Task HandleAsync(TRequest request, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IServalConfigurator.cs b/src/Serval/src/Serval.Shared.Contracts/IServalConfigurator.cs new file mode 100644 index 000000000..5e7611b33 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IServalConfigurator.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.Configuration; + +namespace Microsoft.Extensions.DependencyInjection; + +public interface IServalConfigurator +{ + IServiceCollection Services { get; } + IConfiguration Configuration { get; } + IMongoDataAccessBuilder DataAccess { get; } + ICollection JobQueues { get; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/JobState.cs b/src/Serval/src/Serval.Shared.Contracts/JobState.cs similarity index 100% rename from src/Serval/src/Serval.Shared/Contracts/JobState.cs rename to src/Serval/src/Serval.Shared.Contracts/JobState.cs diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MissingParentProjectError.cs b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs similarity index 58% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MissingParentProjectError.cs rename to src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs index fb76591dc..bb382db14 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MissingParentProjectError.cs +++ b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs @@ -1,6 +1,6 @@ -namespace SIL.ServiceToolkit.Models; +namespace Serval.Shared.Contracts; -public record MissingParentProjectError +public record MissingParentProjectErrorContract { public required string ProjectName { get; init; } public required string ParentProjectName { get; init; } diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Shared.Contracts/MonolingualCorpusContract.cs similarity index 69% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MonolingualCorpus.cs rename to src/Serval/src/Serval.Shared.Contracts/MonolingualCorpusContract.cs index 5b366a71b..3da620fc1 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/MonolingualCorpus.cs +++ b/src/Serval/src/Serval.Shared.Contracts/MonolingualCorpusContract.cs @@ -1,16 +1,14 @@ -namespace SIL.ServiceToolkit.Models; +namespace Serval.Shared.Contracts; -public record MonolingualCorpus +public record MonolingualCorpusContract { public required string Id { get; set; } public required string Language { get; set; } - public required IReadOnlyList Files { get; set; } + public required List Files { get; set; } public HashSet? TrainOnTextIds { get; set; } public Dictionary>? TrainOnChapters { get; set; } public HashSet? InferenceTextIds { get; set; } public Dictionary>? InferenceChapters { get; set; } - public bool TrainOnAll { get; set; } - public bool PretranslateAll { get; set; } public bool IsFiltered => TrainOnTextIds != null || TrainOnChapters != null || InferenceTextIds != null || InferenceChapters != null; diff --git a/src/Serval/src/Serval.Shared.Contracts/ParallelCorpusContract.cs b/src/Serval/src/Serval.Shared.Contracts/ParallelCorpusContract.cs new file mode 100644 index 000000000..45d921c7f --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/ParallelCorpusContract.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Contracts; + +public record ParallelCorpusContract +{ + public required string Id { get; set; } + public List SourceCorpora { get; set; } = []; + public List TargetCorpora { get; set; } = []; +} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Row.cs b/src/Serval/src/Serval.Shared.Contracts/ParallelRowContract.cs similarity index 70% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Row.cs rename to src/Serval/src/Serval.Shared.Contracts/ParallelRowContract.cs index 1e37cd055..a73aab598 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Row.cs +++ b/src/Serval/src/Serval.Shared.Contracts/ParallelRowContract.cs @@ -1,6 +1,6 @@ -namespace SIL.ServiceToolkit.Models; +namespace Serval.Shared.Contracts; -public record Row( +public record ParallelRowContract( string TextId, IReadOnlyList SourceRefs, IReadOnlyList TargetRefs, diff --git a/src/Serval/src/Serval.Shared.Contracts/PhaseContract.cs b/src/Serval/src/Serval.Shared.Contracts/PhaseContract.cs new file mode 100644 index 000000000..ce982f998 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/PhaseContract.cs @@ -0,0 +1,9 @@ +namespace Serval.Shared.Contracts; + +public record PhaseContract +{ + public required PhaseStage Stage { get; init; } + public int? Step { get; init; } + public int? StepCount { get; init; } + public DateTime? Started { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/PhaseStage.cs b/src/Serval/src/Serval.Shared.Contracts/PhaseStage.cs similarity index 100% rename from src/Serval/src/Serval.Shared/Contracts/PhaseStage.cs rename to src/Serval/src/Serval.Shared.Contracts/PhaseStage.cs diff --git a/src/Serval/src/Serval.Shared.Contracts/Serval.Shared.Contracts.csproj b/src/Serval/src/Serval.Shared.Contracts/Serval.Shared.Contracts.csproj new file mode 100644 index 000000000..f71fd21f6 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/Serval.Shared.Contracts.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/TrainingDataType.cs b/src/Serval/src/Serval.Shared.Contracts/TrainingDataType.cs similarity index 63% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Models/TrainingDataType.cs rename to src/Serval/src/Serval.Shared.Contracts/TrainingDataType.cs index 46f23035b..cda170b78 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/TrainingDataType.cs +++ b/src/Serval/src/Serval.Shared.Contracts/TrainingDataType.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Models; +namespace Serval.Shared.Contracts; public enum TrainingDataType { diff --git a/src/Serval/src/Serval.Shared.Contracts/UsfmVersificationErrorContract.cs b/src/Serval/src/Serval.Shared.Contracts/UsfmVersificationErrorContract.cs new file mode 100644 index 000000000..96b184deb --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/UsfmVersificationErrorContract.cs @@ -0,0 +1,21 @@ +namespace Serval.Shared.Contracts; + +public enum UsfmVersificationErrorType +{ + MissingChapter, + MissingVerse, + ExtraVerse, + InvalidVerseRange, + MissingVerseSegment, + ExtraVerseSegment, + InvalidChapterNumber, + InvalidVerseNumber, +} + +public record UsfmVersificationErrorContract +{ + public required UsfmVersificationErrorType Type { get; init; } + public required string ProjectName { get; init; } + public required string ExpectedVerseRef { get; init; } + public required string ActualVerseRef { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Configuration/IServalBuilder.cs b/src/Serval/src/Serval.Shared/Configuration/IServalBuilder.cs deleted file mode 100644 index f37283e33..000000000 --- a/src/Serval/src/Serval.Shared/Configuration/IServalBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public interface IServalBuilder -{ - IServiceCollection Services { get; } - IConfiguration Configuration { get; } -} diff --git a/src/Serval/src/Serval.Shared/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Shared/Configuration/IServalBuilderExtensions.cs deleted file mode 100644 index bce1f02d3..000000000 --- a/src/Serval/src/Serval.Shared/Configuration/IServalBuilderExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServalBuilderExtensions -{ - public static IServalBuilder AddDataFileOptions(this IServalBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IServalBuilder AddApiOptions(this IServalBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IServalBuilder AddMemoryDataAccess( - this IServalBuilder builder, - Action configure - ) - { - builder.Services.AddMemoryDataAccess(configure); - return builder; - } - - public static IServalBuilder AddMongoDataAccess( - this IServalBuilder builder, - Action configure - ) - { - string? mongoConnectionString = builder.Configuration.GetConnectionString("Mongo"); - if (mongoConnectionString is null) - throw new InvalidOperationException("Mongo connection string not configured"); - builder.Services.AddMongoDataAccess(mongoConnectionString, "Serval", configure); - builder.Services.AddHealthChecks().AddMongoDb(name: "Mongo"); - return builder; - } - - public static IServalBuilder AddMongoOutbox(this IServalBuilder builder) - { - string? mongoConnectionString = builder.Configuration.GetConnectionString("Mongo"); - if (mongoConnectionString is null) - throw new InvalidOperationException("Mongo connection string not configured"); - builder.Services.AddOutbox(builder.Configuration, x => x.UseMongo(mongoConnectionString)); - builder.Services.AddHealthChecks().AddOutbox(); - return builder; - } - - public static IServalBuilder AddOutboxDeliveryService(this IServalBuilder builder) - { - builder.Services.AddOutbox(x => x.UseDeliveryService()); - return builder; - } -} diff --git a/src/Serval/src/Serval.Shared/Configuration/IServalConfiguratorExtensions.cs b/src/Serval/src/Serval.Shared/Configuration/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..6d404ca9e --- /dev/null +++ b/src/Serval/src/Serval.Shared/Configuration/IServalConfiguratorExtensions.cs @@ -0,0 +1,31 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddHandlers(this IServalConfigurator builder, Assembly assembly) + { + foreach (Type type in assembly.GetTypes()) + { + foreach (Type intf in type.GetInterfaces()) + { + if (intf.IsGenericType) + { + Type genericType = intf.GetGenericTypeDefinition(); + if (genericType == typeof(IRequestHandler<,>)) + { + builder.Services.AddScoped(intf, type); + } + else if (genericType == typeof(IRequestHandler<>)) + { + builder.Services.AddScoped(intf, type); + } + else if (genericType == typeof(IEventHandler<>)) + { + builder.Services.AddScoped(intf, type); + } + } + } + } + return builder; + } +} diff --git a/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs index a6f64f22a..36376d25d 100644 --- a/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs +++ b/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs @@ -2,9 +2,50 @@ public static class IServiceCollectionExtensions { - public static IServalBuilder AddServal(this IServiceCollection services, IConfiguration configuration) + public static IServiceCollection AddServal( + this IServiceCollection services, + IConfiguration configuration, + Action configure + ) { - services.AddFileSystem(); - return new ServalBuilder(services, configuration); + services.AddTransient(); + services.AddSingleton(); + services.AddScoped(); + + services.Configure(configuration.GetSection(DataFileOptions.Key)); + services.Configure(configuration.GetSection(ApiOptions.Key)); + + string? mongoConnectionString = configuration.GetConnectionString("Mongo"); + if (mongoConnectionString is null) + throw new InvalidOperationException("Mongo connection string not configured"); + IMongoDataAccessBuilder dataAccess = services.AddMongoDataAccess(mongoConnectionString, "Serval"); + services.AddHealthChecks().AddMongoDb(name: "Mongo"); + + services.AddHangfire(c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_180) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage( + configuration.GetConnectionString("Hangfire"), + new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy(), + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + } + ) + ); + + ServalConfigurator configurator = new(services, configuration, dataAccess); + configure(configurator); + + services.AddHangfireServer(o => o.Queues = [.. configurator.JobQueues]); + services.AddHealthChecks().AddCheck("Hangfire"); + + return services; } } diff --git a/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs b/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs deleted file mode 100644 index f2f25f4f6..000000000 --- a/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public class ServalBuilder(IServiceCollection services, IConfiguration configuration) : IServalBuilder -{ - public IServiceCollection Services { get; } = services; - public IConfiguration Configuration { get; } = configuration; -} diff --git a/src/Serval/src/Serval.Shared/Configuration/ServalConfigurator.cs b/src/Serval/src/Serval.Shared/Configuration/ServalConfigurator.cs new file mode 100644 index 000000000..8e7384fee --- /dev/null +++ b/src/Serval/src/Serval.Shared/Configuration/ServalConfigurator.cs @@ -0,0 +1,13 @@ +namespace Microsoft.Extensions.DependencyInjection; + +internal class ServalConfigurator( + IServiceCollection services, + IConfiguration configuration, + IMongoDataAccessBuilder dataAccess +) : IServalConfigurator +{ + public IServiceCollection Services { get; } = services; + public IConfiguration Configuration { get; } = configuration; + public IMongoDataAccessBuilder DataAccess { get; } = dataAccess; + public ICollection JobQueues { get; } = []; +} diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusFileResult.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusFileResult.cs deleted file mode 100644 index 953d705be..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/CorpusFileResult.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record CorpusFileResult -{ - public required DataFileResult File { get; init; } - public required string TextId { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusNotFound.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusNotFound.cs deleted file mode 100644 index 81f9246b5..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/CorpusNotFound.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record CorpusNotFound -{ - public required string CorpusId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs deleted file mode 100644 index 0c0f83802..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record CorpusResult -{ - public required string CorpusId { get; init; } - public required string Language { get; init; } - public string? Name { get; init; } - public required IReadOnlyList Files { get; set; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs deleted file mode 100644 index 402658f38..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record CorpusUpdated -{ - public required string CorpusId { get; init; } - public required IReadOnlyList Files { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileDeleted.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileDeleted.cs deleted file mode 100644 index f0be50203..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/DataFileDeleted.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record DataFileDeleted -{ - public required string DataFileId { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileNotFound.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileNotFound.cs deleted file mode 100644 index f40df3681..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/DataFileNotFound.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record DataFileNotFound -{ - public required string DataFileId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileResult.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileResult.cs deleted file mode 100644 index 0dff0331b..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/DataFileResult.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record DataFileResult -{ - public required string DataFileId { get; init; } - public required string Name { get; init; } - public required string Filename { get; init; } - public required FileFormat Format { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs deleted file mode 100644 index 7968f28a4..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record DataFileUpdated -{ - public required string DataFileId { get; init; } - public required string Filename { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/DeleteDataFile.cs b/src/Serval/src/Serval.Shared/Contracts/DeleteDataFile.cs deleted file mode 100644 index 417e7c08b..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/DeleteDataFile.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record DeleteDataFile -{ - public required string DataFileId { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/GetCorpus.cs b/src/Serval/src/Serval.Shared/Contracts/GetCorpus.cs deleted file mode 100644 index a29b4f125..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/GetCorpus.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record GetCorpus -{ - public required string CorpusId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/GetDataFile.cs b/src/Serval/src/Serval.Shared/Contracts/GetDataFile.cs deleted file mode 100644 index 0ec2f1472..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/GetDataFile.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record GetDataFile -{ - public required string DataFileId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinished.cs b/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinished.cs deleted file mode 100644 index 73df98021..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinished.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record TranslationBuildFinished -{ - public required string BuildId { get; init; } - public required string EngineId { get; init; } - public required string Owner { get; init; } - public required JobState BuildState { get; init; } - public required string Message { get; init; } - public required DateTime DateFinished { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildStarted.cs b/src/Serval/src/Serval.Shared/Contracts/TranslationBuildStarted.cs deleted file mode 100644 index 04a9a5dfd..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildStarted.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record TranslationBuildStarted -{ - public required string BuildId { get; init; } - public required string EngineId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs deleted file mode 100644 index 44886a444..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record WordAlignmentBuildFinished -{ - public required string BuildId { get; init; } - public required string EngineId { get; init; } - public required string Owner { get; init; } - public required JobState BuildState { get; init; } - public required string Message { get; init; } - public required DateTime DateFinished { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs deleted file mode 100644 index 08a6375f7..000000000 --- a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Shared.Contracts; - -public record WordAlignmentBuildStarted -{ - public required string BuildId { get; init; } - public required string EngineId { get; init; } - public required string Owner { get; init; } -} diff --git a/src/Serval/src/Serval.Shared/Controllers/NotSupportedExceptionFilter.cs b/src/Serval/src/Serval.Shared/Controllers/NotSupportedExceptionFilter.cs index 1f3aff8fd..633ad7875 100644 --- a/src/Serval/src/Serval.Shared/Controllers/NotSupportedExceptionFilter.cs +++ b/src/Serval/src/Serval.Shared/Controllers/NotSupportedExceptionFilter.cs @@ -4,10 +4,7 @@ public class NotSupportedExceptionFilter : ExceptionFilterAttribute { public override void OnException(ExceptionContext context) { - if ( - context.Exception is NotSupportedException - || context.Exception is RpcException rpcException && rpcException.StatusCode == StatusCode.Unimplemented - ) + if (context.Exception is NotSupportedException) { context.ExceptionHandled = true; context.Result = new StatusCodeResult(StatusCodes.Status405MethodNotAllowed); diff --git a/src/Serval/src/Serval.Shared/Controllers/OperationCancelledExceptionFilter.cs b/src/Serval/src/Serval.Shared/Controllers/OperationCanceledExceptionFilter.cs similarity index 61% rename from src/Serval/src/Serval.Shared/Controllers/OperationCancelledExceptionFilter.cs rename to src/Serval/src/Serval.Shared/Controllers/OperationCanceledExceptionFilter.cs index 40b494d1d..331276c0b 100644 --- a/src/Serval/src/Serval.Shared/Controllers/OperationCancelledExceptionFilter.cs +++ b/src/Serval/src/Serval.Shared/Controllers/OperationCanceledExceptionFilter.cs @@ -1,18 +1,15 @@ namespace Serval.Shared.Controllers; -public class OperationCancelledExceptionFilter(ILoggerFactory loggerFactory) : ExceptionFilterAttribute +public class OperationCanceledExceptionFilter(ILoggerFactory loggerFactory) : ExceptionFilterAttribute { - private readonly ILogger _logger = loggerFactory.CreateLogger(); + private readonly ILogger _logger = loggerFactory.CreateLogger(); public override void OnException(ExceptionContext context) { - if ( - context.Exception is OperationCanceledException - || context.Exception is RpcException rpcEx && rpcEx.StatusCode == StatusCode.Cancelled - ) + if (context.Exception is OperationCanceledException) { _logger.LogInformation( - "Request {RequestMethod}:{RequestPath} was cancelled", + "Request {RequestMethod}:{RequestPath} was canceled", context.HttpContext.Request.Method, context.HttpContext.Request.Path ); diff --git a/src/Serval/src/Serval.Shared/Controllers/ServalControllerBase.cs b/src/Serval/src/Serval.Shared/Controllers/ServalControllerBase.cs index 82727023b..4db472281 100644 --- a/src/Serval/src/Serval.Shared/Controllers/ServalControllerBase.cs +++ b/src/Serval/src/Serval.Shared/Controllers/ServalControllerBase.cs @@ -2,7 +2,7 @@ [ApiController] [Produces("application/json")] -[TypeFilter(typeof(OperationCancelledExceptionFilter))] +[TypeFilter(typeof(OperationCanceledExceptionFilter))] [TypeFilter(typeof(NotSupportedExceptionFilter))] [TypeFilter(typeof(ServiceUnavailableExceptionFilter))] [TypeFilter(typeof(ErrorResultFilter))] diff --git a/src/Serval/src/Serval.Shared/Controllers/ServiceUnavailableExceptionFilter.cs b/src/Serval/src/Serval.Shared/Controllers/ServiceUnavailableExceptionFilter.cs index 73586c75f..c3caea3f3 100644 --- a/src/Serval/src/Serval.Shared/Controllers/ServiceUnavailableExceptionFilter.cs +++ b/src/Serval/src/Serval.Shared/Controllers/ServiceUnavailableExceptionFilter.cs @@ -7,10 +7,7 @@ public class ServiceUnavailableExceptionFilter(ILoggerFactory loggerFactory) : E public override void OnException(ExceptionContext context) { - if ( - (context.Exception is TimeoutException) - || (context.Exception is RpcException rpcEx && rpcEx.StatusCode == StatusCode.Unavailable) - ) + if (context.Exception is TimeoutException) { _logger.Log(LogLevel.Error, context.Exception, "A user tried to access an unavailable service."); context.Result = new StatusCodeResult(StatusCodes.Status503ServiceUnavailable); diff --git a/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs b/src/Serval/src/Serval.Shared/Dtos/AlignedWordPairDto.cs similarity index 83% rename from src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs rename to src/Serval/src/Serval.Shared/Dtos/AlignedWordPairDto.cs index 9e7fa507a..edcc59c1a 100644 --- a/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/AlignedWordPairDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record AlignedWordPairDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterConfigDto.cs b/src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterConfigDto.cs similarity index 85% rename from src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterConfigDto.cs rename to src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterConfigDto.cs index 7b0be59a5..430d7f8d4 100644 --- a/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterConfigDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record ParallelCorpusFilterConfigDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterDto.cs b/src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterDto.cs similarity index 85% rename from src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterDto.cs rename to src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterDto.cs index d51e953a5..e60f22da5 100644 --- a/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilterDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/ParallelCorpusFilterDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record ParallelCorpusFilterDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/PhaseDto.cs b/src/Serval/src/Serval.Shared/Dtos/PhaseDto.cs similarity index 84% rename from src/Serval/src/Serval.Shared/Contracts/PhaseDto.cs rename to src/Serval/src/Serval.Shared/Dtos/PhaseDto.cs index c1c37065a..2927c9f19 100644 --- a/src/Serval/src/Serval.Shared/Contracts/PhaseDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/PhaseDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record PhaseDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs b/src/Serval/src/Serval.Shared/Dtos/QueueDto.cs similarity index 78% rename from src/Serval/src/Serval.Shared/Contracts/QueueDto.cs rename to src/Serval/src/Serval.Shared/Dtos/QueueDto.cs index ade49773c..5bc66481a 100644 --- a/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/QueueDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record QueueDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/ResourceLinkDto.cs b/src/Serval/src/Serval.Shared/Dtos/ResourceLinkDto.cs similarity index 58% rename from src/Serval/src/Serval.Shared/Contracts/ResourceLinkDto.cs rename to src/Serval/src/Serval.Shared/Dtos/ResourceLinkDto.cs index efd7eb53d..ef3d3c589 100644 --- a/src/Serval/src/Serval.Shared/Contracts/ResourceLinkDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/ResourceLinkDto.cs @@ -1,6 +1,6 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; -public class ResourceLinkDto +public record ResourceLinkDto { public required string Id { get; init; } public required string Url { get; init; } diff --git a/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs b/src/Serval/src/Serval.Shared/Dtos/TrainingCorpusConfigDto.cs similarity index 92% rename from src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs rename to src/Serval/src/Serval.Shared/Dtos/TrainingCorpusConfigDto.cs index 5358f1dcd..584398dbb 100644 --- a/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/TrainingCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record TrainingCorpusConfigDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs b/src/Serval/src/Serval.Shared/Dtos/TrainingCorpusDto.cs similarity index 92% rename from src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs rename to src/Serval/src/Serval.Shared/Dtos/TrainingCorpusDto.cs index 5050f6aa2..af555ca70 100644 --- a/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/TrainingCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record TrainingCorpusDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs b/src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildFinishedDto.cs similarity index 89% rename from src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs rename to src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildFinishedDto.cs index f7ec9936d..5e0898a69 100644 --- a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildFinishedDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record WordAlignmentBuildFinishedDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs b/src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildStartedDto.cs similarity index 81% rename from src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs rename to src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildStartedDto.cs index 00f1c178e..fc4ddd19e 100644 --- a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs +++ b/src/Serval/src/Serval.Shared/Dtos/WordAlignmentBuildStartedDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Contracts; +namespace Serval.Shared.Dtos; public record WordAlignmentBuildStartedDto { diff --git a/src/Serval/src/Serval.WordAlignment/Models/AlignedWordPair.cs b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs similarity index 81% rename from src/Serval/src/Serval.WordAlignment/Models/AlignedWordPair.cs rename to src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs index 19ccc7567..48f01f86e 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/AlignedWordPair.cs +++ b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Models; +namespace Serval.Shared.Models; public record AlignedWordPair { diff --git a/src/Serval/src/Serval.Shared/Models/BuildPhase.cs b/src/Serval/src/Serval.Shared/Models/Phase.cs similarity index 53% rename from src/Serval/src/Serval.Shared/Models/BuildPhase.cs rename to src/Serval/src/Serval.Shared/Models/Phase.cs index 2239b7aed..29511d8dd 100644 --- a/src/Serval/src/Serval.Shared/Models/BuildPhase.cs +++ b/src/Serval/src/Serval.Shared/Models/Phase.cs @@ -1,14 +1,8 @@ namespace Serval.Shared.Models; -public enum BuildPhaseStage +public record Phase { - Train, - Inference, -} - -public record BuildPhase -{ - public required BuildPhaseStage Stage { get; init; } + public required PhaseStage Stage { get; init; } public int? Step { get; init; } public int? StepCount { get; init; } public DateTime? Started { get; init; } diff --git a/src/Serval/src/Serval.Shared/Properties/AssemblyInfo.cs b/src/Serval/src/Serval.Shared/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..cbfe01de5 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Serval.Shared.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index bd2222d6a..827921d4d 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -10,24 +10,23 @@ $(NoWarn);CS1591;CS1573 + + - - - - - + + + - + - diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusBundle.cs b/src/Serval/src/Serval.Shared/Services/CorpusBundle.cs similarity index 74% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusBundle.cs rename to src/Serval/src/Serval.Shared/Services/CorpusBundle.cs index 6bca340cc..9eb75f0c6 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusBundle.cs +++ b/src/Serval/src/Serval.Shared/Services/CorpusBundle.cs @@ -1,6 +1,4 @@ -using ZipParatextProjectTextUpdater = SIL.ServiceToolkit.Services.ZipParatextProjectTextUpdater; - -namespace SIL.ServiceToolkit.Utils; +namespace Serval.Shared.Services; public class CorpusBundle { @@ -10,54 +8,54 @@ private readonly Dictionary< > _settings; public IEnumerable<( - ParallelCorpus ParallelCorpus, - MonolingualCorpus MonolingualCorpus, - IReadOnlyList CorpusFile, + ParallelCorpusContract ParallelCorpus, + MonolingualCorpusContract MonolingualCorpus, + IReadOnlyList CorpusFile, IReadOnlyList TextCorpora )> SourceTextCorpora { get; } public IEnumerable<( - ParallelCorpus ParallelCorpus, - MonolingualCorpus MonolingualCorpus, - IReadOnlyList CorpusFile, + ParallelCorpusContract ParallelCorpus, + MonolingualCorpusContract MonolingualCorpus, + IReadOnlyList CorpusFile, IReadOnlyList TextCorpora )> TargetTextCorpora { get; } public IEnumerable<( - ParallelCorpus ParallelCorpus, - MonolingualCorpus MonolingualCorpus, - IReadOnlyList CorpusFile, + ParallelCorpusContract ParallelCorpus, + MonolingualCorpusContract MonolingualCorpus, + IReadOnlyList CorpusFile, IReadOnlyList TextCorpora )> TextCorpora => SourceTextCorpora.Concat(TargetTextCorpora); public IEnumerable<( - ParallelCorpus ParallelCorpus, - MonolingualCorpus MonolingualCorpus, - IReadOnlyList CorpusFile, + ParallelCorpusContract ParallelCorpus, + MonolingualCorpusContract MonolingualCorpus, + IReadOnlyList CorpusFile, IReadOnlyList TextCorpora )> SourceTermCorpora { get; } public IEnumerable<( - ParallelCorpus ParallelCorpus, - MonolingualCorpus MonolingualCorpus, - IReadOnlyList CorpusFile, + ParallelCorpusContract ParallelCorpus, + MonolingualCorpusContract MonolingualCorpus, + IReadOnlyList CorpusFile, IReadOnlyList TextCorpora )> TargetTermCorpora { get; } - public IReadOnlyList ParallelCorpora { get; } + public IReadOnlyList ParallelCorpora { get; } - public CorpusBundle(IEnumerable parallelCorpora) + public CorpusBundle(IEnumerable parallelCorpora) { ParallelCorpora = parallelCorpora.ToArray(); _settings = []; - IEnumerable corpusFiles = parallelCorpora.SelectMany(corpus => + IEnumerable corpusFiles = parallelCorpora.SelectMany(corpus => corpus.SourceCorpora.Concat(corpus.TargetCorpora).SelectMany(c => c.Files) ); List<(string Location, ParatextProjectSettings Settings)> paratextProjects = []; - foreach (CorpusFile file in corpusFiles.Where(f => f.Format == FileFormat.Paratext)) + foreach (CorpusFileContract file in corpusFiles.Where(f => f.Format == FileFormat.Paratext)) { using IZipContainer archive = new ZipContainer(file.Location); - ParatextProjectSettings settings = new Services.ZipParatextProjectSettingsParser(archive).Parse(); + ParatextProjectSettings settings = new ZipParatextProjectSettingsParser(archive).Parse(); paratextProjects.Add((file.Location, settings)); } @@ -81,25 +79,45 @@ public CorpusBundle(IEnumerable parallelCorpora) SourceTextCorpora = parallelCorpora.SelectMany(parallelCorpus => parallelCorpus.SourceCorpora.Select(corpus => - (parallelCorpus, corpus, corpus.Files, CreateTextCorpora(corpus.Files)) + ( + parallelCorpus, + corpus, + (IReadOnlyList)corpus.Files, + CreateTextCorpora(corpus.Files) + ) ) ); TargetTextCorpora = parallelCorpora.SelectMany(parallelCorpus => parallelCorpus.TargetCorpora.Select(corpus => - (parallelCorpus, corpus, corpus.Files, CreateTextCorpora(corpus.Files)) + ( + parallelCorpus, + corpus, + (IReadOnlyList)corpus.Files, + CreateTextCorpora(corpus.Files) + ) ) ); SourceTermCorpora = parallelCorpora.SelectMany(parallelCorpus => parallelCorpus.SourceCorpora.Select(corpus => - (parallelCorpus, corpus, corpus.Files, CreateTermCorpora(corpus.Files)) + ( + parallelCorpus, + corpus, + (IReadOnlyList)corpus.Files, + CreateTermCorpora(corpus.Files) + ) ) ); TargetTermCorpora = parallelCorpora.SelectMany(parallelCorpus => parallelCorpus.TargetCorpora.Select(corpus => - (parallelCorpus, corpus, corpus.Files, CreateTermCorpora(corpus.Files)) + ( + parallelCorpus, + corpus, + (IReadOnlyList)corpus.Files, + CreateTermCorpora(corpus.Files) + ) ) ); } @@ -147,12 +165,12 @@ public ZipParatextProjectTextUpdater GetTextUpdater(string location) return new ZipParatextProjectTextUpdater(container, parentSettings); } - protected virtual IReadOnlyList CreateTextCorpora(IReadOnlyList files) + protected virtual IReadOnlyList CreateTextCorpora(IReadOnlyList files) { List corpora = []; List> textFileCorpora = []; - foreach (CorpusFile file in files) + foreach (CorpusFileContract file in files) { switch (file.Format) { @@ -197,10 +215,10 @@ protected virtual IReadOnlyList CreateTextCorpora(IReadOnlyList CreateTermCorpora(IReadOnlyList files) + private IReadOnlyList CreateTermCorpora(IReadOnlyList files) { List corpora = []; - foreach (CorpusFile file in files) + foreach (CorpusFileContract file in files) { switch (file.Format) { diff --git a/src/Serval/src/Serval.Shared/Services/EventRouter.cs b/src/Serval/src/Serval.Shared/Services/EventRouter.cs new file mode 100644 index 000000000..103e4ded5 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/EventRouter.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace Serval.Shared.Services; + +public class EventRouter(IServiceProvider serviceProvider) : IEventRouter +{ + private readonly IServiceProvider _serviceProvider = serviceProvider; + + public Task PublishAsync(TEvent evt, CancellationToken cancellationToken = default) + where TEvent : IEvent + { + return Task.WhenAll( + _serviceProvider + .GetServices>() + .Select(handler => handler.HandleAsync(evt, cancellationToken)) + ); + } +} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/FileSystem.cs b/src/Serval/src/Serval.Shared/Services/FileSystem.cs similarity index 92% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/FileSystem.cs rename to src/Serval/src/Serval.Shared/Services/FileSystem.cs index 8a0dddfda..00b96a5c1 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/FileSystem.cs +++ b/src/Serval/src/Serval.Shared/Services/FileSystem.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class FileSystem : IFileSystem { diff --git a/src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs b/src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs deleted file mode 100644 index 453f120b5..000000000 --- a/src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Serval.Health.V1; - -namespace Serval.Shared.Services; - -public class GrpcServiceHealthCheck(GrpcClientFactory grpcClientFactory) : IHealthCheck -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - HealthApi.HealthApiClient client = _grpcClientFactory.CreateClient( - $"{context.Registration.Name}-Health" - ); - HealthCheckResponse? healthCheckResponse = await client.HealthCheckAsync( - new Google.Protobuf.WellKnownTypes.Empty(), - cancellationToken: cancellationToken - ); - if (healthCheckResponse is null) - { - return HealthCheckResult.Unhealthy( - $"Health check for {context.Registration.Name} failed with response null" - ); - } - // map health report to health check result - HealthCheckResult healthCheckResult = new( - status: (HealthStatus)healthCheckResponse.Status, - description: context.Registration.Name, - exception: string.IsNullOrEmpty(healthCheckResponse.Error) - ? null - : new HealthCheckException(healthCheckResponse.Error), - data: healthCheckResponse.Data.ToDictionary(kvp => kvp.Key, kvp => (object)kvp.Value) - ); - return healthCheckResult; - } -} - -public class HealthCheckException(string? message) : Exception(message) { } diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs b/src/Serval/src/Serval.Shared/Services/HangfireHealthCheck.cs similarity index 95% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs rename to src/Serval/src/Serval.Shared/Services/HangfireHealthCheck.cs index 0beeff334..6eba340de 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs +++ b/src/Serval/src/Serval.Shared/Services/HangfireHealthCheck.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class HangfireHealthCheck(JobStorage jobStorage, IOptions options) : IHealthCheck { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IFileSystem.cs b/src/Serval/src/Serval.Shared/Services/IFileSystem.cs similarity index 83% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IFileSystem.cs rename to src/Serval/src/Serval.Shared/Services/IFileSystem.cs index 8f1823772..55d93f5a5 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IFileSystem.cs +++ b/src/Serval/src/Serval.Shared/Services/IFileSystem.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public interface IFileSystem { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IZipContainer.cs b/src/Serval/src/Serval.Shared/Services/IZipContainer.cs similarity index 79% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IZipContainer.cs rename to src/Serval/src/Serval.Shared/Services/IZipContainer.cs index e3431e6b8..7c0beac1f 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IZipContainer.cs +++ b/src/Serval/src/Serval.Shared/Services/IZipContainer.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public interface IZipContainer : IDisposable { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusService.cs b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs similarity index 56% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusService.cs rename to src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs index a07194f4a..de0c1b3c7 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs @@ -1,8 +1,7 @@ -using System.Globalization; -using SIL.Machine.Translation; +using SIL.Machine.PunctuationAnalysis; using SIL.Scripture; -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class ParallelCorpusService : IParallelCorpusService { @@ -11,25 +10,25 @@ public class ParallelCorpusService : IParallelCorpusService public IReadOnlyList<( string ParallelCorpusId, string MonolingualCorpusId, - IReadOnlyList Errors - )> AnalyzeUsfmVersification(IEnumerable parallelCorpora) + IReadOnlyList Errors + )> AnalyzeUsfmVersification(IEnumerable parallelCorpora) { CorpusBundle corpusBundle = new(parallelCorpora); List<( string ParallelCorpusId, string MonolingualCorpusId, - IReadOnlyList Errors + IReadOnlyList Errors )> errorsPerCorpus = []; foreach ( ( - ParallelCorpus parallelCorpus, - MonolingualCorpus monolingualCorpus, - IReadOnlyList files, + ParallelCorpusContract parallelCorpus, + MonolingualCorpusContract monolingualCorpus, + IReadOnlyList files, _ ) in corpusBundle.TextCorpora ) { - foreach (CorpusFile file in files.Where(f => f.Format == FileFormat.Paratext)) + foreach (CorpusFileContract file in files.Where(f => f.Format == FileFormat.Paratext)) { using ZipArchive zipArchive = ZipFile.OpenRead(file.Location); IReadOnlyList errors = new ZipParatextProjectVersificationErrorDetector( @@ -38,27 +37,73 @@ IReadOnlyList Errors ).GetUsfmVersificationErrors(books: GetBooks(monolingualCorpus)); if (errors.Count > 0) { - errorsPerCorpus.Add((parallelCorpus.Id, monolingualCorpus.Id, errors)); + errorsPerCorpus.Add( + ( + parallelCorpus.Id, + monolingualCorpus.Id, + errors + .Select(e => new UsfmVersificationErrorContract + { + Type = Map(e.Type), + ProjectName = e.ProjectName, + ExpectedVerseRef = e.ExpectedVerseRef, + ActualVerseRef = e.ActualVerseRef, + }) + .ToList() + ) + ); } } } return errorsPerCorpus; } - public QuoteConventionAnalysis AnalyzeTargetQuoteConvention(IEnumerable parallelCorpora) + private static Contracts.UsfmVersificationErrorType Map(SIL.Machine.Corpora.UsfmVersificationErrorType type) + { + return type switch + { + SIL.Machine.Corpora.UsfmVersificationErrorType.MissingChapter => Contracts + .UsfmVersificationErrorType + .MissingChapter, + SIL.Machine.Corpora.UsfmVersificationErrorType.MissingVerse => Contracts + .UsfmVersificationErrorType + .MissingVerse, + SIL.Machine.Corpora.UsfmVersificationErrorType.ExtraVerse => Contracts + .UsfmVersificationErrorType + .ExtraVerse, + SIL.Machine.Corpora.UsfmVersificationErrorType.InvalidVerseRange => Contracts + .UsfmVersificationErrorType + .InvalidVerseRange, + SIL.Machine.Corpora.UsfmVersificationErrorType.MissingVerseSegment => Contracts + .UsfmVersificationErrorType + .MissingVerseSegment, + SIL.Machine.Corpora.UsfmVersificationErrorType.ExtraVerseSegment => Contracts + .UsfmVersificationErrorType + .ExtraVerseSegment, + SIL.Machine.Corpora.UsfmVersificationErrorType.InvalidChapterNumber => Contracts + .UsfmVersificationErrorType + .InvalidChapterNumber, + SIL.Machine.Corpora.UsfmVersificationErrorType.InvalidVerseNumber => Contracts + .UsfmVersificationErrorType + .InvalidVerseNumber, + _ => throw new InvalidOperationException($"Unknown USFM versification error type: {type}"), + }; + } + + public string AnalyzeTargetQuoteConvention(IEnumerable parallelCorpora) { CorpusBundle corpusBundle = new(parallelCorpora); Dictionary> analyses = []; foreach ( ( - ParallelCorpus parallelCorpus, - MonolingualCorpus targetMonolingualCorpus, - IReadOnlyList corpusFiles, + ParallelCorpusContract parallelCorpus, + MonolingualCorpusContract targetMonolingualCorpus, + IReadOnlyList corpusFiles, _ ) in corpusBundle.TargetTextCorpora ) { - foreach (CorpusFile file in corpusFiles.Where(f => f.Format == FileFormat.Paratext)) + foreach (CorpusFileContract file in corpusFiles.Where(f => f.Format == FileFormat.Paratext)) { using ZipArchive zipArchive = ZipFile.OpenRead(file.Location); var quoteConventionDetector = new ZipParatextProjectQuoteConventionDetector( @@ -89,29 +134,30 @@ public QuoteConventionAnalysis AnalyzeTargetQuoteConvention(IEnumerable QuoteConventionAnalysis.CombineWithWeightedAverage(kvp.Value)).ToList() - ); + var analysis = QuoteConventionAnalysis.CombineWithWeightedAverage([ + .. analyses.Select(kvp => QuoteConventionAnalysis.CombineWithWeightedAverage(kvp.Value)), + ]); + return analysis?.BestQuoteConvention?.Name ?? string.Empty; } public IReadOnlyList<( string ParallelCorpusId, string MonolingualCorpusId, - MissingParentProjectError - )> FindMissingParentProjects(IEnumerable parallelCorpora) + MissingParentProjectErrorContract Error + )> FindMissingParentProjects(IEnumerable parallelCorpora) { CorpusBundle corpusBundle = new(parallelCorpora); - List<(string, string, MissingParentProjectError)> errors = []; + List<(string, string, MissingParentProjectErrorContract)> errors = []; foreach ( ( - ParallelCorpus parallelCorpus, - MonolingualCorpus monolingualCorpus, - IReadOnlyList files, + ParallelCorpusContract parallelCorpus, + MonolingualCorpusContract monolingualCorpus, + IReadOnlyList files, _ ) in corpusBundle.TextCorpora ) { - foreach (CorpusFile file in files) + foreach (CorpusFileContract file in files) { ParatextProjectSettings? settings = corpusBundle.GetSettings(file.Location); if (settings != null && settings.HasParent && settings.Parent == null) @@ -131,9 +177,9 @@ public QuoteConventionAnalysis AnalyzeTargetQuoteConvention(IEnumerable parallelCorpora, - Func train, - Func inference, + IEnumerable parallelCorpora, + Func train, + Func inference, bool useKeyTerms = false, HashSet? ignoreUsfmMarkers = null ) @@ -141,10 +187,10 @@ public async Task PreprocessAsync( await PreprocessAsync(new CorpusBundle(parallelCorpora), train, inference, useKeyTerms, ignoreUsfmMarkers); } - public async Task PreprocessAsync( + private static async Task PreprocessAsync( CorpusBundle corpusBundle, - Func train, - Func inference, + Func train, + Func inference, bool useKeyTerms = false, HashSet? ignoreUsfmMarkers = null ) @@ -152,7 +198,6 @@ public async Task PreprocessAsync( ignoreUsfmMarkers ??= []; bool parallelTrainingDataPresent = false; - List keyTermTrainingData = []; // Iterate over USFM and Text training corpora separately. // This is not only because they use different keys, but if we have text corpora @@ -196,14 +241,15 @@ .. corpusBundle.TargetTextCorpora.SelectMany(c => } // Align source and target training data - ParallelTextRow[] trainingRows = - [ - .. sourceTrainingCorpus.AlignRows(targetTrainingCorpus, allSourceRows: true, allTargetRows: true), - ]; + IEnumerable trainingRows = sourceTrainingCorpus.AlignRows( + targetTrainingCorpus, + allSourceRows: true, + allTargetRows: true + ); // After merging segments across ranges, run the 'train' preprocessing function // on each training row and record whether any parallel training data was present - foreach (Row row in CollapseRanges(trainingRows)) + foreach (ParallelRowContract row in CollapseRanges(trainingRows)) { await train(row, TrainingDataType.Text); if (!parallelTrainingDataPresent && row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) @@ -228,30 +274,9 @@ .. corpusBundle.TargetTextCorpora.SelectMany(c => targetCorpus = targetCorpus.Where(IsScriptureRow); } - if (useKeyTerms) - { - // Create a terms corpus for each corpus file - ITextCorpus[] sourceTermCorpora = corpusBundle.SourceTermCorpora.SelectMany(c => c.TextCorpora).ToArray(); - ITextCorpus[] targetTermCorpora = corpusBundle.TargetTermCorpora.SelectMany(c => c.TextCorpora).ToArray(); - - // As with scripture data, interlace the source rows randomly - // but choose the first non-empty target row, then align - IParallelTextCorpus parallelKeyTermCorpus = sourceTermCorpora - .ChooseRandom(Seed) - .AlignRows(targetTermCorpora.ChooseFirst()); - - // Only train on unique key terms pairs - foreach (ParallelTextRow row in parallelKeyTermCorpus.DistinctBy(row => (row.SourceText, row.TargetText))) - { - keyTermTrainingData.Add( - new Row(row.TextId, row.SourceRefs, row.TargetRefs, row.SourceText, row.TargetText, 1) - ); - } - } - // Since we ultimately need to provide inferences for a particular parallel corpus, // we need to preprocess the content on which to inference per parallel corpus - foreach (ParallelCorpus parallelCorpus in corpusBundle.ParallelCorpora) + foreach (ParallelCorpusContract parallelCorpus in corpusBundle.ParallelCorpora) { // Filter the text corpora based on the filters specified in the monolingual corpora ITextCorpus sourceInferencingCorpus = corpusBundle @@ -276,14 +301,14 @@ .. corpusBundle.TargetTextCorpora.SelectMany(c => // content for inferencing (the target is only needed in some contexts like word alignment) // as well as the target training corpus in order to determine whether a row was already // used in training. - INParallelTextCorpus inferencingCorpus = new ITextCorpus[] + IEnumerable inferencingCorpus = new ITextCorpus[] { sourceInferencingCorpus, targetInferencingCorpus, targetCorpus, }.AlignMany([true, false, false]); - foreach ((Row row, bool isInTrainingData) in CollapseInferencingRanges(inferencingCorpus.ToArray())) + foreach ((ParallelRowContract row, bool isInTrainingData) in CollapseInferencingRanges(inferencingCorpus)) { await inference(row, isInTrainingData, parallelCorpus.Id); } @@ -294,15 +319,36 @@ .. corpusBundle.TargetTextCorpora.SelectMany(c => // filtered by the filters specified in the monolingual corpora. if (useKeyTerms && parallelTrainingDataPresent) { - foreach (Row row in keyTermTrainingData) + // Create a terms corpus for each corpus file + ITextCorpus[] sourceTermCorpora = corpusBundle.SourceTermCorpora.SelectMany(c => c.TextCorpora).ToArray(); + ITextCorpus[] targetTermCorpora = corpusBundle.TargetTermCorpora.SelectMany(c => c.TextCorpora).ToArray(); + + // As with scripture data, interlace the source rows randomly + // but choose the first non-empty target row, then align + IParallelTextCorpus parallelKeyTermCorpus = sourceTermCorpora + .ChooseRandom(Seed) + .AlignRows(targetTermCorpora.ChooseFirst()); + + // Only train on unique key terms pairs + foreach (ParallelTextRow row in parallelKeyTermCorpus.DistinctBy(row => (row.SourceText, row.TargetText))) { - await train(row, TrainingDataType.KeyTerm); + await train( + new ParallelRowContract( + row.TextId, + row.SourceRefs, + row.TargetRefs, + row.SourceText, + row.TargetText, + 1 + ), + TrainingDataType.KeyTerm + ); } } } private static ITextCorpus FilterInferencingCorpora( - MonolingualCorpus corpus, + MonolingualCorpusContract corpus, ITextCorpus textCorpus, HashSet ignoreUsfmMarkers ) @@ -321,7 +367,7 @@ HashSet ignoreUsfmMarkers return textCorpus.Where(row => row.Ref is not ScriptureRef sr || !HasIgnorableMarker(sr, ignoreUsfmMarkers)); } - private static ITextCorpus FilterTrainingCorpora(MonolingualCorpus corpus, ITextCorpus textCorpus) + private static ITextCorpus FilterTrainingCorpora(MonolingualCorpusContract corpus, ITextCorpus textCorpus) { textCorpus = textCorpus.Transform(CleanSegment); if (corpus.TrainOnTextIds is not null) @@ -337,7 +383,7 @@ private static ITextCorpus FilterTrainingCorpora(MonolingualCorpus corpus, IText return textCorpus; } - private static IEnumerable CollapseRanges(ParallelTextRow[] rows) + private static IEnumerable CollapseRanges(IEnumerable rows) { StringBuilder srcSegBuffer = new(); StringBuilder trgSegBuffer = new(); @@ -354,7 +400,7 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) && (!row.IsSourceInRange || row.IsSourceRangeStart) ) { - yield return new Row( + yield return new ParallelRowContract( textId, sourceRefs, targetRefs, @@ -392,7 +438,14 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) continue; } - yield return new Row(textId, sourceRefs, targetRefs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); + yield return new ParallelRowContract( + textId, + sourceRefs, + targetRefs, + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + 1 + ); srcSegBuffer.Clear(); trgSegBuffer.Clear(); @@ -401,11 +454,20 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) } if (hasUnfinishedRange) { - yield return new Row(textId, sourceRefs, targetRefs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); + yield return new ParallelRowContract( + textId, + sourceRefs, + targetRefs, + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + 1 + ); } } - private static IEnumerable<(Row, bool)> CollapseInferencingRanges(NParallelTextRow[] rows) + private static IEnumerable<(ParallelRowContract, bool)> CollapseInferencingRanges( + IEnumerable rows + ) { StringBuilder srcSegBuffer = new(); StringBuilder trgSegBuffer = new(); @@ -426,7 +488,14 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) ) { yield return ( - new Row(textId, sourceRefs, targetRefs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1), + new ParallelRowContract( + textId, + sourceRefs, + targetRefs, + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + 1 + ), isInTrainingData ); @@ -463,7 +532,14 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) } yield return ( - new Row(textId, sourceRefs, targetRefs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1), + new ParallelRowContract( + textId, + sourceRefs, + targetRefs, + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + 1 + ), isInTrainingData ); @@ -476,7 +552,14 @@ private static IEnumerable CollapseRanges(ParallelTextRow[] rows) if (hasUnfinishedRange) { yield return ( - new Row(textId, sourceRefs, targetRefs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1), + new ParallelRowContract( + textId, + sourceRefs, + targetRefs, + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + 1 + ), isInTrainingData ); } @@ -506,7 +589,7 @@ private static TextRow CleanSegment(TextRow row) return row; } - private static HashSet? GetBooks(MonolingualCorpus corpus) + private static HashSet? GetBooks(MonolingualCorpusContract corpus) { if (!corpus.IsFiltered) return null; @@ -533,284 +616,8 @@ private static TextRow CleanSegment(TextRow row) return [.. books.Select(bookName => Canon.BookIdToNumber(bookName))]; } - public string UpdateSourceUsfm( - IReadOnlyList parallelCorpora, - string corpusId, - string bookId, - IReadOnlyList rows, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior embedBehavior, - UpdateUsfmMarkerBehavior styleBehavior, - bool placeParagraphMarkers, - IEnumerable? remarks, - string? targetQuoteConvention - ) - { - return UpdateUsfm( - parallelCorpora, - corpusId, - bookId, - rows, - UpdateUsfmTextBehavior.StripExisting, - paragraphBehavior, - embedBehavior, - styleBehavior, - placeParagraphMarkers ? [new PlaceMarkersUsfmUpdateBlockHandler()] : null, - remarks, - targetQuoteConvention, - isSource: true - ); - } - - public string UpdateTargetUsfm( - IReadOnlyList parallelCorpora, - string corpusId, - string bookId, - IReadOnlyList rows, - UpdateUsfmTextBehavior textBehavior, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior embedBehavior, - UpdateUsfmMarkerBehavior styleBehavior, - IEnumerable? remarks, - string? targetQuoteConvention - ) - { - return UpdateUsfm( - parallelCorpora, - corpusId, - bookId, - rows, - textBehavior, - paragraphBehavior, - embedBehavior, - styleBehavior, - updateBlockHandlers: null, - remarks, - targetQuoteConvention, - isSource: false - ); - } - - private static string UpdateUsfm( - IReadOnlyList parallelCorpora, - string corpusId, - string bookId, - IEnumerable rows, - UpdateUsfmTextBehavior textBehavior, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior embedBehavior, - UpdateUsfmMarkerBehavior styleBehavior, - IEnumerable? updateBlockHandlers, - IEnumerable? remarks, - string? targetQuoteConvention, - bool isSource - ) - { - CorpusBundle corpusBundle = new(parallelCorpora); - ParallelCorpus corpus = corpusBundle.ParallelCorpora.Single(c => c.Id == corpusId); - CorpusFile sourceFile = corpus.SourceCorpora[0].Files[0]; - CorpusFile targetFile = corpus.TargetCorpora[0].Files[0]; - ParatextProjectSettings? sourceSettings = corpusBundle.GetSettings(sourceFile.Location); - ParatextProjectSettings? targetSettings = corpusBundle.GetSettings(targetFile.Location); - - using ZipParatextProjectTextUpdater updater = corpusBundle.GetTextUpdater( - isSource ? sourceFile.Location : targetFile.Location - ); - string usfm = - updater.UpdateUsfm( - bookId, - rows.Select(p => - Map( - p, - isSource, - sourceSettings?.Versification, - targetSettings?.Versification, - paragraphBehavior, - styleBehavior - ) - ) - .Where(row => row.Refs.Any()) - .OrderBy(row => row.Refs[0]) - .ToArray(), - isSource ? sourceSettings?.FullName : targetSettings?.FullName, - textBehavior, - paragraphBehavior, - embedBehavior, - styleBehavior, - updateBlockHandlers: updateBlockHandlers, - remarks: remarks, - errorHandler: (_) => true, - compareSegments: isSource - ) ?? ""; - - if (!string.IsNullOrEmpty(targetQuoteConvention)) - usfm = DenormalizeQuotationMarks(usfm, targetQuoteConvention); - return usfm; - } - - private static UpdateUsfmRow Map( - ParallelRow row, - bool isSource, - ScrVers? sourceVersification, - ScrVers? targetVersification, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior styleBehavior - ) - { - Dictionary? metadata = null; - if (row.Alignment is not null) - { - metadata = new Dictionary - { - { - PlaceMarkersAlignmentInfo.MetadataKey, - new PlaceMarkersAlignmentInfo( - row.SourceTokens, - row.TargetTokens, - CreateWordAlignmentMatrix(row), - paragraphBehavior, - styleBehavior - ) - }, - }; - } - - ScriptureRef[] refs; - if (isSource) - { - refs = ( - row.SourceRefs.Any() - ? Map(row.SourceRefs, sourceVersification) - : Map(row.TargetRefs, targetVersification) - ).ToArray(); - } - else - { - // the pretranslations are generated from the source book and inserted into the target book - // use relaxed references since the USFM structure may not be the same - refs = Map(row.TargetRefs, targetVersification).Select(r => r.ToRelaxed()).ToArray(); - } - - return new UpdateUsfmRow(refs, row.TargetText, metadata); - } - - private static IEnumerable Map(IEnumerable refs, ScrVers? versification) - { - return refs.Select(r => - { - ScriptureRef.TryParse(r, versification, out ScriptureRef sr); - return sr; - }) - .Where(r => !r.IsEmpty); - } - - private static WordAlignmentMatrix? CreateWordAlignmentMatrix(ParallelRow row) - { - if (row.Alignment is null || row.SourceTokens is null || row.TargetTokens is null) - { - return null; - } - - var matrix = new WordAlignmentMatrix(row.SourceTokens.Count, row.TargetTokens.Count); - foreach (AlignedWordPair wordPair in row.Alignment) - matrix[wordPair.SourceIndex, wordPair.TargetIndex] = true; - - return matrix; - } - - private static string DenormalizeQuotationMarks(string usfm, string quoteConvention) - { - QuoteConvention targetQuoteConvention = QuoteConventions.Standard.GetQuoteConventionByName(quoteConvention); - if (targetQuoteConvention is null) - return usfm; - - QuotationMarkDenormalizationFirstPass quotationMarkDenormalizationFirstPass = new(targetQuoteConvention); - - UsfmParser.Parse(usfm, quotationMarkDenormalizationFirstPass); - List<(int ChapterNumber, QuotationMarkUpdateStrategy Strategy)> bestChapterStrategies = - quotationMarkDenormalizationFirstPass.FindBestChapterStrategies(); - - QuotationMarkDenormalizationUsfmUpdateBlockHandler quotationMarkDenormalizer = new( - targetQuoteConvention, - new QuotationMarkUpdateSettings( - chapterStrategies: bestChapterStrategies.Select(tuple => tuple.Strategy).ToList() - ) - ); - int denormalizableChapterCount = bestChapterStrategies.Count(tup => - tup.Strategy != QuotationMarkUpdateStrategy.Skip - ); - List remarks = []; - string quotationDenormalizationRemark; - if (denormalizableChapterCount == bestChapterStrategies.Count) - { - quotationDenormalizationRemark = - "The quote style in all chapters has been automatically adjusted to match the rest of the project."; - } - else if (denormalizableChapterCount > 0) - { - quotationDenormalizationRemark = - "The quote style in the following chapters has been automatically adjusted to match the rest of the project: " - + GetChapterRangesString( - bestChapterStrategies - .Where(tuple => tuple.Strategy != QuotationMarkUpdateStrategy.Skip) - .Select(tuple => tuple.ChapterNumber) - .ToList() - ) - + "."; - } - else - { - quotationDenormalizationRemark = - "The quote style was not automatically adjusted to match the rest of your project in any chapters."; - } - remarks.Add(quotationDenormalizationRemark); - - var updater = new UpdateUsfmParserHandler(updateBlockHandlers: [quotationMarkDenormalizer], remarks: remarks); - UsfmParser.Parse(usfm, updater); - - usfm = updater.GetUsfm(); - return usfm; - } - - public static string GetChapterRangesString(List chapterNumbers) - { - chapterNumbers = chapterNumbers.Order().ToList(); - int start = chapterNumbers[0]; - int end = chapterNumbers[0]; - List chapterRangeStrings = []; - foreach (int chapterNumber in chapterNumbers[1..]) - { - if (chapterNumber == end + 1) - { - end = chapterNumber; - } - else - { - if (start == end) - { - chapterRangeStrings.Add(start.ToString(CultureInfo.InvariantCulture)); - } - else - { - chapterRangeStrings.Add($"{start}-{end}"); - } - start = chapterNumber; - end = chapterNumber; - } - } - if (start == end) - { - chapterRangeStrings.Add(start.ToString(CultureInfo.InvariantCulture)); - } - else - { - chapterRangeStrings.Add($"{start}-{end}"); - } - return string.Join(", ", chapterRangeStrings); - } - public Dictionary> GetChapters( - IReadOnlyList parallelCorpora, + IReadOnlyList parallelCorpora, string fileLocation, string scriptureRange ) diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipContainer.cs b/src/Serval/src/Serval.Shared/Services/ZipContainer.cs similarity index 94% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipContainer.cs rename to src/Serval/src/Serval.Shared/Services/ZipContainer.cs index 506efb738..2ad006587 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipContainer.cs +++ b/src/Serval/src/Serval.Shared/Services/ZipContainer.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class ZipContainer(string fileName) : DisposableBase, IZipContainer { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectFileHandler.cs b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectFileHandler.cs similarity index 97% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectFileHandler.cs rename to src/Serval/src/Serval.Shared/Services/ZipParatextProjectFileHandler.cs index 79a9cd49d..cae8e9777 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectFileHandler.cs +++ b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectFileHandler.cs @@ -1,6 +1,4 @@ -using SIL.IO; - -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class ZipParatextProjectFileHandler(IZipContainer container) : IParatextProjectFileHandler { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectSettingsParser.cs b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs similarity index 86% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectSettingsParser.cs rename to src/Serval/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs index 5d3075d58..ca8ba9de8 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectSettingsParser.cs +++ b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class ZipParatextProjectSettingsParser( IZipContainer projectContainer, diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectTextUpdater.cs b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectTextUpdater.cs similarity index 95% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectTextUpdater.cs rename to src/Serval/src/Serval.Shared/Services/ZipParatextProjectTextUpdater.cs index 506c8dd2a..adcf0f79a 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ZipParatextProjectTextUpdater.cs +++ b/src/Serval/src/Serval.Shared/Services/ZipParatextProjectTextUpdater.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class ZipParatextProjectTextUpdater(IZipContainer projectContainer, ParatextProjectSettings? settings = null) : ParatextProjectTextUpdaterBase( diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs index a55e44177..b11f8428a 100644 --- a/src/Serval/src/Serval.Shared/Usings.cs +++ b/src/Serval/src/Serval.Shared/Usings.cs @@ -1,8 +1,13 @@ global using System.Diagnostics; +global using System.IO.Compression; +global using System.Reflection; +global using System.Text; global using System.Text.Json; global using System.Text.Json.Serialization; -global using Grpc.Core; -global using Grpc.Net.ClientFactory; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; @@ -10,8 +15,13 @@ global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.Diagnostics.HealthChecks; global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; global using Serval.Shared.Contracts; global using Serval.Shared.Models; +global using Serval.Shared.Services; global using Serval.Shared.Utils; global using SIL.DataAccess; +global using SIL.IO; +global using SIL.Machine.Corpora; +global using SIL.ObjectModel; diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/TaskEx.cs b/src/Serval/src/Serval.Shared/Utils/TaskEx.cs similarity index 97% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/TaskEx.cs rename to src/Serval/src/Serval.Shared/Utils/TaskEx.cs index bfc73e212..231aaa471 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/TaskEx.cs +++ b/src/Serval/src/Serval.Shared/Utils/TaskEx.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Utils; +namespace Serval.Shared.Utils; public static class TaskEx { diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs new file mode 100644 index 000000000..26c31b531 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -0,0 +1,12 @@ +namespace Serval.Translation.Contracts; + +public record ExecutionDataContract +{ + public int? TrainCount { get; init; } + public int? PretranslateCount { get; init; } + public IReadOnlyList? Warnings { get; init; } + public string? EngineSourceLanguageTag { get; init; } + public string? EngineTargetLanguageTag { get; init; } + public string? ResolvedSourceLanguage { get; init; } + public string? ResolvedTargetLanguage { get; init; } +} diff --git a/src/Serval/src/Serval.Translation.Contracts/IServalConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation.Contracts/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..76f98ae11 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/IServalConfiguratorExtensions.cs @@ -0,0 +1,14 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddTranslationEngine( + this IServalConfigurator configurator, + string engineType + ) + where TEngineService : class, ITranslationEngineService + { + configurator.Services.AddKeyedScoped(engineType.ToLowerInvariant()); + return configurator; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs b/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs similarity index 59% rename from src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs rename to src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs index ce358333e..f70d063dc 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs @@ -1,19 +1,16 @@ -namespace Serval.Machine.Shared.Services; +namespace Serval.Translation.Contracts; public interface ITranslationEngineService { - EngineType Type { get; } - Task CreateAsync( string engineId, - string? engineName, string sourceLanguage, string targetLanguage, + string? engineName = null, bool? isModelPersisted = null, CancellationToken cancellationToken = default ); Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - Task UpdateAsync( string engineId, string? sourceLanguage, @@ -21,15 +18,17 @@ Task UpdateAsync( CancellationToken cancellationToken = default ); - Task> TranslateAsync( + Task> TranslateAsync( string engineId, int n, string segment, CancellationToken cancellationToken = default ); - - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - + Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ); Task TrainSegmentPairAsync( string engineId, string sourceSegment, @@ -37,20 +36,21 @@ Task TrainSegmentPairAsync( bool sentenceStart, CancellationToken cancellationToken = default ); + Task GetModelDownloadUrlAsync( + string engineId, + CancellationToken cancellationToken = default + ); Task StartBuildAsync( string engineId, string buildId, - string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, + string? options = null, CancellationToken cancellationToken = default ); Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); - - int GetQueueSize(); - - bool IsLanguageNativeToModel(string language, out string internalCode); + Task GetQueueSizeAsync(CancellationToken cancellationToken = default); + Task GetLanguageInfoAsync(string language, CancellationToken cancellationToken = default); } diff --git a/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs b/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs new file mode 100644 index 000000000..23f4621ff --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs @@ -0,0 +1,43 @@ +namespace Serval.Translation.Contracts; + +public interface ITranslationPlatformService +{ + Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildCompletedAsync( + string buildId, + int corpusSize, + double confidence, + CancellationToken cancellationToken = default + ); + Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); + Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); + Task UpdateBuildStatusAsync( + string buildId, + BuildProgressStatusContract progressStatus, + int? queueDepth = null, + IReadOnlyCollection? phases = null, + DateTime? started = null, + DateTime? completed = null, + CancellationToken cancellationToken = default + ); + Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default); + Task IncrementEngineCorpusSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); + Task InsertPretranslationsAsync( + string engineId, + IAsyncEnumerable pretranslations, + CancellationToken cancellationToken = default + ); + Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + ExecutionDataContract executionData, + CancellationToken cancellationToken = default + ); + Task UpdateTargetQuoteConventionAsync( + string engineId, + string buildId, + string quoteConvention, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.Translation.Contracts/LanguageInfoContract.cs b/src/Serval/src/Serval.Translation.Contracts/LanguageInfoContract.cs new file mode 100644 index 000000000..3dc96bd2d --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/LanguageInfoContract.cs @@ -0,0 +1,7 @@ +namespace Serval.Translation.Contracts; + +public record LanguageInfoContract +{ + public required bool IsNative { get; set; } + public string? InternalCode { get; set; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs b/src/Serval/src/Serval.Translation.Contracts/ModelDownloadUrlContract.cs similarity index 65% rename from src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs rename to src/Serval/src/Serval.Translation.Contracts/ModelDownloadUrlContract.cs index 798fe1757..1380546ee 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ModelDownloadUrlContract.cs @@ -1,6 +1,6 @@ -namespace Serval.Machine.Shared.Models; +namespace Serval.Translation.Contracts; -public record ModelDownloadUrl +public record ModelDownloadUrlContract { public required string Url { get; init; } public required int ModelRevision { get; init; } diff --git a/src/Serval/src/Serval.Translation/Models/Phrase.cs b/src/Serval/src/Serval.Translation.Contracts/PhraseContract.cs similarity index 71% rename from src/Serval/src/Serval.Translation/Models/Phrase.cs rename to src/Serval/src/Serval.Translation.Contracts/PhraseContract.cs index cbaf7149a..67eacc9fd 100644 --- a/src/Serval/src/Serval.Translation/Models/Phrase.cs +++ b/src/Serval/src/Serval.Translation.Contracts/PhraseContract.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Contracts; -public record Phrase +public record PhraseContract { public required int SourceSegmentStart { get; set; } public required int SourceSegmentEnd { get; set; } diff --git a/src/Serval/src/Serval.Translation.Contracts/PretranslationContract.cs b/src/Serval/src/Serval.Translation.Contracts/PretranslationContract.cs new file mode 100644 index 000000000..cd0cdc013 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/PretranslationContract.cs @@ -0,0 +1,14 @@ +namespace Serval.Translation.Contracts; + +public record PretranslationContract +{ + public required string CorpusId { get; init; } + public required string TextId { get; init; } + public required IReadOnlyList SourceRefs { get; init; } + public required IReadOnlyList TargetRefs { get; init; } + public required string Translation { get; init; } + public IReadOnlyList? SourceTokens { get; init; } + public IReadOnlyList? TranslationTokens { get; init; } + public IReadOnlyList? Alignment { get; init; } + public double? Confidence { get; init; } +} diff --git a/src/Serval/src/Serval.Translation.Contracts/Serval.Translation.Contracts.csproj b/src/Serval/src/Serval.Translation.Contracts/Serval.Translation.Contracts.csproj new file mode 100644 index 000000000..f4041dd28 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/Serval.Translation.Contracts.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + diff --git a/src/Serval/src/Serval.Translation.Contracts/TranslationBuildFinished.cs b/src/Serval/src/Serval.Translation.Contracts/TranslationBuildFinished.cs new file mode 100644 index 000000000..9a6f51338 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/TranslationBuildFinished.cs @@ -0,0 +1,10 @@ +namespace Serval.Translation.Contracts; + +public record TranslationBuildFinished( + string BuildId, + string EngineId, + string Owner, + JobState BuildState, + string Message, + DateTime DateFinished +) : IEvent; diff --git a/src/Serval/src/Serval.Translation.Contracts/TranslationBuildStarted.cs b/src/Serval/src/Serval.Translation.Contracts/TranslationBuildStarted.cs new file mode 100644 index 000000000..315ab4b06 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/TranslationBuildStarted.cs @@ -0,0 +1,3 @@ +namespace Serval.Translation.Contracts; + +public record TranslationBuildStarted(string BuildId, string EngineId, string Owner) : IEvent; diff --git a/src/Serval/src/Serval.Translation/Models/TranslationResult.cs b/src/Serval/src/Serval.Translation.Contracts/TranslationResultContract.cs similarity index 60% rename from src/Serval/src/Serval.Translation/Models/TranslationResult.cs rename to src/Serval/src/Serval.Translation.Contracts/TranslationResultContract.cs index bcd11a761..0cb91435a 100644 --- a/src/Serval/src/Serval.Translation/Models/TranslationResult.cs +++ b/src/Serval/src/Serval.Translation.Contracts/TranslationResultContract.cs @@ -1,12 +1,12 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Contracts; -public record TranslationResult +public record TranslationResultContract { public required string Translation { get; set; } public required IReadOnlyList SourceTokens { get; set; } public required IReadOnlyList TargetTokens { get; set; } public required IReadOnlyList Confidences { get; set; } public required IReadOnlyList> Sources { get; set; } - public required IReadOnlyList Alignment { get; set; } - public required IReadOnlyList Phrases { get; set; } + public required IReadOnlyList Alignment { get; set; } + public required IReadOnlyList Phrases { get; set; } } diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationSource.cs b/src/Serval/src/Serval.Translation.Contracts/TranslationSource.cs similarity index 66% rename from src/Serval/src/Serval.Translation/Contracts/TranslationSource.cs rename to src/Serval/src/Serval.Translation.Contracts/TranslationSource.cs index 468cfa2bd..b1b9d17c0 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationSource.cs +++ b/src/Serval/src/Serval.Translation.Contracts/TranslationSource.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Contracts; public enum TranslationSource { diff --git a/src/Serval/src/Serval.Translation.Contracts/Usings.cs b/src/Serval/src/Serval.Translation.Contracts/Usings.cs new file mode 100644 index 000000000..1af4e177e --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/Usings.cs @@ -0,0 +1,2 @@ +global using Serval.Shared.Contracts; +global using Serval.Translation.Contracts; diff --git a/src/Serval/src/Serval.Translation/Models/WordGraphArc.cs b/src/Serval/src/Serval.Translation.Contracts/WordGraphArcContract.cs similarity index 75% rename from src/Serval/src/Serval.Translation/Models/WordGraphArc.cs rename to src/Serval/src/Serval.Translation.Contracts/WordGraphArcContract.cs index bbc238aee..801ed0876 100644 --- a/src/Serval/src/Serval.Translation/Models/WordGraphArc.cs +++ b/src/Serval/src/Serval.Translation.Contracts/WordGraphArcContract.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Contracts; -public record WordGraphArc +public record WordGraphArcContract { public required int PrevState { get; set; } public required int NextState { get; set; } @@ -9,6 +9,6 @@ public record WordGraphArc public required IReadOnlyList Confidences { get; set; } public required int SourceSegmentStart { get; set; } public required int SourceSegmentEnd { get; set; } - public required IReadOnlyList Alignment { get; set; } + public required IReadOnlyList Alignment { get; set; } public required IReadOnlyList> Sources { get; set; } } diff --git a/src/Serval/src/Serval.Translation/Models/WordGraph.cs b/src/Serval/src/Serval.Translation.Contracts/WordGraphContract.cs similarity index 57% rename from src/Serval/src/Serval.Translation/Models/WordGraph.cs rename to src/Serval/src/Serval.Translation.Contracts/WordGraphContract.cs index 0afd4d21b..8053d1c0a 100644 --- a/src/Serval/src/Serval.Translation/Models/WordGraph.cs +++ b/src/Serval/src/Serval.Translation.Contracts/WordGraphContract.cs @@ -1,9 +1,9 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Contracts; -public record WordGraph +public record WordGraphContract { public required IReadOnlyList SourceTokens { get; set; } public required double InitialStateScore { get; set; } public required IReadOnlySet FinalStates { get; set; } - public required IReadOnlyList Arcs { get; set; } + public required IReadOnlyList Arcs { get; set; } } diff --git a/src/Serval/src/Serval.Translation/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IEndpointRouteBuilderExtensions.cs deleted file mode 100644 index d2faf24af..000000000 --- a/src/Serval/src/Serval.Translation/Configuration/IEndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Microsoft.AspNetCore.Builder; - -public static class IEndpointRouteBuilderExtensions -{ - public static IEndpointRouteBuilder MapServalTranslationServices(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - - return builder; - } -} diff --git a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs deleted file mode 100644 index d9b91b0f1..000000000 --- a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMediatorRegistrationConfiguratorExtensions -{ - public static IMediatorRegistrationConfigurator AddTranslationConsumers( - this IMediatorRegistrationConfigurator configurator - ) - { - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs deleted file mode 100644 index fa8586409..000000000 --- a/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMemoryDataAccessConfiguratorExtensions -{ - public static IMemoryDataAccessConfigurator AddTranslationRepositories( - this IMemoryDataAccessConfigurator configurator - ) - { - configurator.AddRepository(); - configurator.AddRepository(); - configurator.AddRepository(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs deleted file mode 100644 index 782699663..000000000 --- a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Serval.Health.V1; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServalBuilderExtensions -{ - public static IServalBuilder AddTranslation(this IServalBuilder builder) - { - builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); - builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); - - builder.Services.AddParallelCorpusService(); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - builder.Services.Configure(builder.Configuration.GetSection(TranslationOptions.Key)); - var translationOptions = new TranslationOptions(); - builder.Configuration.GetSection(TranslationOptions.Key).Bind(translationOptions); - - foreach (EngineInfo engine in translationOptions.Engines) - { - builder.Services.AddGrpcClient( - engine.Type, - o => o.Address = new Uri(engine.Address) - ); - builder.Services.AddGrpcClient( - $"{engine.Type}-Health", - o => o.Address = new Uri(engine.Address) - ); - builder.Services.AddHealthChecks().AddCheck(engine.Type); - } - - builder.Services.AddOutbox(x => - { - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - }); - - return builder; - } -} diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs similarity index 81% rename from src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs rename to src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs index 5c6dbed3e..c1d7731d0 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs @@ -1,15 +1,27 @@ -using MongoDB.Bson; -using MongoDB.Driver; - namespace Microsoft.Extensions.DependencyInjection; -public static class IMongoDataAccessConfiguratorExtensions +public static class IServalConfiguratorExtensions { - public static IMongoDataAccessConfigurator AddTranslationRepositories( - this IMongoDataAccessConfigurator configurator - ) + public static IServalConfigurator AddTranslation(this IServalConfigurator configurator) { - configurator.AddRepository( + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + + configurator.AddTranslationDataAccess(); + + configurator.AddHandlers(Assembly.GetExecutingAssembly()); + + return configurator; + } + + public static IServalConfigurator AddTranslationDataAccess(this IServalConfigurator configurator) + { + configurator.DataAccess.AddRepository( "translation.engines", init: [ @@ -25,11 +37,11 @@ this IMongoDataAccessConfigurator configurator c => c.UpdateManyAsync( Builders.Filter.Exists(e => e.ParallelCorpora, false), - Builders.Update.Set(e => e.ParallelCorpora, []) + Builders.Update.Set(e => e.ParallelCorpora, new List()) ), ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "translation.builds", init: [ @@ -76,7 +88,7 @@ this IMongoDataAccessConfigurator configurator MongoMigrations.MigrateTargetQuoteConvention, ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "translation.pretranslations", init: [ @@ -118,6 +130,7 @@ this IMongoDataAccessConfigurator configurator ), ] ); + return configurator; } } diff --git a/src/Serval/src/Serval.Translation/Configuration/MongoMigrations.cs b/src/Serval/src/Serval.Translation/Configuration/MongoMigrations.cs index aa5b1a19a..99dd5f736 100644 --- a/src/Serval/src/Serval.Translation/Configuration/MongoMigrations.cs +++ b/src/Serval/src/Serval.Translation/Configuration/MongoMigrations.cs @@ -1,6 +1,3 @@ -using MongoDB.Bson; -using MongoDB.Driver; - namespace Serval.Translation.Configuration; public class MongoMigrations diff --git a/src/Serval/src/Serval.Translation/Configuration/TranslationOptions.cs b/src/Serval/src/Serval.Translation/Configuration/TranslationOptions.cs deleted file mode 100644 index 0c558c18c..000000000 --- a/src/Serval/src/Serval.Translation/Configuration/TranslationOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Translation.Configuration; - -public class TranslationOptions -{ - public const string Key = "Translation"; - - public List Engines { get; set; } = new List(); -} - -public class EngineInfo -{ - public string Type { get; set; } = ""; - public string Address { get; set; } = ""; -} diff --git a/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs deleted file mode 100644 index b096d30b4..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace Serval.Translation.Consumers; - -public class CorpusUpdatedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.UpdateCorpusFilesAsync( - context.Message.CorpusId, - context.Message.Files.Select(Map).ToList(), - context.CancellationToken - ); - } - - private static CorpusFile Map(CorpusFileResult corpusFile) - { - return new CorpusFile - { - Id = corpusFile.File.DataFileId, - TextId = corpusFile.TextId ?? corpusFile.File.Name, - Filename = corpusFile.File.Filename, - Format = corpusFile.File.Format, - }; - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/DataFileDeletedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/DataFileDeletedConsumer.cs deleted file mode 100644 index fdaaff429..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/DataFileDeletedConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Translation.Consumers; - -public class DataFileDeletedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.DeleteAllCorpusFilesAsync(context.Message.DataFileId, context.CancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs deleted file mode 100644 index b75572f7c..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Serval.Translation.Consumers; - -public class DataFileUpdatedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.UpdateDataFileFilenameFilesAsync( - context.Message.DataFileId, - context.Message.Filename, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/EngineCancelBuildConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/EngineCancelBuildConsumer.cs deleted file mode 100644 index 5e81517e5..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/EngineCancelBuildConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Translation.Consumers; - -public class EngineCancelBuildConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.CancelBuild) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - CancelBuildRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.CancelBuildAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/EngineCreateConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/EngineCreateConsumer.cs deleted file mode 100644 index 484104a21..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/EngineCreateConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Translation.Consumers; - -public class EngineCreateConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.Create) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - CreateRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.CreateAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/EngineDeleteConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/EngineDeleteConsumer.cs deleted file mode 100644 index 24ad538c8..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/EngineDeleteConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Translation.Consumers; - -public class EngineDeleteConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.Delete) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - DeleteRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.DeleteAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/EngineStartBuildConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/EngineStartBuildConsumer.cs deleted file mode 100644 index 43410b3e0..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/EngineStartBuildConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Translation.Consumers; - -public class EngineStartBuildConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.StartBuild) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - StartBuildRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.StartBuildAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Consumers/EngineUpdateConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/EngineUpdateConsumer.cs deleted file mode 100644 index 164e79377..000000000 --- a/src/Serval/src/Serval.Translation/Consumers/EngineUpdateConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Translation.Consumers; - -public class EngineUpdateConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.Update) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - UpdateRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.UpdateAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/LanguageInfoDto.cs b/src/Serval/src/Serval.Translation/Contracts/LanguageInfoDto.cs deleted file mode 100644 index ab200ad12..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/LanguageInfoDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record LanguageInfoDto -{ - public required string EngineType { get; init; } - public required bool IsNative { get; init; } - public string? InternalCode { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/ModelDownloadUrlDto.cs b/src/Serval/src/Serval.Translation/Contracts/ModelDownloadUrlDto.cs deleted file mode 100644 index cd05cd85c..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/ModelDownloadUrlDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Contracts; - -public class ModelDownloadUrlDto -{ - public string Url { get; set; } = default!; - public int ModelRevision { get; set; } = default!; - public DateTime ExpiresAt { get; set; } = default!; -} diff --git a/src/Serval/src/Serval.Translation/Contracts/PhraseDto.cs b/src/Serval/src/Serval.Translation/Contracts/PhraseDto.cs deleted file mode 100644 index 72446c9ed..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/PhraseDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record PhraseDto -{ - public required int SourceSegmentStart { get; init; } - public required int SourceSegmentEnd { get; init; } - public required int TargetSegmentCut { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationFormat.cs b/src/Serval/src/Serval.Translation/Contracts/PretranslationFormat.cs deleted file mode 100644 index 8a54f2b01..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationFormat.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Translation.Contracts; - -public enum PretranslationFormat -{ - Json, - Usfm, -} diff --git a/src/Serval/src/Serval.Translation/Contracts/SegmentPairDto.cs b/src/Serval/src/Serval.Translation/Contracts/SegmentPairDto.cs deleted file mode 100644 index 01fc0993c..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/SegmentPairDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record SegmentPairDto -{ - public required string SourceSegment { get; init; } - public required string TargetSegment { get; init; } - public required bool SentenceStart { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs deleted file mode 100644 index e80242168..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record TranslationBuildConfigDto -{ - public string? Name { get; init; } - public IReadOnlyList? TrainOn { get; init; } - public IReadOnlyList? Pretranslate { get; init; } - - /// - /// { - /// "property" : "value" - /// } - /// - public object? Options { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationEngineConfigDto.cs deleted file mode 100644 index 96ed6d5ed..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineConfigDto.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record TranslationEngineConfigDto -{ - /// - /// The translation engine name. - /// - public string? Name { get; init; } - - /// - /// The source language tag. - /// - public required string SourceLanguage { get; init; } - - /// - /// The target language tag. - /// - public required string TargetLanguage { get; init; } - - /// - /// The translation engine type. - /// - public required string Type { get; init; } - - /// - /// The model is saved when built and can be retrieved. - /// - public bool? IsModelPersisted { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineUpdateConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationEngineUpdateConfigDto.cs deleted file mode 100644 index 561e45490..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineUpdateConfigDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Contracts; - -public class TranslationEngineUpdateConfigDto -{ - public string? SourceLanguage { get; init; } - - public string? TargetLanguage { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationResultDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationResultDto.cs deleted file mode 100644 index 940eaaefa..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationResultDto.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record TranslationResultDto -{ - public required string Translation { get; init; } - public required IReadOnlyList SourceTokens { get; init; } - public required IReadOnlyList TargetTokens { get; init; } - public required IReadOnlyList Confidences { get; init; } - public required IReadOnlyList> Sources { get; init; } - public required IReadOnlyList Alignment { get; init; } - public required IReadOnlyList Phrases { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/WordGraphArcDto.cs b/src/Serval/src/Serval.Translation/Contracts/WordGraphArcDto.cs deleted file mode 100644 index a368c0fe8..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/WordGraphArcDto.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record WordGraphArcDto -{ - public required int PrevState { get; init; } - public required int NextState { get; init; } - public required double Score { get; init; } - public required IReadOnlyList TargetTokens { get; init; } - public required IReadOnlyList Confidences { get; init; } - public required int SourceSegmentStart { get; init; } - public required int SourceSegmentEnd { get; init; } - public required IReadOnlyList Alignment { get; init; } - public required IReadOnlyList> Sources { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/WordGraphDto.cs b/src/Serval/src/Serval.Translation/Contracts/WordGraphDto.cs deleted file mode 100644 index 627cb7504..000000000 --- a/src/Serval/src/Serval.Translation/Contracts/WordGraphDto.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record WordGraphDto -{ - public required IReadOnlyList SourceTokens { get; init; } - public required float InitialStateScore { get; init; } - public required ISet FinalStates { get; init; } - public required IReadOnlyList Arcs { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationBuildsController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationBuildsController.cs index 1a5ec9669..c8b0f05da 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationBuildsController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationBuildsController.cs @@ -6,9 +6,8 @@ namespace Serval.Translation.Controllers; public class TranslationBuildsController( IOptionsMonitor apiOptions, IAuthorizationService authService, - IBuildService buildService, - IUrlService urlService -) : TranslationControllerBase(authService, urlService) + IBuildService buildService +) : ServalControllerBase(authService) { /// /// Get all builds for your translation engines that are created after the specified date. @@ -27,9 +26,12 @@ IUrlService urlService [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> GetAllBuildsCreatedAfterAsync( [FromQuery(Name = "created-after")] DateTime? createdAfter, + [FromServices] DtoMapper mapper, CancellationToken cancellationToken ) { + if (createdAfter is not null) + createdAfter = DateTime.SpecifyKind(createdAfter.Value, DateTimeKind.Utc); IEnumerable builds; if (createdAfter is null) { @@ -39,7 +41,7 @@ CancellationToken cancellationToken { builds = await buildService.GetAllCreatedAfterAsync(Owner, createdAfter, cancellationToken); } - return builds.Select(Map); + return builds.Select(mapper.Map); } /// @@ -66,6 +68,7 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> GetNextFinishedBuildAsync( [FromQuery(Name = "finished-after")] DateTime finishedAfter, + [FromServices] DtoMapper mapper, CancellationToken cancellationToken ) { @@ -77,7 +80,7 @@ CancellationToken cancellationToken return change.Type switch { EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), - _ => change.Entity is null ? StatusCode(StatusCodes.Status408RequestTimeout) : Map(change.Entity), + _ => change.Entity is null ? StatusCode(StatusCodes.Status408RequestTimeout) : mapper.Map(change.Entity), }; } } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs deleted file mode 100644 index b967b78c6..000000000 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs +++ /dev/null @@ -1,102 +0,0 @@ -namespace Serval.Translation.Controllers; - -[ApiVersion(1.0)] -[Route("api/v{version:apiVersion}/translation/engine-types")] -[OpenApiTag("Translation Engines")] -public class TranslationEngineTypesController(IAuthorizationService authService, IEngineService engineService) - : ServalControllerBase(authService) -{ - private readonly IEngineService _engineService = engineService; - - /// - /// Get queue information for a given engine type - /// - /// A valid engine type: smt-transfer, nmt, or echo - /// - /// Queue information for the specified engine type - /// The client is not authenticated - /// The authenticated client cannot perform the operation - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{engineType}/queues")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetQueueAsync( - [NotNull] string engineType, - CancellationToken cancellationToken - ) - { - try - { - return Map( - await _engineService.GetQueueAsync(engineType.ToPascalCase(), cancellationToken: cancellationToken) - ); - } - catch (InvalidOperationException ioe) - { - return BadRequest(ioe.Message); - } - } - - /// - /// Get information regarding a language for a given engine type - /// - /// - /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally - /// and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. - /// **Base Models available** - /// * **NLLB-200**: This is the only base NMT translation model currently available. - /// * The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). - /// Response format: - /// * **`engineType`**: See above - /// * **`isNative`**: Whether the base translation model supports this language without fine-tuning. - /// * **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). - /// - /// A valid engine type: nmt, echo, or smt-transfer - /// The language to retrieve information on. - /// - /// Language information for the specified engine type - /// The client is not authenticated - /// The authenticated client cannot perform the operation - /// The method is not supported - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{engineType}/languages/{language}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - public async Task> GetLanguageInfoAsync( - [NotNull] string engineType, - [NotNull] string language, - CancellationToken cancellationToken - ) - { - try - { - return Map( - await _engineService.GetLanguageInfoAsync( - engineType: engineType.ToPascalCase(), - language: language, - cancellationToken: cancellationToken - ) - ); - } - catch (InvalidOperationException ioe) - { - return BadRequest(ioe.Message); - } - } - - private static QueueDto Map(Queue source) => - new() { Size = source.Size, EngineType = source.EngineType.ToKebabCase() }; - - private static LanguageInfoDto Map(LanguageInfo source) => - new() - { - EngineType = source.EngineType.ToKebabCase(), - IsNative = source.IsNative, - InternalCode = source.InternalCode, - }; -} diff --git a/src/Serval/src/Serval.Translation/Contracts/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs similarity index 91% rename from src/Serval/src/Serval.Translation/Contracts/ExecutionDataDto.cs rename to src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index 9b773c417..46384ed91 100644 --- a/src/Serval/src/Serval.Translation/Contracts/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record ExecutionDataDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs b/src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs similarity index 87% rename from src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs rename to src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs index dc609ce18..8faf0e94a 100644 --- a/src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record ParallelCorpusAnalysisDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusConfigDto.cs b/src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusConfigDto.cs similarity index 89% rename from src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusConfigDto.cs rename to src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusConfigDto.cs index 58756e3a7..6158abcfa 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record PretranslateCorpusConfigDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusDto.cs b/src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusDto.cs similarity index 89% rename from src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusDto.cs rename to src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusDto.cs index 14fde7161..ecc4eaeba 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslateCorpusDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/PretranslateCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record PretranslateCorpusDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationDto.cs b/src/Serval/src/Serval.Translation/Dtos/PretranslationDto.cs similarity index 90% rename from src/Serval/src/Serval.Translation/Contracts/PretranslationDto.cs rename to src/Serval/src/Serval.Translation/Dtos/PretranslationDto.cs index d64877418..524ba03b1 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/PretranslationDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record PretranslationDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs similarity index 97% rename from src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs index e6770f98d..90a3a0b35 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationBuildDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusConfigDto.cs similarity index 91% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationCorpusConfigDto.cs index d865a4c92..e0cd06261 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationCorpusConfigDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusDto.cs similarity index 92% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationCorpusDto.cs index da770f539..a1cc728cd 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationCorpusDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileConfigDto.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileConfigDto.cs index e9b967ec2..aaedf01f9 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationCorpusFileConfigDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileDto.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileDto.cs index f57874084..c2c7a8d09 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusFileDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationCorpusFileDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusUpdateConfigDto.cs similarity index 94% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationCorpusUpdateConfigDto.cs index 8dd963112..28687f489 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationCorpusUpdateConfigDto.cs @@ -1,6 +1,6 @@ using System.ComponentModel.DataAnnotations; -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationCorpusUpdateConfigDto : IValidatableObject { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationEngineDto.cs similarity index 93% rename from src/Serval/src/Serval.Translation/Contracts/TranslationEngineDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationEngineDto.cs index 522ae1656..615cfb936 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationEngineDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationEngineDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationEngineDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusConfigDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusConfigDto.cs similarity index 89% rename from src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusConfigDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusConfigDto.cs index b043c758b..1f0ed1323 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationParallelCorpusConfigDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusDto.cs similarity index 89% rename from src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusDto.cs index 063da1d29..9677bb897 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationParallelCorpusDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusUpdateDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusUpdateDto.cs similarity index 94% rename from src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusUpdateDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusUpdateDto.cs index b47447d07..9eccfd6b4 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationParallelCorpusUpdateDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationParallelCorpusUpdateDto.cs @@ -1,6 +1,6 @@ using System.ComponentModel.DataAnnotations; -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Dtos; public record TranslationParallelCorpusUpdateConfigDto : IValidatableObject { diff --git a/src/Serval/src/Serval.Translation/Features/EngineTypes/GetLanguageInfo.cs b/src/Serval/src/Serval.Translation/Features/EngineTypes/GetLanguageInfo.cs new file mode 100644 index 000000000..e83fdc5d9 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/EngineTypes/GetLanguageInfo.cs @@ -0,0 +1,79 @@ +namespace Serval.Translation.Features.EngineTypes; + +public record LanguageInfoDto +{ + public required string EngineType { get; init; } + public required bool IsNative { get; init; } + public string? InternalCode { get; init; } +} + +public record GetLanguageInfo(string EngineType, string Language) : IRequest; + +public record GetLanguageInfoResponse(LanguageInfoDto? LanguageInfo = null); + +public class GetLanguageInfoHandler(IEngineServiceFactory engineServiceFactory) + : IRequestHandler +{ + public async Task HandleAsync(GetLanguageInfo request, CancellationToken cancellationToken) + { + if (engineServiceFactory.TryGetEngineService(request.EngineType, out ITranslationEngineService? engineService)) + { + LanguageInfoContract languageInfo = await engineService.GetLanguageInfoAsync( + request.Language, + cancellationToken + ); + return new( + new LanguageInfoDto + { + EngineType = request.EngineType.ToKebabCase(), + InternalCode = languageInfo.InternalCode, + IsNative = languageInfo.IsNative, + } + ); + } + return new(); + } +} + +public partial class TranslationEngineTypesController +{ + /// + /// Get information regarding a language for a given engine type + /// + /// + /// This endpoint exists primarily to support `nmt` model-training since `echo` and `smt-transfer` engines support all languages equally. Given a language tag, it provides the ISO 639-3 code that the tag maps to internally + /// and whether it is supported in the NLLB 200 model without training. This is useful for determining if a language is a good candidate for a source language. + /// **Base Models available** + /// * **NLLB-200**: This is the only base NMT translation model currently available. + /// * The languages supported by the base model can be found [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md). + /// Response format: + /// * **`engineType`**: See above + /// * **`isNative`**: Whether the base translation model supports this language without fine-tuning. + /// * **`internalCode`**: The translation model's internal language code. See more details about how the language tag is mapped to an internal code [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt, echo, or smt-transfer + /// The language to retrieve information on. + /// + /// Language information for the specified engine type + /// The client is not authenticated + /// The authenticated client cannot perform the operation + /// The method is not supported + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{engineType}/languages/{language}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + public async Task> GetLanguageInfoAsync( + [NotNull] string engineType, + [NotNull] string language, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetLanguageInfoResponse response = await handler.HandleAsync(new(engineType, language), cancellationToken); + if (response.LanguageInfo is not null) + return Ok(response.LanguageInfo); + return NotFound(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/EngineTypes/GetQueue.cs b/src/Serval/src/Serval.Translation/Features/EngineTypes/GetQueue.cs new file mode 100644 index 000000000..028035b69 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/EngineTypes/GetQueue.cs @@ -0,0 +1,48 @@ +namespace Serval.Translation.Features.EngineTypes; + +public record GetQueue(string EngineType) : IRequest; + +public record GetQueueResponse(QueueDto? Queue = null); + +public class GetQueueHandler(IEngineServiceFactory engineServiceFactory) : IRequestHandler +{ + public async Task HandleAsync(GetQueue request, CancellationToken cancellationToken) + { + if (engineServiceFactory.TryGetEngineService(request.EngineType, out ITranslationEngineService? engineService)) + { + int size = await engineService.GetQueueSizeAsync(cancellationToken); + return new(new QueueDto { EngineType = request.EngineType.ToKebabCase(), Size = size }); + } + return new(); + } +} + +public partial class TranslationEngineTypesController +{ + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: smt-transfer, nmt, or echo + /// + /// Queue information for the specified engine type + /// The client is not authenticated + /// The authenticated client cannot perform the operation + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{engineType}/queues")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetQueueAsync( + [NotNull] string engineType, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetQueueResponse response = await handler.HandleAsync(new(engineType), cancellationToken); + if (response.Queue is not null) + return Ok(response.Queue); + return NotFound(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/EngineTypes/TranslationEngineTypesController.cs b/src/Serval/src/Serval.Translation/Features/EngineTypes/TranslationEngineTypesController.cs new file mode 100644 index 000000000..6e55cbc0e --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/EngineTypes/TranslationEngineTypesController.cs @@ -0,0 +1,7 @@ +namespace Serval.Translation.Features.EngineTypes; + +[ApiVersion(1.0)] +[Route("api/v{version:apiVersion}/translation/engine-types")] +[OpenApiTag("Translation Engines")] +public partial class TranslationEngineTypesController(IAuthorizationService authService) + : ServalControllerBase(authService); diff --git a/src/Serval/src/Serval.Translation/Features/Engines/CancelBuild.cs b/src/Serval/src/Serval.Translation/Features/Engines/CancelBuild.cs new file mode 100644 index 000000000..9e13b833e --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/CancelBuild.cs @@ -0,0 +1,82 @@ +namespace Serval.Translation.Features.Engines; + +public record CancelBuild(string Owner, string EngineId) : IRequest; + +public record CancelBuildResponse( + [property: MemberNotNullWhen(true, nameof(Build))] bool IsBuildRunning, + TranslationBuildDto? Build = null +); + +public class CancelBuildHandler( + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository builds, + IEngineServiceFactory engineServiceFactory, + DtoMapper mapper +) : IRequestHandler +{ + public async Task HandleAsync(CancelBuild request, CancellationToken cancellationToken) + { + return await dataAccessContext.WithTransactionAsync( + async (ct) => + { + Engine? engine = await engines.GetAsync(request.EngineId, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + + string? buildId = await engineServiceFactory + .GetEngineService(engine.Type) + .CancelBuildAsync(request.EngineId, ct); + if (buildId is null) + return new CancelBuildResponse(IsBuildRunning: false); + + Build? currentBuild = await builds.GetAsync(buildId, ct); + if (currentBuild is null) + return new CancelBuildResponse(IsBuildRunning: false); + + return new CancelBuildResponse(IsBuildRunning: true, mapper.Map(currentBuild)); + }, + cancellationToken: cancellationToken + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Cancel the current build job (whether pending or active) for a translation engine + /// + /// + /// + /// The translation engine id + /// + /// The build job was cancelled successfully. + /// There is no active build job. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist. + /// The translation engine does not support cancelling builds. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateTranslationEngines)] + [HttpPost("{id}/current-build/cancel")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> CancelBuildAsync( + [NotNull] string id, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + CancelBuildResponse response = await handler.HandleAsync(new(Owner, id), cancellationToken); + if (response.IsBuildRunning) + return Ok(response.Build); + return NoContent(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/CreateEngine.cs b/src/Serval/src/Serval.Translation/Features/Engines/CreateEngine.cs new file mode 100644 index 000000000..9aa3a9559 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/CreateEngine.cs @@ -0,0 +1,145 @@ +namespace Serval.Translation.Features.Engines; + +public record TranslationEngineConfigDto +{ + /// + /// The translation engine name. + /// + public string? Name { get; init; } + + /// + /// The source language tag. + /// + public required string SourceLanguage { get; init; } + + /// + /// The target language tag. + /// + public required string TargetLanguage { get; init; } + + /// + /// The translation engine type. + /// + public required string Type { get; init; } + + /// + /// The model is saved when built and can be retrieved. + /// + public bool? IsModelPersisted { get; init; } +} + +public record CreateEngine(string Owner, TranslationEngineConfigDto EngineConfig) : IRequest; + +public record CreateEngineResponse(TranslationEngineDto Engine); + +public class CreateEngineHandler( + IDataAccessContext dataAccessContext, + IRepository engines, + IEngineServiceFactory engineServiceFactory, + DtoMapper mapper +) : IRequestHandler +{ + public async Task HandleAsync( + CreateEngine request, + CancellationToken cancellationToken = default + ) + { + if (!engineServiceFactory.EngineTypeExists(request.EngineConfig.Type)) + throw new InvalidOperationException($"'{request.EngineConfig.Type}' is an invalid engine type."); + + return await dataAccessContext.WithTransactionAsync( + async (ct) => + { + Engine engine = new() + { + Name = request.EngineConfig.Name, + SourceLanguage = request.EngineConfig.SourceLanguage, + TargetLanguage = request.EngineConfig.TargetLanguage, + Type = request.EngineConfig.Type.ToPascalCase(), + Owner = request.Owner, + Corpora = [], + IsModelPersisted = request.EngineConfig.IsModelPersisted, + DateCreated = DateTime.UtcNow, + }; + await engines.InsertAsync(engine, ct); + + await engineServiceFactory + .GetEngineService(engine.Type) + .CreateAsync( + engine.Id, + engine.SourceLanguage, + engine.TargetLanguage, + engine.Name, + engine.IsModelPersisted, + ct + ); + return new CreateEngineResponse(mapper.Map(engine)); + }, + cancellationToken + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Create a new translation engine + /// + /// + /// ## Parameters + /// * **`name`**: (optional) A name to help identify and distinguish the translation engine. + /// * Recommendation: Create a multi-part name to distinguish between projects, uses, etc. + /// * The name does not have to be unique, as the engine is uniquely identified by the auto-generated id + /// * **`sourceLanguage`**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) + /// * **`targetLanguage`**: The target language code (a valid IETF language tag is recommended) + /// * **`type`**: **`smt-transfer`** or **`nmt`** or **`echo`** + /// * **`isModelPersisted`**: (optional) - see below + /// ### smt-transfer + /// The Statistical Machine Translation Transfer Learning engine is primarily used for translation suggestions. Typical endpoints: translate, get-word-graph, train-segment + /// * **`isModelPersisted`**: (default to `true`) All models are persistent and can be updated with train-segment. False is not supported. + /// ### nmt + /// The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// * **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. + /// + /// If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. + /// + /// Typical endpoints: pretranslate + /// ### echo + /// The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. + /// ## Sample request: + /// + /// { + /// "name": "myTeam:myProject:myEngine", + /// "sourceLanguage": "el", + /// "targetLanguage": "en", + /// "type": "nmt" + /// "isModelPersisted": true + /// } + /// + /// + /// The translation engine configuration (see above) + /// + /// The new translation engine + /// Bad request. Is the engine type correct? + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.CreateTranslationEngines)] + [HttpPost] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> CreateAsync( + [FromBody] TranslationEngineConfigDto engineConfig, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + CreateEngineResponse response = await handler.HandleAsync(new(Owner, engineConfig), cancellationToken); + + return Created(response.Engine.Url, response.Engine); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/DeleteEngine.cs b/src/Serval/src/Serval.Translation/Features/Engines/DeleteEngine.cs new file mode 100644 index 000000000..faa447882 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/DeleteEngine.cs @@ -0,0 +1,66 @@ +namespace Serval.Translation.Features.Engines; + +public record DeleteEngine(string Owner, string EngineId) : IRequest; + +public class DeleteEngineHandler( + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository builds, + IRepository pretranslations, + IEngineServiceFactory engineServiceFactory +) : IRequestHandler +{ + public async Task HandleAsync(DeleteEngine request, CancellationToken cancellationToken = default) + { + await dataAccessContext.WithTransactionAsync( + async (ct) => + { + Engine? engine = await engines.GetAsync(request.EngineId, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + + engine = await engines.DeleteAsync(request.EngineId, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + + await builds.DeleteAllAsync(b => b.EngineRef == request.EngineId, ct); + await pretranslations.DeleteAllAsync(pt => pt.EngineRef == request.EngineId, ct); + + await engineServiceFactory.GetEngineService(engine.Type).DeleteAsync(request.EngineId, ct); + }, + cancellationToken + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Delete a translation engine + /// + /// The translation engine id + /// + /// The engine was successfully deleted. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist and therefore cannot be deleted. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.DeleteTranslationEngines)] + [HttpDelete("{id}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task DeleteAsync( + [NotNull] string id, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + await handler.HandleAsync(new(Owner, id), cancellationToken); + return Ok(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/GetAllEngines.cs b/src/Serval/src/Serval.Translation/Features/Engines/GetAllEngines.cs new file mode 100644 index 000000000..ef0f802fd --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/GetAllEngines.cs @@ -0,0 +1,43 @@ +namespace Serval.Translation.Features.Engines; + +public record GetAllEngines(string Owner) : IRequest; + +public record GetAllEnginesResponse(IEnumerable Engines); + +public class GetAllEnginesHandler(IRepository engines, DtoMapper mapper) + : IRequestHandler +{ + public async Task HandleAsync(GetAllEngines request, CancellationToken cancellationToken) + { + IEnumerable dtos = ( + await engines.GetAllAsync(e => e.Owner == request.Owner, cancellationToken) + ).Select(mapper.Map); + return new(dtos); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Get all translation engines + /// + /// + /// The engines + /// The client is not authenticated. + /// The authenticated client cannot perform the operation. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAllAsync( + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetAllEnginesResponse response = await handler.HandleAsync(new(Owner), cancellationToken); + return response.Engines; + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/GetEngine.cs b/src/Serval/src/Serval.Translation/Features/Engines/GetEngine.cs new file mode 100644 index 000000000..de8be5114 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/GetEngine.cs @@ -0,0 +1,49 @@ +namespace Serval.Translation.Features.Engines; + +public record GetEngine(string Owner, string EngineId) : IRequest; + +public record GetEngineResponse(TranslationEngineDto Engine); + +public class GetEngineHandler(IRepository engines, DtoMapper mapper) + : IRequestHandler +{ + public async Task HandleAsync(GetEngine request, CancellationToken cancellationToken) + { + Engine? engine = await engines.GetAsync(request.EngineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + return new(mapper.Map(engine)); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Get a translation engine by unique id + /// + /// The translation engine id + /// + /// The translation engine + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}", Name = Endpoints.GetTranslationEngine)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAsync( + [NotNull] string id, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetEngineResponse response = await handler.HandleAsync(new(Owner, id), cancellationToken); + return Ok(response.Engine); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/GetModelDownloadUrl.cs b/src/Serval/src/Serval.Translation/Features/Engines/GetModelDownloadUrl.cs new file mode 100644 index 000000000..00639b0d6 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/GetModelDownloadUrl.cs @@ -0,0 +1,89 @@ +namespace Serval.Translation.Features.Engines; + +public record ModelDownloadUrlDto +{ + public required string Url { get; init; } + public required int ModelRevision { get; init; } + public required DateTime ExpiresAt { get; init; } +} + +public record GetModelDownloadUrl(string Owner, string EngineId) : IRequest; + +public record GetModelDownloadUrlResponse(bool IsModelAvailable, ModelDownloadUrlDto? ModelDownloadUrl = null); + +public class GetModelDownloadUrlHandler(IRepository engines, IEngineServiceFactory engineServiceFactory) + : IRequestHandler +{ + public async Task HandleAsync( + GetModelDownloadUrl request, + CancellationToken cancellationToken + ) + { + Engine? engine = await engines.GetAsync(request.EngineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + + if (engine.ModelRevision == 0) + return new(IsModelAvailable: false); + + ModelDownloadUrlContract url = await engineServiceFactory + .GetEngineService(engine.Type) + .GetModelDownloadUrlAsync(engine.Id, cancellationToken); + return new( + IsModelAvailable: true, + new ModelDownloadUrlDto + { + Url = url.Url, + ModelRevision = url.ModelRevision, + ExpiresAt = url.ExpiresAt, + } + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Get a link to download the NMT translation model of the last build that was successfully saved. + /// + /// + /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, + /// then the model from the most recent successful build can be downloaded. + /// + /// The endpoint will return a URL that can be used to download the model for up to 1 hour + /// after the request is made. If the URL is not used within that time, a new request will need to be made. + /// + /// The download itself is created by g-zipping together the folder containing the fine tuned model + /// with all necessary supporting files. This zipped folder is then named by the pattern: + /// * <engine_id>_<model_revision>.tar.gz + /// + /// The translation engine id + /// + /// The url to download the model. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist or there is no saved model. + /// The translation engine does not support downloading builds. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/model-download-url")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetModelDownloadUrlAsync( + [NotNull] string id, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetModelDownloadUrlResponse response = await handler.HandleAsync(new(Owner, id), cancellationToken); + if (response.IsModelAvailable) + return Ok(response.ModelDownloadUrl); + return NotFound(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/GetWordGraph.cs b/src/Serval/src/Serval.Translation/Features/Engines/GetWordGraph.cs new file mode 100644 index 000000000..f24188455 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/GetWordGraph.cs @@ -0,0 +1,122 @@ +namespace Serval.Translation.Features.Engines; + +public record WordGraphDto +{ + public required IReadOnlyList SourceTokens { get; init; } + public required float InitialStateScore { get; init; } + public required IReadOnlySet FinalStates { get; init; } + public required IReadOnlyList Arcs { get; init; } +} + +public record WordGraphArcDto +{ + public required int PrevState { get; init; } + public required int NextState { get; init; } + public required double Score { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Confidences { get; init; } + public required int SourceSegmentStart { get; init; } + public required int SourceSegmentEnd { get; init; } + public required IReadOnlyList Alignment { get; init; } + public required IReadOnlyList> Sources { get; init; } +} + +public record GetWordGraph(string Owner, string EngineId, string Segment) : IRequest; + +public record GetWordGraphResponse( + [property: MemberNotNullWhen(true, nameof(GetWordGraphResponse.WordGraph))] bool IsAvailable, + WordGraphDto? WordGraph = null +); + +public class GetWordGraphHandler(IRepository engines, IEngineServiceFactory engineServiceFactory) + : IRequestHandler +{ + public async Task HandleAsync( + GetWordGraph request, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await engines.GetAsync(request.EngineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + if (engine.ModelRevision == 0) + return new(IsAvailable: false); + + WordGraphContract wordGraph = await engineServiceFactory + .GetEngineService(engine.Type) + .GetWordGraphAsync(request.EngineId, request.Segment, cancellationToken); + + return new( + IsAvailable: true, + new WordGraphDto + { + SourceTokens = wordGraph.SourceTokens, + InitialStateScore = (float)wordGraph.InitialStateScore, + FinalStates = wordGraph.FinalStates, + Arcs = wordGraph + .Arcs.Select(a => new WordGraphArcDto + { + PrevState = a.PrevState, + NextState = a.NextState, + Score = Math.Round(a.Score, 8), + TargetTokens = a.TargetTokens, + Confidences = a.Confidences.Select(c => Math.Round(c, 8)).ToList(), + SourceSegmentStart = a.SourceSegmentStart, + SourceSegmentEnd = a.SourceSegmentEnd, + Alignment = a + .Alignment.Select(wp => new AlignedWordPairDto + { + SourceIndex = wp.SourceIndex, + TargetIndex = wp.TargetIndex, + }) + .ToList(), + Sources = a.Sources, + }) + .ToList(), + } + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Get the word graph that represents all possible translations of a segment of text + /// + /// The translation engine id + /// The source segment + /// + /// The word graph result + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpPost("{id}/get-word-graph")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetWordGraphAsync( + [NotNull] string id, + [FromBody] string segment, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + GetWordGraphResponse response = await handler.HandleAsync(new(Owner, id, segment), cancellationToken); + if (!response.IsAvailable) + return Conflict(); + _logger.LogInformation("Got word graph for engine {EngineId}", id); + return Ok(response.WordGraph); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs b/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs new file mode 100644 index 000000000..4871e877e --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs @@ -0,0 +1,434 @@ +namespace Serval.Translation.Features.Engines; + +public record TranslationBuildConfigDto +{ + public string? Name { get; init; } + public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? Pretranslate { get; init; } + + /// + /// { + /// "property" : "value" + /// } + /// + public object? Options { get; init; } +} + +public record StartBuild(string Owner, string EngineId, TranslationBuildConfigDto BuildConfig) + : IRequest; + +public record StartBuildResponse( + [property: MemberNotNullWhen(false, nameof(Build))] bool IsBuildRunning, + TranslationBuildDto? Build = null +); + +public class StartBuildHandler( + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository builds, + ContractMapper contractMapper, + IEngineServiceFactory engineFactory, + ILogger logger, + DtoMapper dtoMapper, + IConfiguration configuration +) : IRequestHandler +{ + private static readonly JsonSerializerOptions ObjectJsonSerializerOptions = new() + { + Converters = { new ObjectToInferredTypesConverter() }, + }; + + public Task HandleAsync(StartBuild request, CancellationToken cancellationToken = default) + { + return dataAccessContext.WithTransactionAsync( + async (ct) => + { + Engine? engine = await engines.GetAsync(request.EngineId, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + + if ( + await builds.ExistsAsync( + b => + b.EngineRef == request.EngineId + && (b.State == JobState.Active || b.State == JobState.Pending), + ct + ) + ) + { + return new StartBuildResponse(IsBuildRunning: true); + } + + Build build = new() + { + EngineRef = engine.Id, + Owner = engine.Owner, + Name = request.BuildConfig.Name, + Pretranslate = Map(engine, request.BuildConfig.Pretranslate), + TrainOn = Map(engine, request.BuildConfig.TrainOn), + Options = MapOptions(request.BuildConfig.Options), + DeploymentVersion = configuration.GetValue("deploymentVersion") ?? "Unknown", + DateCreated = DateTime.UtcNow, + }; + await builds.InsertAsync(build, ct); + + IReadOnlyList corpora = contractMapper.Map(build, engine); + + string? buildOptions = null; + if (build.Options is not null) + buildOptions = JsonSerializer.Serialize(build.Options); + try + { + var buildRequestSummary = new JsonObject + { + ["Event"] = "BuildRequest", + ["EngineId"] = engine.Id, + ["BuildId"] = build.Id, + ["CorpusCount"] = corpora.Count, + ["ModelRevision"] = engine.ModelRevision, + ["ClientId"] = engine.Owner, + }; + string? buildOptionsStr = null; + try + { + buildRequestSummary.Add("Options", JsonNode.Parse(buildOptions ?? "null")); + } + catch (JsonException) + { + buildRequestSummary.Add( + "Options", + "Build \"Options\" failed parsing: " + (buildOptionsStr ?? "null") + ); + } + logger.LogInformation("{request}", buildRequestSummary.ToJsonString()); + } + catch (JsonException) + { + logger.LogInformation("Error parsing build request summary."); + } + + await engineFactory + .GetEngineService(engine.Type) + .StartBuildAsync(engine.Id, build.Id, corpora, buildOptions, ct); + return new StartBuildResponse(IsBuildRunning: false, Build: dtoMapper.Map(build)); + }, + cancellationToken + ); + } + +#pragma warning disable CS0612 // Type or member is obsolete + private static List? Map(Engine engine, IReadOnlyList? source) + { + if (source is null) + return null; + + if ( + source.Where(p => p.ParallelCorpusId != null).Select(p => p.ParallelCorpusId).Distinct().Count() + != source.Count(p => p.ParallelCorpusId != null) + ) + { + throw new InvalidOperationException($"Each ParallelCorpusId may only be specified once."); + } + + if ( + source.Where(p => p.CorpusId != null).Select(p => p.CorpusId).Distinct().Count() + != source.Count(p => p.CorpusId != null) + ) + { + throw new InvalidOperationException($"Each CorpusId may only be specified once."); + } + + var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); + var parallelCorpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); + var pretranslateCorpora = new List(); + foreach (PretranslateCorpusConfigDto pcc in source) + { + if (pcc.CorpusId != null) + { + if (pcc.ParallelCorpusId != null) + { + throw new InvalidOperationException($"Only one of ParallelCorpusId and CorpusId can be set."); + } + if (!corpusIds.Contains(pcc.CorpusId)) + { + throw new InvalidOperationException( + $"The corpus {pcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." + ); + } + Corpus corpus = engine.Corpora.Single(c => c.Id == pcc.CorpusId); + if (corpus.SourceFiles.Count == 0 && corpus.TargetFiles.Count == 0) + { + throw new InvalidOperationException( + $"The corpus {pcc.CorpusId} is not valid: This corpus does not have any source or target files." + ); + } + if (pcc.TextIds != null && pcc.ScriptureRange != null) + { + throw new InvalidOperationException( + $"The corpus {pcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." + ); + } + pretranslateCorpora.Add( + new PretranslateCorpus + { + CorpusRef = pcc.CorpusId, + TextIds = pcc.TextIds?.ToList(), + ScriptureRange = pcc.ScriptureRange, + } + ); + } + else + { + if (pcc.ParallelCorpusId == null) + { + throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); + } + if (!parallelCorpusIds.Contains(pcc.ParallelCorpusId)) + { + throw new InvalidOperationException( + $"The parallel corpus {pcc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." + ); + } + ParallelCorpus corpus = engine.ParallelCorpora.Single(pc => pc.Id == pcc.ParallelCorpusId); + if (corpus.SourceCorpora.Count == 0 && corpus.TargetCorpora.Count == 0) + { + throw new InvalidOperationException( + $"The corpus {pcc.ParallelCorpusId} does not have source or target corpora associated with it." + ); + } + if ( + pcc.SourceFilters != null + && pcc.SourceFilters.Count > 0 + && ( + pcc.SourceFilters.Select(sf => sf.CorpusId).Distinct().Count() > 1 + || pcc.SourceFilters[0].CorpusId + != engine.ParallelCorpora.Single(pc => pc.Id == pcc.ParallelCorpusId).SourceCorpora[0].Id + ) + ) + { + throw new InvalidOperationException( + $"Only the first source corpus in a parallel corpus may be filtered for pretranslation." + ); + } + pretranslateCorpora.Add( + new PretranslateCorpus + { + ParallelCorpusRef = pcc.ParallelCorpusId, + SourceFilters = pcc.SourceFilters?.Select(Map).ToList(), + } + ); + } + } + return pretranslateCorpora; + } + + private static List? Map(Engine engine, IReadOnlyList? source) + { + if (source is null) + return null; + + if ( + source.Where(p => p.ParallelCorpusId != null).Select(p => p.ParallelCorpusId).Distinct().Count() + != source.Count(p => p.ParallelCorpusId != null) + ) + { + throw new InvalidOperationException($"Each ParallelCorpusId may only be specified once."); + } + + if ( + source.Where(p => p.CorpusId != null).Select(p => p.CorpusId).Distinct().Count() + != source.Count(p => p.CorpusId != null) + ) + { + throw new InvalidOperationException($"Each CorpusId may only be specified once."); + } + + var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); + var parallelCorpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); + var trainOnCorpora = new List(); + foreach (TrainingCorpusConfigDto tcc in source) + { + if (tcc.CorpusId != null) + { + if (tcc.ParallelCorpusId != null) + { + throw new InvalidOperationException($"Only one of ParallelCorpusId and CorpusId can be set."); + } + if (!corpusIds.Contains(tcc.CorpusId)) + { + throw new InvalidOperationException( + $"The corpus {tcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." + ); + } + Corpus corpus = engine.Corpora.Single(c => c.Id == tcc.CorpusId); + if (corpus.SourceFiles.Count == 0 && corpus.TargetFiles.Count == 0) + { + throw new InvalidOperationException( + $"The corpus {tcc.CorpusId} is not valid: This corpus does not have any source or target files." + ); + } + if (tcc.TextIds != null && tcc.ScriptureRange != null) + { + throw new InvalidOperationException( + $"The corpus {tcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." + ); + } + trainOnCorpora.Add( + new TrainingCorpus + { + CorpusRef = tcc.CorpusId, + TextIds = tcc.TextIds?.ToList(), + ScriptureRange = tcc.ScriptureRange, + } + ); + } + else + { + if (tcc.ParallelCorpusId == null) + { + throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); + } + if (!parallelCorpusIds.Contains(tcc.ParallelCorpusId)) + { + throw new InvalidOperationException( + $"The parallel corpus {tcc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." + ); + } + ParallelCorpus corpus = engine.ParallelCorpora.Single(pc => pc.Id == tcc.ParallelCorpusId); + if (corpus.SourceCorpora.Count == 0 && corpus.TargetCorpora.Count == 0) + { + throw new InvalidOperationException( + $"The corpus {tcc.ParallelCorpusId} does not have source or target corpora associated with it." + ); + } + foreach (MonolingualCorpus monolingualCorpus in corpus.SourceCorpora.Concat(corpus.TargetCorpora)) + { + if (monolingualCorpus.Files.Count == 0) + { + throw new InvalidOperationException( + $"The corpus {monolingualCorpus.Id} referenced in parallel corpus {corpus.Id} does not have any files associated with it." + ); + } + } + trainOnCorpora.Add( + new TrainingCorpus + { + ParallelCorpusRef = tcc.ParallelCorpusId, + SourceFilters = tcc.SourceFilters?.Select(Map).ToList(), + TargetFilters = tcc.TargetFilters?.Select(Map).ToList(), + } + ); + } + } + return trainOnCorpora; + } +#pragma warning restore CS0612 // Type or member is obsolete + + private static ParallelCorpusFilter Map(ParallelCorpusFilterConfigDto source) + { + if (source.TextIds != null && source.ScriptureRange != null) + { + throw new InvalidOperationException( + $"The parallel corpus filter for corpus {source.CorpusId} is not valid: At most, one of TextIds and ScriptureRange can be set." + ); + } + return new ParallelCorpusFilter + { + CorpusRef = source.CorpusId, + TextIds = source.TextIds, + ScriptureRange = source.ScriptureRange, + }; + } + + private static Dictionary? MapOptions(object? source) + { + if (source is null) + return null; + try + { + return JsonSerializer.Deserialize>( + source.ToString()!, + ObjectJsonSerializerOptions + ); + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to parse field 'options' : {e.Message}", e); + } + } +} + +public partial class TranslationEnginesController +{ + /// + /// Starts a build job for a translation engine. + /// + /// + /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). + /// Specifying a corpus: + /// * A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. + /// * A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. + /// + /// Filtering by text id or chapter: + /// * Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). + /// * Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). + /// * All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + /// + /// Filter - train on all or none + /// * If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively + /// * If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. + /// * If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. + /// * If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. + /// + /// Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, + /// the following text will be pretranslated: + /// * Text segments that are in the source but do not exist in the target. + /// * Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. + /// If the engine does not support pretranslation, these fields have no effect. + /// Pretranslating uses the same filtering as training. + /// + /// The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + /// See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + /// See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. + /// See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + /// + /// Note that when using a parallel corpus: + /// * If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. + /// * If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + /// + /// The translation engine id + /// The build config (see remarks) + /// + /// The new build job + /// The build configuration was invalid. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist. + /// There is already an active/pending build or a build in the process of being canceled. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateTranslationEngines)] + [HttpPost("{id}/builds")] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> StartBuildAsync( + [NotNull] string id, + [FromBody] TranslationBuildConfigDto buildConfig, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + StartBuildResponse response = await handler.HandleAsync(new(Owner, id, buildConfig), cancellationToken); + + if (response.IsBuildRunning) + return Conflict(); + + return Created(response.Build.Url, response.Build); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/TrainSegment.cs b/src/Serval/src/Serval.Translation/Features/Engines/TrainSegment.cs new file mode 100644 index 000000000..8ce681865 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/TrainSegment.cs @@ -0,0 +1,87 @@ +namespace Serval.Translation.Features.Engines; + +public record SegmentPairDto +{ + public required string SourceSegment { get; init; } + public required string TargetSegment { get; init; } + public required bool SentenceStart { get; init; } +} + +public record TrainSegment(string Owner, string EngineId, SegmentPairDto SegmentPair) : IRequest; + +public record TrainSegmentResponse(bool IsAvailable); + +public class TrainSegmentHandler(IRepository engines, IEngineServiceFactory engineServiceFactory) + : IRequestHandler +{ + public async Task HandleAsync( + TrainSegment request, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await engines.GetAsync(request.EngineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + if (engine.ModelRevision == 0) + return new(IsAvailable: false); + + await engineServiceFactory + .GetEngineService(engine.Type) + .TrainSegmentPairAsync( + engine.Id, + request.SegmentPair.SourceSegment, + request.SegmentPair.TargetSegment, + request.SegmentPair.SentenceStart, + cancellationToken + ); + return new(IsAvailable: true); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Incrementally train a translation engine with a segment pair + /// + /// + /// A segment pair consists of a source and target segment as well as a boolean flag `sentenceStart` + /// that should be set to `true` if this segment pair forms the beginning of a sentence. (This information + /// will be used to reconstruct proper capitalization when training/inferencing). + /// + /// The translation engine id + /// The segment pair + /// + /// The engine was trained successfully. + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateTranslationEngines)] + [HttpPost("{id}/train-segment")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task TrainSegmentAsync( + [NotNull] string id, + [FromBody] SegmentPairDto segmentPair, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + TrainSegmentResponse response = await handler.HandleAsync(new(Owner, id, segmentPair), cancellationToken); + if (!response.IsAvailable) + return Conflict(); + _logger.LogInformation("Trained segment pair for engine {EngineId}", id); + return Ok(); + } +} diff --git a/src/Serval/src/Serval.Translation/Features/Engines/Translate.cs b/src/Serval/src/Serval.Translation/Features/Engines/Translate.cs new file mode 100644 index 000000000..2c46323a0 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/Translate.cs @@ -0,0 +1,153 @@ +namespace Serval.Translation.Features.Engines; + +public record TranslationResultDto +{ + public required string Translation { get; init; } + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Confidences { get; init; } + public required IReadOnlyList> Sources { get; init; } + public required IReadOnlyList Alignment { get; init; } + public required IReadOnlyList Phrases { get; init; } +} + +public record PhraseDto +{ + public required int SourceSegmentStart { get; init; } + public required int SourceSegmentEnd { get; init; } + public required int TargetSegmentCut { get; init; } +} + +public record Translate(string Owner, string EngineId, string Segment, int N = 1) : IRequest; + +public record TranslateResponse( + [property: MemberNotNullWhen(true, nameof(Results))] bool IsAvailable, + IEnumerable? Results = null +); + +public class TranslateHandler(IRepository engines, IEngineServiceFactory engineServiceFactory) + : IRequestHandler +{ + public async Task HandleAsync(Translate request, CancellationToken cancellationToken = default) + { + Engine? engine = await engines.GetAsync(request.EngineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + if (engine.ModelRevision == 0) + return new(IsAvailable: false); + + IReadOnlyList results = await engineServiceFactory + .GetEngineService(engine.Type) + .TranslateAsync(request.EngineId, request.N, request.Segment, cancellationToken); + + return new( + IsAvailable: true, + results.Select(r => new TranslationResultDto + { + Translation = r.Translation, + SourceTokens = r.SourceTokens, + TargetTokens = r.TargetTokens, + Confidences = r.Confidences.Select(c => Math.Round(c, 8)).ToList(), + Sources = r.Sources, + Alignment = r + .Alignment.Select(wp => new AlignedWordPairDto + { + SourceIndex = wp.SourceIndex, + TargetIndex = wp.TargetIndex, + }) + .ToList(), + Phrases = r + .Phrases.Select(p => new PhraseDto + { + SourceSegmentStart = p.SourceSegmentStart, + SourceSegmentEnd = p.SourceSegmentEnd, + TargetSegmentCut = p.TargetSegmentCut, + }) + .ToList(), + }) + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Translate a segment of text + /// + /// The translation engine id + /// The source segment + /// + /// The translation result + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built before it can translate segments. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpPost("{id}/translate")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> TranslateAsync( + [NotNull] string id, + [FromBody] string segment, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + TranslateResponse response = await handler.HandleAsync(new Translate(Owner, id, segment), cancellationToken); + if (!response.IsAvailable) + return Conflict(); + _logger.LogInformation("Translated segment for engine {EngineId}", id); + return Ok(response.Results?.First()); + } + + /// + /// Returns the top N translations of a segment + /// + /// The translation engine id + /// The number of translations to generate + /// The source segment + /// + /// The translation results + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built before it can translate segments. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpPost("{id}/translate/{n}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> TranslateNAsync( + [NotNull] string id, + [NotNull] int n, + [FromBody] string segment, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken + ) + { + TranslateResponse response = await handler.HandleAsync(new(Owner, id, segment, n), cancellationToken); + if (!response.IsAvailable) + return Conflict(); + _logger.LogInformation("Translated {n} segments for engine {EngineId}", n, id); + return Ok(response.Results); + } +} diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Features/Engines/TranslationEnginesController.cs similarity index 57% rename from src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs rename to src/Serval/src/Serval.Translation/Features/Engines/TranslationEnginesController.cs index 001bdb5bc..432f751c2 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Features/Engines/TranslationEnginesController.cs @@ -1,388 +1,26 @@ -namespace Serval.Translation.Controllers; +namespace Serval.Translation.Features.Engines; #pragma warning disable CS0612 // Type or member is obsolete [ApiVersion(1.0)] [Route("api/v{version:apiVersion}/translation/engines")] [OpenApiTag("Translation Engines")] -public class TranslationEnginesController( +public partial class TranslationEnginesController( IAuthorizationService authService, IEngineService engineService, IBuildService buildService, IPretranslationService pretranslationService, IOptionsMonitor apiOptions, - IConfiguration configuration, IUrlService urlService, ILogger logger -) : TranslationControllerBase(authService, urlService) +) : ServalControllerBase(authService) { - private static readonly JsonSerializerOptions ObjectJsonSerializerOptions = new() - { - Converters = { new ObjectToInferredTypesConverter() }, - }; private readonly IEngineService _engineService = engineService; private readonly IBuildService _buildService = buildService; private readonly IPretranslationService _pretranslationService = pretranslationService; private readonly IOptionsMonitor _apiOptions = apiOptions; private readonly IUrlService _urlService = urlService; private readonly ILogger _logger = logger; - private readonly IConfiguration _configuration = configuration; - - /// - /// Get all translation engines - /// - /// - /// The engines - /// The client is not authenticated. - /// The authenticated client cannot perform the operation. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetAllAsync(CancellationToken cancellationToken) - { - return (await _engineService.GetAllAsync(Owner, cancellationToken)).Select(Map); - } - - /// - /// Get a translation engine by unique id - /// - /// The translation engine id - /// - /// The translation engine - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}", Name = Endpoints.GetTranslationEngine)] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetAsync( - [NotNull] string id, - CancellationToken cancellationToken - ) - { - Engine engine = await _engineService.GetAsync(id, cancellationToken); - await AuthorizeAsync(engine); - return Ok(Map(engine)); - } - - /// - /// Create a new translation engine - /// - /// - /// ## Parameters - /// * **`name`**: (optional) A name to help identify and distinguish the translation engine. - /// * Recommendation: Create a multi-part name to distinguish between projects, uses, etc. - /// * The name does not have to be unique, as the engine is uniquely identified by the auto-generated id - /// * **`sourceLanguage`**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) - /// * **`targetLanguage`**: The target language code (a valid IETF language tag is recommended) - /// * **`type`**: **`smt-transfer`** or **`nmt`** or **`echo`** - /// * **`isModelPersisted`**: (optional) - see below - /// ### smt-transfer - /// The Statistical Machine Translation Transfer Learning engine is primarily used for translation suggestions. Typical endpoints: translate, get-word-graph, train-segment - /// * **`isModelPersisted`**: (default to `true`) All models are persistent and can be updated with train-segment. False is not supported. - /// ### nmt - /// The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). - /// * **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. - /// - /// If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. - /// - /// Typical endpoints: pretranslate - /// ### echo - /// The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. - /// ## Sample request: - /// - /// { - /// "name": "myTeam:myProject:myEngine", - /// "sourceLanguage": "el", - /// "targetLanguage": "en", - /// "type": "nmt" - /// "isModelPersisted": true - /// } - /// - /// - /// The translation engine configuration (see above) - /// - /// The new translation engine - /// Bad request. Is the engine type correct? - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.CreateTranslationEngines)] - [HttpPost] - [ProducesResponseType(StatusCodes.Status201Created)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> CreateAsync( - [FromBody] TranslationEngineConfigDto engineConfig, - CancellationToken cancellationToken - ) - { - Engine engine = Map(engineConfig); - Engine updatedEngine = await _engineService.CreateAsync(engine, cancellationToken); - TranslationEngineDto dto = Map(updatedEngine); - return Created(dto.Url, dto); - } - - /// - /// Delete a translation engine - /// - /// The translation engine id - /// - /// The engine was successfully deleted. - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist and therefore cannot be deleted. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.DeleteTranslationEngines)] - [HttpDelete("{id}")] - [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) - { - await AuthorizeAsync(id, cancellationToken); - await _engineService.DeleteAsync(id, cancellationToken); - return Ok(); - } - - /// - /// Update the source and/or target languages of a translation engine - /// - /// - /// ## Sample request: - /// - /// { - /// "sourceLanguage": "en", - /// "targetLanguage": "en" - /// } - /// - /// - /// The translation engine id - /// - /// The engine language was successfully updated. - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist and therefore cannot be updated. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.UpdateTranslationEngines)] - [HttpPatch("{id}")] - [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task UpdateAsync( - [FromRoute] string id, - [FromBody] TranslationEngineUpdateConfigDto request, - CancellationToken cancellationToken = default - ) - { - await AuthorizeAsync(id, cancellationToken); - - if ( - request is null - || (string.IsNullOrWhiteSpace(request.SourceLanguage) && string.IsNullOrWhiteSpace(request.TargetLanguage)) - ) - { - return BadRequest("sourceLanguage or targetLanguage is required."); - } - - await _engineService.UpdateAsync( - id, - string.IsNullOrWhiteSpace(request.SourceLanguage) ? null : request.SourceLanguage, - string.IsNullOrWhiteSpace(request.TargetLanguage) ? null : request.TargetLanguage, - cancellationToken - ); - - return Ok(); - } - - /// - /// Translate a segment of text - /// - /// The translation engine id - /// The source segment - /// - /// The translation result - /// Bad request - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// The method is not supported. - /// The engine needs to be built before it can translate segments. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpPost("{id}/translate")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> TranslateAsync( - [NotNull] string id, - [FromBody] string segment, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - TranslationResult? result = await _engineService.TranslateAsync(id, segment, cancellationToken); - if (result is null) - return Conflict(); - _logger.LogInformation("Translated segment for engine {EngineId}", id); - return Ok(Map(result)); - } - - /// - /// Returns the top N translations of a segment - /// - /// The translation engine id - /// The number of translations to generate - /// The source segment - /// - /// The translation results - /// Bad request - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// The method is not supported. - /// The engine needs to be built before it can translate segments. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpPost("{id}/translate/{n}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task>> TranslateNAsync( - [NotNull] string id, - [NotNull] int n, - [FromBody] string segment, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - IEnumerable? results = await _engineService.TranslateAsync( - id, - n, - segment, - cancellationToken - ); - if (results is null) - return Conflict(); - _logger.LogInformation("Translated {n} segments for engine {EngineId}", n, id); - return Ok(results.Select(Map)); - } - - /// - /// Get the word graph that represents all possible translations of a segment of text - /// - /// The translation engine id - /// The source segment - /// - /// The word graph result - /// Bad request - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// The method is not supported. - /// The engine needs to be built first. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpPost("{id}/get-word-graph")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetWordGraphAsync( - [NotNull] string id, - [FromBody] string segment, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - WordGraph? wordGraph = await _engineService.GetWordGraphAsync(id, segment, cancellationToken); - if (wordGraph is null) - return Conflict(); - _logger.LogInformation("Got word graph for engine {EngineId}", id); - return Ok(Map(wordGraph)); - } - - /// - /// Incrementally train a translation engine with a segment pair - /// - /// - /// A segment pair consists of a source and target segment as well as a boolean flag `sentenceStart` - /// that should be set to `true` if this segment pair forms the beginning of a sentence. (This information - /// will be used to reconstruct proper capitalization when training/inferencing). - /// - /// The translation engine id - /// The segment pair - /// - /// The engine was trained successfully. - /// Bad request - /// The client is not authenticated. - /// The authenticated client cannot perform the operation or does not own the translation engine. - /// The engine does not exist. - /// The method is not supported. - /// The engine needs to be built first. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.UpdateTranslationEngines)] - [HttpPost("{id}/train-segment")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task TrainSegmentAsync( - [NotNull] string id, - [FromBody] SegmentPairDto segmentPair, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - if ( - !await _engineService.TrainSegmentPairAsync( - id, - segmentPair.SourceSegment, - segmentPair.TargetSegment, - segmentPair.SentenceStart, - cancellationToken - ) - ) - { - return Conflict(); - } - _logger.LogInformation("Trained segment pair for engine {EngineId}", id); - return Ok(); - } /// /// Add a corpus to a translation engine (obsolete - use parallel corpora instead) @@ -407,7 +45,7 @@ CancellationToken cancellationToken /// /// The translation engine id /// The corpus configuration (see remarks) - /// + /// /// /// /// The added corpus @@ -428,14 +66,14 @@ CancellationToken cancellationToken public async Task> AddCorpusAsync( [NotNull] string id, [FromBody] TranslationCorpusConfigDto corpusConfig, - [FromServices] IRequestClient getDataFileClient, + [FromServices] IRequestHandler getDataFileHandler, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken ) { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - Corpus corpus = await MapAsync(getDataFileClient, idGenerator.GenerateId(), corpusConfig, cancellationToken); + Corpus corpus = await MapAsync(getDataFileHandler, idGenerator.GenerateId(), corpusConfig, cancellationToken); await _engineService.AddCorpusAsync(id, corpus, cancellationToken); TranslationCorpusDto dto = Map(id, corpus); return Created(dto.Url, dto); @@ -451,7 +89,7 @@ CancellationToken cancellationToken /// The translation engine id /// The corpus id /// The corpus configuration - /// The data file client + /// The data file handler /// /// The corpus was updated successfully /// Bad request @@ -472,7 +110,7 @@ public async Task> UpdateCorpusAsync( [NotNull] string id, [NotNull] string corpusId, [FromBody] TranslationCorpusUpdateConfigDto corpusConfig, - [FromServices] IRequestClient getDataFileClient, + [FromServices] IRequestHandler getDataFileHandler, CancellationToken cancellationToken ) { @@ -482,10 +120,10 @@ CancellationToken cancellationToken corpusId, corpusConfig.SourceFiles is null ? null - : await MapAsync(getDataFileClient, corpusConfig.SourceFiles, cancellationToken), + : await MapAsync(getDataFileHandler, corpusConfig.SourceFiles, cancellationToken), corpusConfig.TargetFiles is null ? null - : await MapAsync(getDataFileClient, corpusConfig.TargetFiles, cancellationToken), + : await MapAsync(getDataFileHandler, corpusConfig.TargetFiles, cancellationToken), cancellationToken ); return Ok(Map(id, corpus)); @@ -831,7 +469,7 @@ CancellationToken cancellationToken /// /// The translation engine id /// The corpus configuration (see remarks) - /// + /// /// /// /// The added corpus @@ -851,7 +489,7 @@ CancellationToken cancellationToken public async Task> AddParallelCorpusAsync( [NotNull] string id, [FromBody] TranslationParallelCorpusConfigDto corpusConfig, - [FromServices] IRequestClient getCorpusClient, + [FromServices] IRequestHandler getCorpusHandler, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken ) @@ -859,7 +497,7 @@ CancellationToken cancellationToken Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); ParallelCorpus corpus = await MapAsync( - getCorpusClient, + getCorpusHandler, idGenerator.GenerateId(), corpusConfig, cancellationToken @@ -878,7 +516,7 @@ CancellationToken cancellationToken /// The translation engine id /// The parallel corpus id /// The corpus configuration - /// The data file client + /// The data file client /// /// The corpus was updated successfully /// Bad request @@ -898,7 +536,7 @@ public async Task> UpdateParallelCorp [NotNull] string id, [NotNull] string parallelCorpusId, [FromBody] TranslationParallelCorpusUpdateConfigDto corpusConfig, - [FromServices] IRequestClient getCorpusClient, + [FromServices] IRequestHandler getCorpusHandler, CancellationToken cancellationToken ) { @@ -908,10 +546,10 @@ CancellationToken cancellationToken parallelCorpusId, corpusConfig.SourceCorpusIds is null ? null - : await MapAsync(getCorpusClient, corpusConfig.SourceCorpusIds, cancellationToken), + : await MapAsync(getCorpusHandler, corpusConfig.SourceCorpusIds, cancellationToken), corpusConfig.TargetCorpusIds is null ? null - : await MapAsync(getCorpusClient, corpusConfig.TargetCorpusIds, cancellationToken), + : await MapAsync(getCorpusHandler, corpusConfig.TargetCorpusIds, cancellationToken), cancellationToken ); return Ok(Map(id, parallelCorpus)); @@ -1256,11 +894,12 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task>> GetAllBuildsAsync( [NotNull] string id, + [FromServices] DtoMapper mapper, CancellationToken cancellationToken ) { await AuthorizeAsync(id, cancellationToken); - return Ok((await _buildService.GetAllAsync(Owner, id, cancellationToken)).Select(Map)); + return Ok((await _buildService.GetAllAsync(Owner, id, cancellationToken)).Select(mapper.Map)); } /// @@ -1299,6 +938,7 @@ public async Task> GetBuildAsync( [NotNull] string buildId, [FromQuery(Name = "min-revision")] long? minRevision, [OpenApiIgnore] [FromQuery(Name = "minRevision")] long? minRevisionCamelCase, + [FromServices] DtoMapper mapper, CancellationToken cancellationToken ) { @@ -1315,90 +955,16 @@ CancellationToken cancellationToken { EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), EntityChangeType.Delete => NotFound(), - _ => Ok(Map(change.Entity!)), + _ => Ok(mapper.Map(change.Entity!)), }; } else { Build build = await _buildService.GetAsync(buildId, cancellationToken); - return Ok(Map(build)); + return Ok(mapper.Map(build)); } } - /// - /// Starts a build job for a translation engine. - /// - /// - /// Specify the corpora and text ids/scripture ranges within those corpora to train on. Only one type of corpus may be used: either (legacy) corpora (see /translation/engines/{id}/corpora) or parallel corpora (see /translation/engines/{id}/parallel-corpora). - /// Specifying a corpus: - /// * A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. - /// * A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - /// - /// Filtering by text id or chapter: - /// * Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). - /// * Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). - /// * All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - /// - /// Filter - train on all or none - /// * If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively - /// * If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. - /// * If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. - /// * If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - /// - /// Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, - /// the following text will be pretranslated: - /// * Text segments that are in the source but do not exist in the target. - /// * Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. - /// If the engine does not support pretranslation, these fields have no effect. - /// Pretranslating uses the same filtering as training. - /// - /// The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. - /// See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. - /// See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. - /// See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - /// - /// Note that when using a parallel corpus: - /// * If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. - /// * If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. - /// - /// The translation engine id - /// The build config (see remarks) - /// - /// The new build job - /// The build configuration was invalid. - /// The client is not authenticated. - /// The authenticated client does not own the translation engine. - /// The engine does not exist. - /// There is already an active/pending build or a build in the process of being canceled. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.UpdateTranslationEngines)] - [HttpPost("{id}/builds")] - [ProducesResponseType(StatusCodes.Status201Created)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> StartBuildAsync( - [NotNull] string id, - [FromBody] TranslationBuildConfigDto buildConfig, - CancellationToken cancellationToken - ) - { - string deploymentVersion = _configuration.GetValue("deploymentVersion") ?? "Unknown"; - - Engine engine = await _engineService.GetAsync(id, cancellationToken); - await AuthorizeAsync(engine); - Build build = Map(engine, buildConfig, deploymentVersion); - - if (!await _engineService.StartBuildAsync(build, cancellationToken)) - return Conflict(); - - TranslationBuildDto dto = Map(build); - return Created(dto.Url, dto); - } - /// /// Get the currently running build job for a translation engine /// @@ -1430,6 +996,7 @@ public async Task> GetCurrentBuildAsync( [NotNull] string id, [FromQuery(Name = "min-revision")] long? minRevision, [OpenApiIgnore] [FromQuery(Name = "minRevision")] long? minRevisionCamelCase, + [FromServices] DtoMapper mapper, CancellationToken cancellationToken ) { @@ -1446,7 +1013,7 @@ CancellationToken cancellationToken { EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), EntityChangeType.Delete => NoContent(), - _ => Ok(Map(change.Entity!)), + _ => Ok(mapper.Map(change.Entity!)), }; } else @@ -1455,87 +1022,10 @@ CancellationToken cancellationToken if (build == null) return NoContent(); - return Ok(Map(build)); + return Ok(mapper.Map(build)); } } - /// - /// Cancel the current build job (whether pending or active) for a translation engine - /// - /// - /// - /// The translation engine id - /// - /// The build job was cancelled successfully. - /// There is no active build job. - /// The client is not authenticated. - /// The authenticated client does not own the translation engine. - /// The engine does not exist. - /// The translation engine does not support cancelling builds. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.UpdateTranslationEngines)] - [HttpPost("{id}/current-build/cancel")] - [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> CancelBuildAsync( - [NotNull] string id, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - Build? build = await _engineService.CancelBuildAsync(id, cancellationToken); - if (build is null) - return NoContent(); - return Ok(Map(build)); - } - - /// - /// Get a link to download the NMT translation model of the last build that was successfully saved. - /// - /// - /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, - /// then the model from the most recent successful build can be downloaded. - /// - /// The endpoint will return a URL that can be used to download the model for up to 1 hour - /// after the request is made. If the URL is not used within that time, a new request will need to be made. - /// - /// The download itself is created by g-zipping together the folder containing the fine tuned model - /// with all necessary supporting files. This zipped folder is then named by the pattern: - /// * <engine_id>_<model_revision>.tar.gz - /// - /// The translation engine id - /// - /// The url to download the model. - /// The client is not authenticated. - /// The authenticated client does not own the translation engine. - /// The engine does not exist or there is no saved model. - /// The translation engine does not support downloading builds. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}/model-download-url")] - [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetModelDownloadUrlAsync( - [NotNull] string id, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - ModelDownloadUrl? modelInfo = await _engineService.GetModelDownloadUrlAsync(id, cancellationToken); - if (modelInfo is null) - return NotFound(); - return Ok(Map(modelInfo)); - } - private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) { Engine engine = await _engineService.GetAsync(id, cancellationToken); @@ -1543,7 +1033,7 @@ private async Task AuthorizeAsync(string id, CancellationToken cancellationToken } private async Task MapAsync( - IRequestClient getDataFileClient, + IRequestHandler getDataFileHandler, string corpusId, TranslationCorpusConfigDto source, CancellationToken cancellationToken @@ -1555,13 +1045,13 @@ CancellationToken cancellationToken Name = source.Name, SourceLanguage = source.SourceLanguage, TargetLanguage = source.TargetLanguage, - SourceFiles = await MapAsync(getDataFileClient, source.SourceFiles, cancellationToken), - TargetFiles = await MapAsync(getDataFileClient, source.TargetFiles, cancellationToken), + SourceFiles = await MapAsync(getDataFileHandler, source.SourceFiles, cancellationToken), + TargetFiles = await MapAsync(getDataFileHandler, source.TargetFiles, cancellationToken), }; } private async Task MapAsync( - IRequestClient getDataFileClient, + IRequestHandler getCorpusHandler, string corpusId, TranslationParallelCorpusConfigDto source, CancellationToken cancellationToken @@ -1570,13 +1060,13 @@ CancellationToken cancellationToken return new ParallelCorpus { Id = corpusId, - SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken), - TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken), + SourceCorpora = await MapAsync(getCorpusHandler, source.SourceCorpusIds, cancellationToken), + TargetCorpora = await MapAsync(getCorpusHandler, source.TargetCorpusIds, cancellationToken), }; } private async Task> MapAsync( - IRequestClient getDataFileClient, + IRequestHandler getDataFileHandler, IEnumerable fileConfigs, CancellationToken cancellationToken ) @@ -1584,23 +1074,23 @@ CancellationToken cancellationToken var files = new List(); foreach (TranslationCorpusFileConfigDto fileConfig in fileConfigs) { - Response response = await getDataFileClient.GetResponse< - DataFileResult, - DataFileNotFound - >(new GetDataFile { DataFileId = fileConfig.FileId, Owner = Owner }, cancellationToken); - if (response.Is(out Response? result)) + GetDataFileResponse response = await getDataFileHandler.HandleAsync( + new(fileConfig.FileId, Owner), + cancellationToken + ); + if (response.IsFound) { files.Add( new CorpusFile { Id = fileConfig.FileId, - Filename = result.Message.Filename, - TextId = fileConfig.TextId ?? result.Message.Name, - Format = result.Message.Format, + Filename = response.File.Filename, + TextId = fileConfig.TextId ?? response.File.Name, + Format = response.File.Format, } ); } - else if (response.Is(out Response? _)) + else { throw new InvalidOperationException($"The data file {fileConfig.FileId} cannot be found."); } @@ -1609,7 +1099,7 @@ CancellationToken cancellationToken } private async Task> MapAsync( - IRequestClient getCorpusClient, + IRequestHandler getCorpusHandler, IEnumerable corpusIds, CancellationToken cancellationToken ) @@ -1617,13 +1107,10 @@ CancellationToken cancellationToken var corpora = new List(); foreach (string corpusId in corpusIds) { - Response response = await getCorpusClient.GetResponse< - CorpusResult, - CorpusNotFound - >(new GetCorpus { CorpusId = corpusId, Owner = Owner }, cancellationToken); - if (response.Is(out Response? result)) + GetCorpusResponse response = await getCorpusHandler.HandleAsync(new(corpusId, Owner), cancellationToken); + if (response.IsFound) { - if (!result.Message.Files.Any()) + if (!response.Corpus.Files.Any()) { throw new InvalidOperationException( $"The corpus {corpusId} does not have any files associated with it." @@ -1633,21 +1120,22 @@ CancellationToken cancellationToken new MonolingualCorpus { Id = corpusId, - Name = result.Message.Name ?? "", - Language = result.Message.Language, - Files = result - .Message.Files.Select(f => new CorpusFile + Name = response.Corpus.Name ?? "", + Language = response.Corpus.Language, + Files = + [ + .. response.Corpus.Files.Select(f => new CorpusFile { Id = f.File.DataFileId, Filename = f.File.Filename, Format = f.File.Format, TextId = f.TextId ?? f.File.Name, - }) - .ToList(), + }), + ], } ); } - else if (response.Is(out Response? _)) + else { throw new InvalidOperationException($"The corpus {corpusId} cannot be found."); } @@ -1655,346 +1143,6 @@ CancellationToken cancellationToken return corpora; } - private Engine Map(TranslationEngineConfigDto source) - { - return new Engine - { - Name = source.Name, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - Type = source.Type.ToPascalCase(), - Owner = Owner, - Corpora = [], - IsModelPersisted = source.IsModelPersisted, - }; - } - - private static Build Map(Engine engine, TranslationBuildConfigDto source, string deploymentVersion) - { - return new Build - { - EngineRef = engine.Id, - Owner = engine.Owner, - Name = source.Name, - Pretranslate = Map(engine, source.Pretranslate), - TrainOn = Map(engine, source.TrainOn), - Options = MapOptions(source.Options), - DeploymentVersion = deploymentVersion, - }; - } - - private static List? Map(Engine engine, IReadOnlyList? source) - { - if (source is null) - return null; - - if ( - source.Where(p => p.ParallelCorpusId != null).Select(p => p.ParallelCorpusId).Distinct().Count() - != source.Count(p => p.ParallelCorpusId != null) - ) - { - throw new InvalidOperationException($"Each ParallelCorpusId may only be specified once."); - } - - if ( - source.Where(p => p.CorpusId != null).Select(p => p.CorpusId).Distinct().Count() - != source.Count(p => p.CorpusId != null) - ) - { - throw new InvalidOperationException($"Each CorpusId may only be specified once."); - } - - var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); - var parallelCorpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); - var pretranslateCorpora = new List(); - foreach (PretranslateCorpusConfigDto pcc in source) - { - if (pcc.CorpusId != null) - { - if (pcc.ParallelCorpusId != null) - { - throw new InvalidOperationException($"Only one of ParallelCorpusId and CorpusId can be set."); - } - if (!corpusIds.Contains(pcc.CorpusId)) - { - throw new InvalidOperationException( - $"The corpus {pcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." - ); - } - Corpus corpus = engine.Corpora.Single(c => c.Id == pcc.CorpusId); - if (corpus.SourceFiles.Count == 0 && corpus.TargetFiles.Count == 0) - { - throw new InvalidOperationException( - $"The corpus {pcc.CorpusId} is not valid: This corpus does not have any source or target files." - ); - } - if (pcc.TextIds != null && pcc.ScriptureRange != null) - { - throw new InvalidOperationException( - $"The corpus {pcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." - ); - } - pretranslateCorpora.Add( - new PretranslateCorpus - { - CorpusRef = pcc.CorpusId, - TextIds = pcc.TextIds?.ToList(), - ScriptureRange = pcc.ScriptureRange, - } - ); - } - else - { - if (pcc.ParallelCorpusId == null) - { - throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); - } - if (!parallelCorpusIds.Contains(pcc.ParallelCorpusId)) - { - throw new InvalidOperationException( - $"The parallel corpus {pcc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." - ); - } - ParallelCorpus corpus = engine.ParallelCorpora.Single(pc => pc.Id == pcc.ParallelCorpusId); - if (corpus.SourceCorpora.Count == 0 && corpus.TargetCorpora.Count == 0) - { - throw new InvalidOperationException( - $"The corpus {pcc.ParallelCorpusId} does not have source or target corpora associated with it." - ); - } - if ( - pcc.SourceFilters != null - && pcc.SourceFilters.Count > 0 - && ( - pcc.SourceFilters.Select(sf => sf.CorpusId).Distinct().Count() > 1 - || pcc.SourceFilters[0].CorpusId - != engine.ParallelCorpora.Single(pc => pc.Id == pcc.ParallelCorpusId).SourceCorpora[0].Id - ) - ) - { - throw new InvalidOperationException( - $"Only the first source corpus in a parallel corpus may be filtered for pretranslation." - ); - } - pretranslateCorpora.Add( - new PretranslateCorpus - { - ParallelCorpusRef = pcc.ParallelCorpusId, - SourceFilters = pcc.SourceFilters?.Select(Map).ToList(), - } - ); - } - } - return pretranslateCorpora; - } - - private static List? Map(Engine engine, IReadOnlyList? source) - { - if (source is null) - return null; - - if ( - source.Where(p => p.ParallelCorpusId != null).Select(p => p.ParallelCorpusId).Distinct().Count() - != source.Count(p => p.ParallelCorpusId != null) - ) - { - throw new InvalidOperationException($"Each ParallelCorpusId may only be specified once."); - } - - if ( - source.Where(p => p.CorpusId != null).Select(p => p.CorpusId).Distinct().Count() - != source.Count(p => p.CorpusId != null) - ) - { - throw new InvalidOperationException($"Each CorpusId may only be specified once."); - } - - var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); - var parallelCorpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); - var trainOnCorpora = new List(); - foreach (TrainingCorpusConfigDto tcc in source) - { - if (tcc.CorpusId != null) - { - if (tcc.ParallelCorpusId != null) - { - throw new InvalidOperationException($"Only one of ParallelCorpusId and CorpusId can be set."); - } - if (!corpusIds.Contains(tcc.CorpusId)) - { - throw new InvalidOperationException( - $"The corpus {tcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." - ); - } - Corpus corpus = engine.Corpora.Single(c => c.Id == tcc.CorpusId); - if (corpus.SourceFiles.Count == 0 && corpus.TargetFiles.Count == 0) - { - throw new InvalidOperationException( - $"The corpus {tcc.CorpusId} is not valid: This corpus does not have any source or target files." - ); - } - if (tcc.TextIds != null && tcc.ScriptureRange != null) - { - throw new InvalidOperationException( - $"The corpus {tcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." - ); - } - trainOnCorpora.Add( - new TrainingCorpus - { - CorpusRef = tcc.CorpusId, - TextIds = tcc.TextIds?.ToList(), - ScriptureRange = tcc.ScriptureRange, - } - ); - } - else - { - if (tcc.ParallelCorpusId == null) - { - throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); - } - if (!parallelCorpusIds.Contains(tcc.ParallelCorpusId)) - { - throw new InvalidOperationException( - $"The parallel corpus {tcc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." - ); - } - ParallelCorpus corpus = engine.ParallelCorpora.Single(pc => pc.Id == tcc.ParallelCorpusId); - if (corpus.SourceCorpora.Count == 0 && corpus.TargetCorpora.Count == 0) - { - throw new InvalidOperationException( - $"The corpus {tcc.ParallelCorpusId} does not have source or target corpora associated with it." - ); - } - foreach (MonolingualCorpus monolingualCorpus in corpus.SourceCorpora.Concat(corpus.TargetCorpora)) - { - if (monolingualCorpus.Files.Count == 0) - { - throw new InvalidOperationException( - $"The corpus {monolingualCorpus.Id} referenced in parallel corpus {corpus.Id} does not have any files associated with it." - ); - } - } - trainOnCorpora.Add( - new TrainingCorpus - { - ParallelCorpusRef = tcc.ParallelCorpusId, - SourceFilters = tcc.SourceFilters?.Select(Map).ToList(), - TargetFilters = tcc.TargetFilters?.Select(Map).ToList(), - } - ); - } - } - return trainOnCorpora; - } - - private static ParallelCorpusFilter Map(ParallelCorpusFilterConfigDto source) - { - if (source.TextIds != null && source.ScriptureRange != null) - { - throw new InvalidOperationException( - $"The parallel corpus filter for corpus {source.CorpusId} is not valid: At most, one of TextIds and ScriptureRange can be set." - ); - } - return new ParallelCorpusFilter - { - CorpusRef = source.CorpusId, - TextIds = source.TextIds, - ScriptureRange = source.ScriptureRange, - }; - } - - private static Dictionary? MapOptions(object? source) - { - try - { - return JsonSerializer.Deserialize>( - source?.ToString() ?? "{}", - ObjectJsonSerializerOptions - ); - } - catch (Exception e) - { - throw new InvalidOperationException($"Unable to parse field 'options' : {e.Message}", e); - } - } - - private TranslationEngineDto Map(Engine source) - { - return new TranslationEngineDto - { - Id = source.Id, - Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.Id }), - Name = source.Name, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - Type = source.Type.ToKebabCase(), - IsModelPersisted = source.IsModelPersisted, - IsBuilding = source.IsBuilding, - ModelRevision = source.ModelRevision, - Confidence = Math.Round(source.Confidence, 8), - CorpusSize = source.CorpusSize, - DateCreated = source.DateCreated, - }; - } - - private TranslationResultDto Map(TranslationResult source) - { - return new TranslationResultDto - { - Translation = source.Translation, - SourceTokens = source.SourceTokens.ToList(), - TargetTokens = source.TargetTokens.ToList(), - Confidences = source.Confidences.Select(c => Math.Round(c, 8)).ToList(), - Sources = source.Sources.ToList(), - Alignment = source.Alignment.Select(Map).ToList(), - Phrases = source.Phrases.Select(Map).ToList(), - }; - } - - private AlignedWordPairDto Map(AlignedWordPair source) - { - return new AlignedWordPairDto { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; - } - - private static PhraseDto Map(Phrase source) - { - return new PhraseDto - { - SourceSegmentStart = source.SourceSegmentStart, - SourceSegmentEnd = source.SourceSegmentEnd, - TargetSegmentCut = source.TargetSegmentCut, - }; - } - - private WordGraphDto Map(WordGraph source) - { - return new WordGraphDto - { - SourceTokens = source.SourceTokens.ToList(), - InitialStateScore = (float)source.InitialStateScore, - FinalStates = source.FinalStates.ToHashSet(), - Arcs = source.Arcs.Select(Map).ToList(), - }; - } - - private WordGraphArcDto Map(WordGraphArc source) - { - return new WordGraphArcDto - { - PrevState = source.PrevState, - NextState = source.NextState, - Score = Math.Round(source.Score, 8), - TargetTokens = source.TargetTokens.ToList(), - Confidences = source.Confidences.Select(c => Math.Round(c, 8)).ToList(), - SourceSegmentStart = source.SourceSegmentStart, - SourceSegmentEnd = source.SourceSegmentEnd, - Alignment = source.Alignment.Select(Map).ToList(), - Sources = source.Sources.ToList(), - }; - } - private static PretranslationDto Map(Pretranslation source) { return new PretranslationDto @@ -2067,16 +1215,6 @@ private TranslationCorpusFileDto Map(CorpusFile source) TextId = source.TextId, }; } - - private static ModelDownloadUrlDto Map(ModelDownloadUrl source) - { - return new ModelDownloadUrlDto - { - Url = source.Url, - ModelRevision = source.ModelRevision, - ExpiresAt = source.ExpiresAt, - }; - } } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/src/Serval.Translation/Features/Engines/UpdateEngine.cs b/src/Serval/src/Serval.Translation/Features/Engines/UpdateEngine.cs new file mode 100644 index 000000000..8a4673c48 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Features/Engines/UpdateEngine.cs @@ -0,0 +1,106 @@ +namespace Serval.Translation.Features.Engines; + +public record TranslationEngineUpdateConfigDto +{ + public string? SourceLanguage { get; init; } + + public string? TargetLanguage { get; init; } +} + +public record UpdateEngine(string Owner, string EngineId, TranslationEngineUpdateConfigDto UpdateConfig) : IRequest; + +public class UpdateEngineHandler( + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository pretranslations, + IEngineServiceFactory engineServiceFactory +) : IRequestHandler +{ + public Task HandleAsync(UpdateEngine request, CancellationToken cancellationToken) + { + return dataAccessContext.WithTransactionAsync( + async (ct) => + { + Engine? engine = await engines.GetAsync(request.EngineId, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + if (engine.Owner != request.Owner) + throw new ForbiddenException(); + + engine = await engines.UpdateAsync( + request.EngineId, + u => + { + if (request.UpdateConfig.SourceLanguage is not null) + u.Set(e => e.SourceLanguage, request.UpdateConfig.SourceLanguage); + if (request.UpdateConfig.TargetLanguage is not null) + u.Set(e => e.TargetLanguage, request.UpdateConfig.TargetLanguage); + }, + cancellationToken: ct + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{request.EngineId}'."); + await pretranslations.DeleteAllAsync(pt => pt.EngineRef == request.EngineId, ct); + + await engineServiceFactory + .GetEngineService(engine.Type) + .UpdateAsync( + request.EngineId, + request.UpdateConfig.SourceLanguage, + request.UpdateConfig.TargetLanguage, + ct + ); + }, + cancellationToken + ); + } +} + +public partial class TranslationEnginesController +{ + /// + /// Update the source and/or target languages of a translation engine + /// + /// + /// ## Sample request: + /// + /// { + /// "sourceLanguage": "en", + /// "targetLanguage": "en" + /// } + /// + /// + /// The translation engine id + /// + /// The engine language was successfully updated. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist and therefore cannot be updated. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateTranslationEngines)] + [HttpPatch("{id}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task UpdateAsync( + [FromRoute] string id, + [FromBody] TranslationEngineUpdateConfigDto request, + [FromServices] IRequestHandler handler, + CancellationToken cancellationToken = default + ) + { + if ( + request is null + || (string.IsNullOrWhiteSpace(request.SourceLanguage) && string.IsNullOrWhiteSpace(request.TargetLanguage)) + ) + { + return BadRequest("sourceLanguage or targetLanguage is required."); + } + + await handler.HandleAsync(new(Owner, id, request), cancellationToken); + + return Ok(); + } +} diff --git a/src/Serval/src/Serval.Translation/Handlers/CorpusUpdatedHandler.cs b/src/Serval/src/Serval.Translation/Handlers/CorpusUpdatedHandler.cs new file mode 100644 index 000000000..2cdcd2c77 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Handlers/CorpusUpdatedHandler.cs @@ -0,0 +1,20 @@ +namespace Serval.Translation.Handlers; + +public class CorpusUpdatedHandler(IEngineService engineService) : IEventHandler +{ + public Task HandleAsync(CorpusUpdated evt, CancellationToken cancellationToken) + { + return engineService.UpdateCorpusFilesAsync(evt.CorpusId, [.. evt.Files.Select(Map)], cancellationToken); + } + + private static CorpusFile Map(CorpusDataFileContract corpusFile) + { + return new CorpusFile + { + Id = corpusFile.File.DataFileId, + TextId = corpusFile.TextId ?? corpusFile.File.Name, + Filename = corpusFile.File.Filename, + Format = corpusFile.File.Format, + }; + } +} diff --git a/src/Serval/src/Serval.Translation/Handlers/DataFileDeletedHandler.cs b/src/Serval/src/Serval.Translation/Handlers/DataFileDeletedHandler.cs new file mode 100644 index 000000000..8f5f4e436 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Handlers/DataFileDeletedHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.Translation.Handlers; + +public class DataFileDeletedHandler(IEngineService engineService) : IEventHandler +{ + public async Task HandleAsync(DataFileDeleted evt, CancellationToken cancellationToken) + { + await engineService.DeleteAllCorpusFilesAsync(evt.DataFileId, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.Translation/Handlers/DataFileUpdatedHandler.cs b/src/Serval/src/Serval.Translation/Handlers/DataFileUpdatedHandler.cs new file mode 100644 index 000000000..dc44fc4c6 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Handlers/DataFileUpdatedHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.Translation.Handlers; + +public class DataFileUpdatedHandler(IEngineService engineService) : IEventHandler +{ + public Task HandleAsync(DataFileUpdated evt, CancellationToken cancellationToken) + { + return engineService.UpdateDataFileFilenameFilesAsync(evt.DataFileId, evt.Filename, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs b/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs deleted file mode 100644 index e4dbfa330..000000000 --- a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Translation.Models; - -public record AlignedWordPair -{ - public required int SourceIndex { get; set; } - public required int TargetIndex { get; set; } -} diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 749b0b8ce..4363b5a58 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -21,7 +21,7 @@ public record Build : IOwnedEntity public DateTime? DateCreated { get; set; } public DateTime? DateStarted { get; set; } public DateTime? DateCompleted { get; set; } - public IReadOnlyList? Phases { get; init; } + public IReadOnlyList? Phases { get; init; } public IReadOnlyList? Analysis { get; init; } public string? TargetQuoteConvention { get; init; } } diff --git a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs b/src/Serval/src/Serval.Translation/Models/CorpusFile.cs similarity index 84% rename from src/Serval/src/Serval.Shared/Models/CorpusFile.cs rename to src/Serval/src/Serval.Translation/Models/CorpusFile.cs index 2739e6059..2672ba56a 100644 --- a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.Translation/Models/CorpusFile.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record CorpusFile { diff --git a/src/Serval/src/Serval.Translation/Models/LanguageInfo.cs b/src/Serval/src/Serval.Translation/Models/LanguageInfo.cs deleted file mode 100644 index 71a82e9f4..000000000 --- a/src/Serval/src/Serval.Translation/Models/LanguageInfo.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Models; - -public record LanguageInfo -{ - public required string EngineType { get; set; } - public required bool IsNative { get; set; } - public string? InternalCode { get; set; } -} diff --git a/src/Serval/src/Serval.Translation/Models/ModelDownloadUrl.cs b/src/Serval/src/Serval.Translation/Models/ModelDownloadUrl.cs deleted file mode 100644 index 3f89de6fe..000000000 --- a/src/Serval/src/Serval.Translation/Models/ModelDownloadUrl.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Models; - -public class ModelDownloadUrl -{ - public string Url { get; set; } = default!; - public int ModelRevision { get; set; } = default!; - public DateTime ExpiresAt { get; set; } = default!; -} diff --git a/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs similarity index 86% rename from src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs rename to src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs index f9c58fb4c..0762e8789 100644 --- a/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs +++ b/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record MonolingualCorpus { diff --git a/src/Serval/src/Serval.Shared/Models/ParallelCorpus.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs similarity index 85% rename from src/Serval/src/Serval.Shared/Models/ParallelCorpus.cs rename to src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs index c0c2c1c30..9f3b36eb4 100644 --- a/src/Serval/src/Serval.Shared/Models/ParallelCorpus.cs +++ b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record ParallelCorpus { diff --git a/src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs similarity index 80% rename from src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs rename to src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs index 026bee9d4..3c233dcb6 100644 --- a/src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs +++ b/src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record ParallelCorpusAnalysis { diff --git a/src/Serval/src/Serval.Shared/Models/ParallelCorpusFilter.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs similarity index 84% rename from src/Serval/src/Serval.Shared/Models/ParallelCorpusFilter.cs rename to src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs index a6b0aec33..e13f6dd2e 100644 --- a/src/Serval/src/Serval.Shared/Models/ParallelCorpusFilter.cs +++ b/src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record ParallelCorpusFilter { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationQuotationMarkBehavior.cs b/src/Serval/src/Serval.Translation/Models/PretranslationQuotationMarkBehavior.cs similarity index 68% rename from src/Serval/src/Serval.Translation/Contracts/PretranslationQuotationMarkBehavior.cs rename to src/Serval/src/Serval.Translation/Models/PretranslationQuotationMarkBehavior.cs index 4861f4548..0abccbffe 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationQuotationMarkBehavior.cs +++ b/src/Serval/src/Serval.Translation/Models/PretranslationQuotationMarkBehavior.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Models; public enum PretranslationNormalizationBehavior { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmMarkerBehavior.cs b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmMarkerBehavior.cs similarity index 69% rename from src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmMarkerBehavior.cs rename to src/Serval/src/Serval.Translation/Models/PretranslationUsfmMarkerBehavior.cs index b0fcf7c45..3b03b2314 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmMarkerBehavior.cs +++ b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmMarkerBehavior.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Models; public enum PretranslationUsfmMarkerBehavior { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTemplate.cs b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmTemplate.cs similarity index 64% rename from src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTemplate.cs rename to src/Serval/src/Serval.Translation/Models/PretranslationUsfmTemplate.cs index c793e9082..9a7c0ac69 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTemplate.cs +++ b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmTemplate.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Models; public enum PretranslationUsfmTemplate { diff --git a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTextOrigin.cs b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmTextOrigin.cs similarity index 75% rename from src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTextOrigin.cs rename to src/Serval/src/Serval.Translation/Models/PretranslationUsfmTextOrigin.cs index b55075749..bfb7a2c4f 100644 --- a/src/Serval/src/Serval.Translation/Contracts/PretranslationUsfmTextOrigin.cs +++ b/src/Serval/src/Serval.Translation/Models/PretranslationUsfmTextOrigin.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Models; public enum PretranslationUsfmTextOrigin { diff --git a/src/Serval/src/Serval.Translation/Properties/AssemblyInfo.cs b/src/Serval/src/Serval.Translation/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..0dc703375 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Properties/AssemblyInfo.cs @@ -0,0 +1 @@ +[assembly: InternalsVisibleTo("Serval.Translation.Tests")] diff --git a/src/Serval/src/Serval.Translation/Serval.Translation.csproj b/src/Serval/src/Serval.Translation/Serval.Translation.csproj index db989964c..6948ce0c8 100644 --- a/src/Serval/src/Serval.Translation/Serval.Translation.csproj +++ b/src/Serval/src/Serval.Translation/Serval.Translation.csproj @@ -10,18 +10,18 @@ $(NoWarn);CS1591;CS1573 + + - - - - + + diff --git a/src/Serval/src/Serval.Translation/Services/CorpusMappingService.cs b/src/Serval/src/Serval.Translation/Services/ContractMapper.cs similarity index 69% rename from src/Serval/src/Serval.Translation/Services/CorpusMappingService.cs rename to src/Serval/src/Serval.Translation/Services/ContractMapper.cs index 1667e5a30..47015ec02 100644 --- a/src/Serval/src/Serval.Translation/Services/CorpusMappingService.cs +++ b/src/Serval/src/Serval.Translation/Services/ContractMapper.cs @@ -1,14 +1,14 @@ namespace Serval.Translation.Services; -public class CorpusMappingService( +public class ContractMapper( IOptionsMonitor dataFileOptions, IParallelCorpusService parallelCorpusService -) : ICorpusMappingService +) { private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; - public IReadOnlyList Map(Build build, Engine engine) + public IReadOnlyList Map(Build build, Engine engine) { if (engine.ParallelCorpora.Any()) { @@ -20,13 +20,9 @@ IParallelCorpusService parallelCorpusService } } - public IReadOnlyList Map( - Build build, - Engine engine, - IReadOnlyList corpora - ) + private List Map(Build build, Engine engine, IReadOnlyList corpora) { - List mappedParallelCorpora = []; + List mappedParallelCorpora = []; Dictionary? trainingCorpora = build.TrainOn?.ToDictionary(c => c.CorpusRef!); Dictionary? pretranslateCorpora = build.Pretranslate?.ToDictionary(c => @@ -47,23 +43,23 @@ Corpus source in corpora.Where(c => TrainingCorpus? trainingCorpus = trainingCorpora?.GetValueOrDefault(source.Id); PretranslateCorpus? pretranslateCorpus = pretranslateCorpora?.GetValueOrDefault(source.Id); - IEnumerable sourceFiles = source.SourceFiles.Select(Map); - IEnumerable targetFiles = source.TargetFiles.Select(Map); - SIL.ServiceToolkit.Models.MonolingualCorpus sourceCorpus = new() + IEnumerable sourceFiles = source.SourceFiles.Select(Map); + IEnumerable targetFiles = source.TargetFiles.Select(Map); + MonolingualCorpusContract sourceCorpus = new() { Id = source.Id, Language = source.SourceLanguage, - Files = source.SourceFiles.Select(Map).ToArray(), - TrainOnAll = trainOnAllCorpora, - PretranslateAll = pretranslateAllCorpora, + Files = [.. source.SourceFiles.Select(Map)], + TrainOnTextIds = trainOnAllCorpora || trainingCorpus is not null ? null : [], + InferenceTextIds = pretranslateAllCorpora || pretranslateCorpus is not null ? null : [], }; - SIL.ServiceToolkit.Models.MonolingualCorpus targetCorpus = new() + MonolingualCorpusContract targetCorpus = new() { Id = source.Id, Language = source.TargetLanguage, - Files = source.TargetFiles.Select(Map).ToArray(), - TrainOnAll = trainOnAllCorpora, - PretranslateAll = pretranslateAllCorpora, + Files = [.. source.TargetFiles.Select(Map)], + TrainOnTextIds = trainOnAllCorpora || trainingCorpus is not null ? null : [], + InferenceTextIds = pretranslateAllCorpora || pretranslateCorpus is not null ? null : [], }; if (trainingCorpus is not null) @@ -79,10 +75,7 @@ Corpus source in corpora.Where(c => if (trainingCorpus.ScriptureRange is not null) { - if ( - targetCorpus.Files.Count > 1 - || targetCorpus.Files[0].Format != SIL.ServiceToolkit.Models.FileFormat.Paratext - ) + if (targetCorpus.Files.Count > 1 || targetCorpus.Files[0].Format != FileFormat.Paratext) { throw new InvalidOperationException( $"The corpus {source.Id} is not compatible with using a scripture range" @@ -90,7 +83,7 @@ Corpus source in corpora.Where(c => } var chapters = _parallelCorpusService .GetChapters( - corpora.Select(c => Map(c, engine)).ToArray(), + [.. corpora.Select(c => Map(c, engine))], GetFilePath(targetCorpus.Files[0].Location), trainingCorpus.ScriptureRange ) @@ -98,8 +91,6 @@ Corpus source in corpora.Where(c => sourceCorpus.TrainOnChapters = chapters; targetCorpus.TrainOnChapters = chapters; } - sourceCorpus.TrainOnAll = sourceCorpus.TrainOnChapters is null && sourceCorpus.TrainOnTextIds is null; - targetCorpus.TrainOnAll = targetCorpus.TrainOnChapters is null && targetCorpus.TrainOnTextIds is null; } if (pretranslateCorpus is not null) @@ -113,10 +104,7 @@ Corpus source in corpora.Where(c => sourceCorpus.InferenceTextIds = pretranslateCorpus.TextIds?.ToHashSet(); if (pretranslateCorpus.ScriptureRange is not null) { - if ( - targetCorpus.Files.Count > 1 - || targetCorpus.Files[0].Format != SIL.ServiceToolkit.Models.FileFormat.Paratext - ) + if (targetCorpus.Files.Count > 1 || targetCorpus.Files[0].Format != FileFormat.Paratext) { throw new InvalidOperationException( $"The corpus {source.Id} is not compatible with using a scripture range" @@ -124,18 +112,14 @@ Corpus source in corpora.Where(c => } sourceCorpus.InferenceChapters = _parallelCorpusService .GetChapters( - corpora.Select(c => Map(c, engine)).ToArray(), + [.. corpora.Select(c => Map(c, engine))], GetFilePath(targetCorpus.Files[0].Location), pretranslateCorpus.ScriptureRange ) .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToHashSet()); } - sourceCorpus.PretranslateAll = - sourceCorpus.InferenceChapters is null && sourceCorpus.InferenceTextIds is null; - targetCorpus.PretranslateAll = - targetCorpus.InferenceChapters is null && targetCorpus.InferenceTextIds is null; } - SIL.ServiceToolkit.Models.ParallelCorpus corpus = new() + ParallelCorpusContract corpus = new() { Id = source.Id, SourceCorpora = [sourceCorpus], @@ -146,12 +130,9 @@ Corpus source in corpora.Where(c => return mappedParallelCorpora; } - private IReadOnlyList Map( - Build build, - IReadOnlyList parallelCorpora - ) + private IReadOnlyList Map(Build build, IReadOnlyList parallelCorpora) { - List mappedParallelCorpora = []; + List mappedParallelCorpora = []; Dictionary? trainingCorpora = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!); Dictionary? pretranslateCorpora = build.Pretranslate?.ToDictionary(c => c.ParallelCorpusRef! @@ -179,11 +160,12 @@ IReadOnlyList parallelCorpora : null; mappedParallelCorpora.Add( - new SIL.ServiceToolkit.Models.ParallelCorpus + new ParallelCorpusContract { Id = source.Id, - SourceCorpora = source - .SourceCorpora.Select(sc => + SourceCorpora = + [ + .. source.SourceCorpora.Select(sc => Map( parallelCorpora, sc, @@ -195,10 +177,11 @@ IReadOnlyList parallelCorpora pretranslateAllCorpora || (pretranslateCorpus is not null && pretranslateCorpus.SourceFilters is null) ) - ) - .ToArray(), - TargetCorpora = source - .TargetCorpora.Select(tc => + ), + ], + TargetCorpora = + [ + .. source.TargetCorpora.Select(tc => Map( parallelCorpora, tc, @@ -209,15 +192,15 @@ IReadOnlyList parallelCorpora || (trainingCorpus is not null && trainingCorpus.TargetFilters is null), pretranslateAllCorpora || pretranslateCorpus is not null ) - ) - .ToArray(), + ), + ], } ); } return mappedParallelCorpora; } - private SIL.ServiceToolkit.Models.MonolingualCorpus Map( + private MonolingualCorpusContract Map( IReadOnlyList parallelCorpora, MonolingualCorpus inputCorpus, ParallelCorpusFilter? trainingFilter, @@ -236,7 +219,7 @@ trainingFilter is not null { trainOnChapters = _parallelCorpusService .GetChapters( - parallelCorpora.Select(Map).ToArray(), + [.. parallelCorpora.Select(Map)], GetFilePath(referenceFileLocation), trainingFilter.ScriptureRange ) @@ -252,20 +235,20 @@ pretranslateFilter is not null { pretranslateChapters = _parallelCorpusService .GetChapters( - parallelCorpora.Select(Map).ToArray(), + [.. parallelCorpora.Select(Map)], GetFilePath(referenceFileLocation), pretranslateFilter.ScriptureRange ) .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToHashSet()); } - var returnCorpus = new SIL.ServiceToolkit.Models.MonolingualCorpus + var returnCorpus = new MonolingualCorpusContract { Id = inputCorpus.Id, Language = inputCorpus.Language, - Files = inputCorpus.Files.Select(Map).ToArray(), - TrainOnAll = trainOnAll, - PretranslateAll = pretranslateAll, + Files = [.. inputCorpus.Files.Select(Map)], + TrainOnTextIds = trainOnAll ? null : [], + InferenceTextIds = pretranslateAll ? null : [], }; if ( @@ -280,7 +263,8 @@ trainingFilter is not null } returnCorpus.TrainOnChapters = trainOnChapters; - returnCorpus.TrainOnTextIds = trainingFilter?.TextIds?.ToHashSet(); + if (trainingFilter is not null) + returnCorpus.TrainOnTextIds = trainingFilter.TextIds?.ToHashSet(); if ( pretranslateFilter is not null @@ -294,24 +278,25 @@ pretranslateFilter is not null } returnCorpus.InferenceChapters = pretranslateChapters; - returnCorpus.InferenceTextIds = pretranslateFilter?.TextIds?.ToHashSet(); + if (pretranslateFilter is not null) + returnCorpus.InferenceTextIds = pretranslateFilter.TextIds?.ToHashSet(); return returnCorpus; } - public SIL.ServiceToolkit.Models.ParallelCorpus Map(Corpus source, Engine engine) + public ParallelCorpusContract Map(Corpus source, Engine engine) { - return new SIL.ServiceToolkit.Models.ParallelCorpus + return new ParallelCorpusContract { Id = source.Id, - SourceCorpora = source.SourceFiles.Select(f => Map(f, engine.SourceLanguage)).ToArray(), - TargetCorpora = source.TargetFiles.Select(f => Map(f, engine.TargetLanguage)).ToArray(), + SourceCorpora = [.. source.SourceFiles.Select(f => Map(f, engine.SourceLanguage))], + TargetCorpora = [.. source.TargetFiles.Select(f => Map(f, engine.TargetLanguage))], }; } - private SIL.ServiceToolkit.Models.MonolingualCorpus Map(CorpusFile source, string language) + private MonolingualCorpusContract Map(CorpusFile source, string language) { - return new SIL.ServiceToolkit.Models.MonolingualCorpus + return new MonolingualCorpusContract { Id = source.Id, Language = language, @@ -319,29 +304,29 @@ private SIL.ServiceToolkit.Models.MonolingualCorpus Map(CorpusFile source, strin }; } - private SIL.ServiceToolkit.Models.CorpusFile Map(CorpusFile source) + private CorpusFileContract Map(CorpusFile source) { - return new SIL.ServiceToolkit.Models.CorpusFile + return new CorpusFileContract { Location = GetFilePath(source.Filename), - Format = (SIL.ServiceToolkit.Models.FileFormat)source.Format, + Format = source.Format, TextId = source.TextId, }; } - private SIL.ServiceToolkit.Models.ParallelCorpus Map(ParallelCorpus source) + private ParallelCorpusContract Map(ParallelCorpus source) { - return new SIL.ServiceToolkit.Models.ParallelCorpus + return new ParallelCorpusContract { Id = source.Id, - SourceCorpora = source.SourceCorpora.Select(Map).ToArray(), - TargetCorpora = source.TargetCorpora.Select(Map).ToArray(), + SourceCorpora = [.. source.SourceCorpora.Select(Map)], + TargetCorpora = [.. source.TargetCorpora.Select(Map)], }; } - private SIL.ServiceToolkit.Models.MonolingualCorpus Map(MonolingualCorpus source) + private MonolingualCorpusContract Map(MonolingualCorpus source) { - return new SIL.ServiceToolkit.Models.MonolingualCorpus + return new MonolingualCorpusContract { Id = source.Id, Language = source.Language, diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationControllerBase.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Controllers/TranslationControllerBase.cs rename to src/Serval/src/Serval.Translation/Services/DtoMapper.cs index 79c4fcfdc..e76f8be52 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationControllerBase.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -1,17 +1,35 @@ -namespace Serval.Translation.Controllers; +namespace Serval.Translation.Services; #pragma warning disable CS0612 // Type or member is obsolete -public abstract class TranslationControllerBase(IAuthorizationService authService, IUrlService urlService) - : ServalControllerBase(authService) +public class DtoMapper(IUrlService urlService) { private readonly IUrlService _urlService = urlService; - protected TranslationBuildDto Map(Build source) + public TranslationEngineDto Map(Engine source) + { + return new() + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.Id }), + Name = source.Name, + SourceLanguage = source.SourceLanguage, + TargetLanguage = source.TargetLanguage, + Type = source.Type.ToKebabCase(), + IsModelPersisted = source.IsModelPersisted, + IsBuilding = source.IsBuilding, + ModelRevision = source.ModelRevision, + Confidence = Math.Round(source.Confidence, 8), + CorpusSize = source.CorpusSize, + DateCreated = source.DateCreated, + }; + } + + public TranslationBuildDto Map(Build source) { string targetQuoteConvention = source.TargetQuoteConvention ?? ""; - return new TranslationBuildDto + return new() { Id = source.Id, Url = _urlService.GetUrl(Endpoints.GetTranslationBuild, new { id = source.EngineRef, buildId = source.Id }), @@ -45,7 +63,7 @@ protected TranslationBuildDto Map(Build source) } private PretranslateCorpusDto Map(string engineId, PretranslateCorpus source) => - new PretranslateCorpusDto + new() { Corpus = source.CorpusRef != null @@ -75,7 +93,7 @@ private PretranslateCorpusDto Map(string engineId, PretranslateCorpus source) => }; private TrainingCorpusDto Map(string engineId, TrainingCorpus source) => - new TrainingCorpusDto + new() { Corpus = source.CorpusRef != null @@ -106,7 +124,7 @@ private TrainingCorpusDto Map(string engineId, TrainingCorpus source) => }; private ParallelCorpusFilterDto Map(ParallelCorpusFilter source) => - new ParallelCorpusFilterDto + new() { Corpus = new ResourceLinkDto { @@ -117,17 +135,17 @@ private ParallelCorpusFilterDto Map(ParallelCorpusFilter source) => ScriptureRange = source.ScriptureRange, }; - private static PhaseDto Map(BuildPhase source) => - new PhaseDto + private static PhaseDto Map(Phase source) => + new() { - Stage = (PhaseStage)source.Stage, + Stage = source.Stage, Step = source.Step, StepCount = source.StepCount, Started = source.Started, }; private static ParallelCorpusAnalysisDto Map(ParallelCorpusAnalysis source, string targetQuoteConvention) => - new ParallelCorpusAnalysisDto + new() { ParallelCorpusRef = source.ParallelCorpusRef, TargetQuoteConvention = targetQuoteConvention, @@ -136,7 +154,7 @@ private static ParallelCorpusAnalysisDto Map(ParallelCorpusAnalysis source, stri }; private static ExecutionDataDto Map(ExecutionData source) => - new ExecutionDataDto + new() { TrainCount = source.TrainCount ?? 0, PretranslateCount = source.PretranslateCount ?? 0, diff --git a/src/Serval/src/Serval.Translation/Services/EngineOutboxConstants.cs b/src/Serval/src/Serval.Translation/Services/EngineOutboxConstants.cs deleted file mode 100644 index 8625d9ba1..000000000 --- a/src/Serval/src/Serval.Translation/Services/EngineOutboxConstants.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Translation.Services; - -public static class EngineOutboxConstants -{ - public const string OutboxId = "TranslationEngine"; - public const string Create = "Create"; - public const string Update = "Update"; - public const string Delete = "Delete"; - public const string StartBuild = "StartBuild"; - public const string CancelBuild = "CancelBuild"; -} diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index e32d3bb5a..b96945c07 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -1,391 +1,15 @@ -using MassTransit.Mediator; -using Serval.Translation.V1; - namespace Serval.Translation.Services; public class EngineService( IRepository engines, - IRepository builds, IRepository pretranslations, - IScopedMediator mediator, - GrpcClientFactory grpcClientFactory, - IDataAccessContext dataAccessContext, - ILoggerFactory loggerFactory, - IOutboxService outboxService, - IOptionsMonitor translationOptions, - ICorpusMappingService corpusMappingService + IRequestHandler deleteDataFileHandler, + IDataAccessContext dataAccessContext ) : OwnedEntityServiceBase(engines), IEngineService { - private readonly IRepository _builds = builds; private readonly IRepository _pretranslations = pretranslations; - private readonly IScopedMediator _mediator = mediator; - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; + private readonly IRequestHandler _deleteDataFileHandler = deleteDataFileHandler; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly ILogger _logger = loggerFactory.CreateLogger(); - private readonly IOutboxService _outboxService = outboxService; - private readonly IOptionsMonitor _translationOptions = translationOptions; - private readonly ICorpusMappingService _corpusMappingService = corpusMappingService; - - public async Task TranslateAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Engine engine = await GetAsync(engineId, cancellationToken); - if (engine.ModelRevision == 0) - return null; - - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - try - { - TranslateResponse response = await client.TranslateAsync( - new TranslateRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - N = 1, - Segment = segment, - }, - cancellationToken: cancellationToken - ); - return Map(response.Results[0]); - } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) - { - return null; - } - } - - public async Task?> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - Engine engine = await GetAsync(engineId, cancellationToken); - if (engine.ModelRevision == 0) - return null; - - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - try - { - TranslateResponse response = await client.TranslateAsync( - new TranslateRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - N = n, - Segment = segment, - }, - cancellationToken: cancellationToken - ); - return response.Results.Select(Map); - } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) - { - return null; - } - } - - public async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Engine engine = await GetAsync(engineId, cancellationToken); - if (engine.ModelRevision == 0) - return null; - - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - try - { - GetWordGraphResponse response = await client.GetWordGraphAsync( - new GetWordGraphRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - Segment = segment, - }, - cancellationToken: cancellationToken - ); - return Map(response.WordGraph); - } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) - { - return null; - } - } - - public async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - Engine engine = await GetAsync(engineId, cancellationToken); - if (engine.ModelRevision == 0) - return false; - - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - try - { - await client.TrainSegmentPairAsync( - new TrainSegmentPairRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - SourceSegment = sourceSegment, - TargetSegment = targetSegment, - SentenceStart = sentenceStart, - }, - cancellationToken: cancellationToken - ); - return true; - } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) - { - return false; - } - } - - public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) - { - if (!_translationOptions.CurrentValue.Engines.Any(e => e.Type == engine.Type)) - throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type."); - - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - engine.DateCreated = DateTime.UtcNow; - await Entities.InsertAsync(engine, ct); - - CreateRequest request = new() - { - EngineType = engine.Type, - EngineId = engine.Id, - SourceLanguage = engine.SourceLanguage, - TargetLanguage = engine.TargetLanguage, - }; - if (engine.IsModelPersisted is not null) - request.IsModelPersisted = engine.IsModelPersisted.Value; - if (engine.Name is not null) - request.EngineName = engine.Name; - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.Create, - engine.Id, - request, - cancellationToken: ct - ); - }, - cancellationToken - ); - return engine; - } - - public async Task UpdateAsync( - string engineId, - string? sourceLanguage, - string? targetLanguage, - CancellationToken cancellationToken = default - ) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Engine? engine = await Entities.UpdateAsync( - engineId, - u => - { - if (sourceLanguage is not null) - u.Set(e => e.SourceLanguage, sourceLanguage); - if (targetLanguage is not null) - u.Set(e => e.TargetLanguage, targetLanguage); - }, - cancellationToken: ct - ); - if (engine is null) - throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); - await _pretranslations.DeleteAllAsync(pt => pt.EngineRef == engineId, ct); - - UpdateRequest request = new() { EngineType = engine.Type, EngineId = engine.Id }; - if (sourceLanguage is not null) - request.SourceLanguage = sourceLanguage; - if (targetLanguage is not null) - request.TargetLanguage = targetLanguage; - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.Update, - engine.Id, - request, - cancellationToken: ct - ); - }, - cancellationToken - ); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Engine? engine = await Entities.DeleteAsync(engineId, ct); - if (engine is null) - throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); - - await _builds.DeleteAllAsync(b => b.EngineRef == engineId, ct); - await _pretranslations.DeleteAllAsync(pt => pt.EngineRef == engineId, ct); - - DeleteRequest request = new() { EngineType = engine.Type, EngineId = engine.Id }; - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.Delete, - engine.Id, - request, - cancellationToken: ct - ); - }, - cancellationToken - ); - } - - public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) - { - return await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - if ( - await _builds.ExistsAsync( - b => - b.EngineRef == build.EngineRef - && (b.State == JobState.Active || b.State == JobState.Pending), - ct - ) - ) - { - return false; - } - - build.DateCreated = DateTime.UtcNow; - await _builds.InsertAsync(build, ct); - - Engine engine = await GetAsync(build.EngineRef, ct); - StartBuildRequest request = new StartBuildRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - BuildId = build.Id, - Corpora = { _corpusMappingService.Map(build, engine).Select(Map) }, - }; - - if (build.Options is not null) - request.Options = JsonSerializer.Serialize(build.Options); - - // Log the build request summary - try - { - var buildRequestSummary = (JsonObject)JsonNode.Parse(JsonSerializer.Serialize(request))!; - // correct build options parsing - buildRequestSummary.Remove("Options"); - try - { - buildRequestSummary.Add("Options", JsonNode.Parse(request.Options)); - } - catch (JsonException) - { - buildRequestSummary.Add( - "Options", - "Build \"Options\" failed parsing: " + (request.Options ?? "null") - ); - } - buildRequestSummary.Add("Event", "BuildRequest"); - buildRequestSummary.Add("ModelRevision", engine.ModelRevision); - buildRequestSummary.Add("ClientId", engine.Owner); - _logger.LogInformation("{request}", buildRequestSummary.ToJsonString()); - } - catch (JsonException) - { - _logger.LogInformation("Error parsing build request summary."); - _logger.LogInformation("{request}", JsonSerializer.Serialize(request)); - } - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engine.Id, - request, - cancellationToken: ct - ); - return true; - }, - cancellationToken - ); - } - - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - Engine engine = await GetAsync(engineId, cancellationToken); - Build? currentBuild = await _builds.GetAsync( - b => b.EngineRef == engine.Id && (b.State == JobState.Active || b.State == JobState.Pending), - cancellationToken - ); - if (currentBuild is null) - return null; - - CancelBuildRequest request = new CancelBuildRequest { EngineType = engine.Type, EngineId = engine.Id }; - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.CancelBuild, - engine.Id, - request, - cancellationToken: cancellationToken - ); - - return currentBuild; - } - - public async Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - Engine engine = await GetAsync(engineId, cancellationToken); - if (engine.ModelRevision == 0) - return null; - - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - try - { - GetModelDownloadUrlResponse result = await client.GetModelDownloadUrlAsync( - new GetModelDownloadUrlRequest { EngineType = engine.Type, EngineId = engine.Id }, - cancellationToken: cancellationToken - ); - return new ModelDownloadUrl - { - Url = result.Url, - ModelRevision = result.ModelRevision, - ExpiresAt = result.ExpiresAt.ToDateTime(), - }; - } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) - { - return null; - } - } public Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken cancellationToken = default) { @@ -396,16 +20,15 @@ public Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken can ); } - public async Task UpdateCorpusAsync( + public Task UpdateCorpusAsync( string engineId, string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + IReadOnlyList? sourceFiles, + IReadOnlyList? targetFiles, CancellationToken cancellationToken = default ) { - Corpus? corpus = null; - await _dataAccessContext.WithTransactionAsync( + return _dataAccessContext.WithTransactionAsync( async (ct) => { Engine? engine = await Entities.UpdateAsync( @@ -417,7 +40,7 @@ await _dataAccessContext.WithTransactionAsync( if (targetFiles is not null) u.Set(c => c.Corpora.FirstMatchingElement().TargetFiles, targetFiles); }, - cancellationToken: cancellationToken + cancellationToken: ct ); if (engine is null) { @@ -426,33 +49,24 @@ await _dataAccessContext.WithTransactionAsync( ); } - await _pretranslations.DeleteAllAsync( - pt => pt.CorpusRef == corpusId, - cancellationToken: cancellationToken - ); - corpus = engine.Corpora.First(c => c.Id == corpusId); + await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == corpusId, ct); + return engine.Corpora.First(c => c.Id == corpusId); }, - cancellationToken: cancellationToken + cancellationToken ); - if (corpus is null) - { - throw new EntityNotFoundException($"Could not find the Corpus '{corpusId}' in Engine '{engineId}'."); - } - return corpus; } - public async Task DeleteCorpusAsync( + public Task DeleteCorpusAsync( string engineId, string corpusId, bool deleteFiles, CancellationToken cancellationToken = default ) { - Engine? originalEngine = null; - await _dataAccessContext.WithTransactionAsync( + return _dataAccessContext.WithTransactionAsync( async (ct) => { - originalEngine = await Entities.UpdateAsync( + Engine? originalEngine = await Entities.UpdateAsync( e => e.Id == engineId, u => u.RemoveAll(e => e.Corpora, c => c.Id == corpusId), returnOriginal: true, @@ -465,25 +79,26 @@ await _dataAccessContext.WithTransactionAsync( ); } await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == corpusId, ct); + + if (deleteFiles && originalEngine != null) + { + foreach ( + string id in originalEngine.Corpora.SelectMany(c => + c.TargetFiles.Select(f => f.Id).Concat(c.SourceFiles.Select(f => f.Id).Distinct()) + ) + ) + { + await _deleteDataFileHandler.HandleAsync(new DeleteDataFile(id), ct); + } + } }, - cancellationToken: cancellationToken + cancellationToken ); - if (deleteFiles && originalEngine != null) - { - foreach ( - string id in originalEngine.Corpora.SelectMany(c => - c.TargetFiles.Select(f => f.Id).Concat(c.SourceFiles.Select(f => f.Id).Distinct()) - ) - ) - { - await _mediator.Send(new { DataFileId = id }, cancellationToken); - } - } } public Task AddParallelCorpusAsync( string engineId, - Shared.Models.ParallelCorpus corpus, + ParallelCorpus corpus, CancellationToken cancellationToken = default ) { @@ -494,16 +109,15 @@ public Task AddParallelCorpusAsync( ); } - public async Task UpdateParallelCorpusAsync( + public Task UpdateParallelCorpusAsync( string engineId, string parallelCorpusId, - IReadOnlyList? sourceCorpora, - IReadOnlyList? targetCorpora, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { - Shared.Models.ParallelCorpus? parallelCorpus = null; - await _dataAccessContext.WithTransactionAsync( + return _dataAccessContext.WithTransactionAsync( async (ct) => { Engine? engine = await Entities.UpdateAsync( @@ -515,43 +129,37 @@ await _dataAccessContext.WithTransactionAsync( if (targetCorpora is not null) u.Set(c => c.ParallelCorpora.FirstMatchingElement().TargetCorpora, targetCorpora); }, - cancellationToken: cancellationToken + cancellationToken: ct ); if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); + } + + await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == parallelCorpusId, cancellationToken: ct); + ParallelCorpus? parallelCorpus = engine.ParallelCorpora.FirstOrDefault(c => c.Id == parallelCorpusId); + if (parallelCorpus is null) { throw new EntityNotFoundException( $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." ); } - - await _pretranslations.DeleteAllAsync( - pt => pt.CorpusRef == parallelCorpusId, - cancellationToken: cancellationToken - ); - parallelCorpus = engine.ParallelCorpora.First(c => c.Id == parallelCorpusId); + return parallelCorpus; }, - cancellationToken: cancellationToken + cancellationToken ); - if (parallelCorpus is null) - { - throw new EntityNotFoundException( - $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." - ); - } - return parallelCorpus; } - public async Task DeleteParallelCorpusAsync( + public Task DeleteParallelCorpusAsync( string engineId, string parallelCorpusId, CancellationToken cancellationToken = default ) { - Engine? originalEngine = null; - await _dataAccessContext.WithTransactionAsync( + return _dataAccessContext.WithTransactionAsync( async (ct) => { - originalEngine = await Entities.UpdateAsync( + Engine? originalEngine = await Entities.UpdateAsync( e => e.Id == engineId, u => u.RemoveAll(e => e.ParallelCorpora, c => c.Id == parallelCorpusId), returnOriginal: true, @@ -565,7 +173,7 @@ await _dataAccessContext.WithTransactionAsync( } await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == parallelCorpusId, ct); }, - cancellationToken: cancellationToken + cancellationToken ); } @@ -581,7 +189,7 @@ await Entities.GetAllAsync( c.SourceCorpora.Any(cs => cs.Files.Any(f => f.Id == dataFileId)) || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId)) ), - cancellationToken: cancellationToken + ct ) ) .SelectMany(e => e.ParallelCorpora.Select(c => c.Id)) @@ -593,7 +201,7 @@ await Entities.GetAllAsync( e.Corpora.Any(c => c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId) ), - cancellationToken + ct ) ) .SelectMany(e => e.Corpora.Select(c => c.Id)) @@ -621,15 +229,15 @@ await Entities.UpdateAllAsync( f => f.Id == dataFileId ); }, - cancellationToken: cancellationToken + cancellationToken: ct ); await _pretranslations.DeleteAllAsync( pt => parallelCorpusIds.Contains(pt.CorpusRef) || corpusIds.Contains(pt.CorpusRef), - cancellationToken: cancellationToken + ct ); }, - cancellationToken: cancellationToken + cancellationToken ); } @@ -715,212 +323,40 @@ await _pretranslations.DeleteAllAsync( ); } - public async Task UpdateCorpusFilesAsync( + public Task UpdateCorpusFilesAsync( string corpusId, - IReadOnlyList files, + IReadOnlyList files, CancellationToken cancellationToken = default ) { - await Entities.UpdateAllAsync( - e => - e.ParallelCorpora.Any(c => - c.SourceCorpora.Any(sc => sc.Id == corpusId) || c.TargetCorpora.Any(tc => tc.Id == corpusId) - ), - u => + return _dataAccessContext.WithTransactionAsync( + async (ct) => { - u.SetAll( - e => e.ParallelCorpora.AllElements().SourceCorpora, - mc => mc.Files, - files, - mc => mc.Id == corpusId - ); - u.SetAll( - e => e.ParallelCorpora.AllElements().TargetCorpora, - mc => mc.Files, - files, - mc => mc.Id == corpusId + await Entities.UpdateAllAsync( + e => + e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(sc => sc.Id == corpusId) || c.TargetCorpora.Any(tc => tc.Id == corpusId) + ), + u => + { + u.SetAll( + e => e.ParallelCorpora.AllElements().SourceCorpora, + mc => mc.Files, + files, + mc => mc.Id == corpusId + ); + u.SetAll( + e => e.ParallelCorpora.AllElements().TargetCorpora, + mc => mc.Files, + files, + mc => mc.Id == corpusId + ); + }, + cancellationToken: ct ); + await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == corpusId, ct); }, - cancellationToken: cancellationToken - ); - await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == corpusId, cancellationToken: cancellationToken); - } - - public async Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engineType); - GetQueueSizeResponse response = await client.GetQueueSizeAsync( - new GetQueueSizeRequest { EngineType = engineType }, - cancellationToken: cancellationToken - ); - return new Queue { Size = response.Size, EngineType = engineType }; - } - - public async Task GetLanguageInfoAsync( - string engineType, - string language, - CancellationToken cancellationToken = default - ) - { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(engineType); - GetLanguageInfoResponse response = await client.GetLanguageInfoAsync( - new GetLanguageInfoRequest { EngineType = engineType, Language = language }, - cancellationToken: cancellationToken + cancellationToken ); - return new LanguageInfo - { - InternalCode = response.InternalCode, - IsNative = response.IsNative, - EngineType = engineType, - }; - } - - private Models.TranslationResult Map(V1.TranslationResult source) - { - return new Models.TranslationResult - { - Translation = source.Translation, - SourceTokens = source.SourceTokens.ToList(), - TargetTokens = source.TargetTokens.ToList(), - Confidences = source.Confidences.ToList(), - Sources = source.Sources.Select(Map).ToList(), - Alignment = source.Alignment.Select(Map).ToList(), - Phrases = source.Phrases.Select(Map).ToList(), - }; - } - - private IReadOnlySet Map(TranslationSources source) - { - return source.Values.Cast().ToHashSet(); - } - - private Models.AlignedWordPair Map(V1.AlignedWordPair source) - { - return new Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; - } - - private Models.Phrase Map(V1.Phrase source) - { - return new Models.Phrase - { - SourceSegmentStart = source.SourceSegmentStart, - SourceSegmentEnd = source.SourceSegmentEnd, - TargetSegmentCut = source.TargetSegmentCut, - }; - } - - private Models.WordGraph Map(V1.WordGraph source) - { - return new Models.WordGraph - { - SourceTokens = source.SourceTokens.ToList(), - InitialStateScore = source.InitialStateScore, - FinalStates = source.FinalStates.ToHashSet(), - Arcs = source.Arcs.Select(Map).ToList(), - }; - } - - private Models.WordGraphArc Map(V1.WordGraphArc source) - { - return new Models.WordGraphArc - { - PrevState = source.PrevState, - NextState = source.NextState, - Score = source.Score, - TargetTokens = source.TargetTokens.ToList(), - Confidences = source.Confidences.ToList(), - SourceSegmentStart = source.SourceSegmentStart, - SourceSegmentEnd = source.SourceSegmentEnd, - Alignment = source.Alignment.Select(Map).ToList(), - Sources = source.Sources.Select(Map).ToList(), - }; - } - - private static V1.ParallelCorpus Map(SIL.ServiceToolkit.Models.ParallelCorpus source) - { - return new V1.ParallelCorpus - { - Id = source.Id, - SourceCorpora = { source.SourceCorpora.Select(Map) }, - TargetCorpora = { source.TargetCorpora.Select(Map) }, - }; - } - - private static V1.MonolingualCorpus Map(SIL.ServiceToolkit.Models.MonolingualCorpus source) - { - var corpus = new V1.MonolingualCorpus - { - Id = source.Id, - Language = source.Language, - Files = { source.Files.Select(Map) }, - }; - - if (source.TrainOnAll) - { - corpus.TrainOnAll = true; - } - else if (source.TrainOnTextIds is not null) - { - corpus.TrainOnTextIds.Add(source.TrainOnTextIds); - } - else if (source.TrainOnChapters is not null) - { - corpus.TrainOnChapters.Add( - source - .TrainOnChapters?.Select(kvp => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - }) - .ToDictionary() - ); - } - - if (source.PretranslateAll) - { - corpus.PretranslateAll = true; - } - else if (source.InferenceTextIds is not null) - { - corpus.PretranslateTextIds.Add(source.InferenceTextIds); - } - else if (source.InferenceChapters is not null) - { - corpus.PretranslateChapters.Add( - source - .InferenceChapters?.Select(kvp => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - }) - .ToDictionary() - ); - } - - return corpus; - } - - private static V1.CorpusFile Map(SIL.ServiceToolkit.Models.CorpusFile source) - { - return new V1.CorpusFile - { - Location = source.Location, - TextId = source.TextId, - Format = Map(source.Format), - }; - } - - private static V1.FileFormat Map(SIL.ServiceToolkit.Models.FileFormat source) - { - return source switch - { - SIL.ServiceToolkit.Models.FileFormat.Text => V1.FileFormat.Text, - SIL.ServiceToolkit.Models.FileFormat.Paratext => V1.FileFormat.Paratext, - _ => throw new InvalidEnumArgumentException(nameof(source)), - }; } } diff --git a/src/Serval/src/Serval.Translation/Services/EngineServiceFactory.cs b/src/Serval/src/Serval.Translation/Services/EngineServiceFactory.cs new file mode 100644 index 000000000..cfaa5fd2c --- /dev/null +++ b/src/Serval/src/Serval.Translation/Services/EngineServiceFactory.cs @@ -0,0 +1,20 @@ +namespace Serval.Translation.Services; + +public class EngineServiceFactory(IServiceProvider serviceProvider) : IEngineServiceFactory +{ + private readonly IServiceProvider _serviceProvider = serviceProvider; + + public bool TryGetEngineService(string engineType, [NotNullWhen(true)] out ITranslationEngineService? service) + { + ITranslationEngineService? engine = _serviceProvider.GetKeyedService( + engineType.ToLowerInvariant() + ); + if (engine is null) + { + service = null; + return false; + } + service = engine; + return true; + } +} diff --git a/src/Serval/src/Serval.Translation/Services/ICorpusMappingService.cs b/src/Serval/src/Serval.Translation/Services/ICorpusMappingService.cs deleted file mode 100644 index 038b7c420..000000000 --- a/src/Serval/src/Serval.Translation/Services/ICorpusMappingService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Translation.Services; - -public interface ICorpusMappingService -{ - IReadOnlyList Map(Build build, Engine engine); -} diff --git a/src/Serval/src/Serval.Translation/Services/IEngineService.cs b/src/Serval/src/Serval.Translation/Services/IEngineService.cs index e79b14b11..3d185fead 100644 --- a/src/Serval/src/Serval.Translation/Services/IEngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/IEngineService.cs @@ -2,49 +2,8 @@ public interface IEngineService { - Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); Task GetAsync(string engineId, CancellationToken cancellationToken = default); - Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); - - Task UpdateAsync( - string engineId, - string? sourceLanguage, - string? targetLanguage, - CancellationToken cancellationToken = default - ); - - Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - - Task TranslateAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ); - - Task?> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ); - - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - - Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ); - - Task StartBuildAsync(Build build, CancellationToken cancellationToken = default); - - Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - - Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); - Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken cancellationToken = default); Task UpdateCorpusAsync( string engineId, @@ -87,12 +46,4 @@ Task UpdateCorpusFilesAsync( IReadOnlyList files, CancellationToken cancellationToken = default ); - - Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); - - Task GetLanguageInfoAsync( - string engineType, - string language, - CancellationToken cancellationToken = default - ); } diff --git a/src/Serval/src/Serval.Translation/Services/IEngineServiceFactory.cs b/src/Serval/src/Serval.Translation/Services/IEngineServiceFactory.cs new file mode 100644 index 000000000..85bd7e636 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Services/IEngineServiceFactory.cs @@ -0,0 +1,6 @@ +namespace Serval.Translation.Services; + +public interface IEngineServiceFactory +{ + bool TryGetEngineService(string engineType, [NotNullWhen(true)] out ITranslationEngineService? service); +} diff --git a/src/Serval/src/Serval.Translation/Services/IPretranslationService.cs b/src/Serval/src/Serval.Translation/Services/IPretranslationService.cs index ede5c9a54..8c13853af 100644 --- a/src/Serval/src/Serval.Translation/Services/IPretranslationService.cs +++ b/src/Serval/src/Serval.Translation/Services/IPretranslationService.cs @@ -2,7 +2,7 @@ public interface IPretranslationService { - Task> GetAllAsync( + Task> GetAllAsync( string engineId, int modelRevision, string corpusId, diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs new file mode 100644 index 000000000..4d27bb58b --- /dev/null +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -0,0 +1,407 @@ +namespace Serval.Translation.Services; + +public class PlatformService( + IRepository builds, + IRepository engines, + IRepository pretranslations, + IDataAccessContext dataAccessContext, + IEventRouter eventRouter +) : ITranslationPlatformService +{ + private const int PretranslationInsertBatchSize = 128; + + private readonly IRepository _builds = builds; + private readonly IRepository _engines = engines; + private readonly IRepository _pretranslations = pretranslations; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IEventRouter _eventRouter = eventRouter; + + public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => u.Set(b => b.State, JobState.Active), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, true), + cancellationToken: ct + ); + if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + } + + await _eventRouter.PublishAsync(new TranslationBuildStarted(build.Id, engine.Id, engine.Owner), ct); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildCompletedAsync( + string buildId, + int corpusSize, + double confidence, + CancellationToken cancellationToken = default + ) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.State, JobState.Completed) + .Set(b => b.Message, "Completed") + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => + u.Set(e => e.Confidence, confidence) + .Set(e => e.CorpusSize, corpusSize) + .Set(e => e.IsBuilding, false) + .Inc(e => e.ModelRevision), + cancellationToken: ct + ); + if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + } + + // delete pretranslations created by the previous build + await _pretranslations.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision < engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new TranslationBuildFinished( + build.Id, + engine.Id, + engine.Owner, + build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.Message, "Canceled") + .Set(b => b.DateFinished, DateTime.UtcNow) + .Set(b => b.State, JobState.Canceled), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + } + + // delete pretranslations that might have been created during the build + await _pretranslations.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new TranslationBuildFinished( + build.Id, + engine.Id, + engine.Owner, + build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.State, JobState.Faulted) + .Set(b => b.Message, message) + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + } + + // delete pretranslations that might have been created during the build + await _pretranslations.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new TranslationBuildFinished( + build.Id, + engine.Id, + engine.Owner, + build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.Message, "Restarting") + .Set(b => b.Step, 0) + .Set(b => b.Progress, 0) + .Set(b => b.State, JobState.Pending), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.GetAsync(build.EngineRef, ct); + if (engine is null) + { + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + } + + // delete pretranslations that might have been created during the build + await _pretranslations.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildStatusAsync( + string buildId, + BuildProgressStatusContract progressStatus, + int? queueDepth = null, + IReadOnlyCollection? phases = null, + DateTime? started = null, + DateTime? completed = null, + CancellationToken cancellationToken = default + ) + { + await _builds.UpdateAsync( + b => b.Id == buildId && (b.State == JobState.Active || b.State == JobState.Pending), + u => + { + u.Set(b => b.Step, progressStatus.Step); + if (progressStatus.PercentCompleted.HasValue) + { + u.Set( + b => b.Progress, + Math.Round(progressStatus.PercentCompleted.Value, 4, MidpointRounding.AwayFromZero) + ); + } + if (progressStatus.Message is not null) + u.Set(b => b.Message, progressStatus.Message); + if (queueDepth.HasValue) + u.Set(b => b.QueueDepth, queueDepth.Value); + if (phases is not null && phases.Count > 0) + { + u.Set( + b => b.Phases, + [ + .. phases.Select(p => new Phase + { + Stage = p.Stage, + Started = p.Started, + Step = p.Step, + StepCount = p.StepCount, + }), + ] + ); + } + if (started.HasValue) + u.Set(b => b.DateStarted, started.Value); + if (completed.HasValue) + u.Set(b => b.DateCompleted, completed.Value); + }, + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) + { + await _builds.UpdateAsync( + b => b.Id == buildId && (b.State == JobState.Active || b.State == JobState.Pending), + u => u.Set(b => b.Step, step), + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + ExecutionDataContract executionData, + CancellationToken cancellationToken = default + ) + { + await _builds.UpdateAsync( + b => b.Id == buildId, + u => + u.Set( + b => b.ExecutionData, + new ExecutionData + { + TrainCount = executionData.TrainCount, + PretranslateCount = executionData.PretranslateCount, + Warnings = executionData.Warnings?.ToList() ?? [], + EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, + EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, + ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, + ResolvedTargetLanguage = executionData.ResolvedTargetLanguage, + } + ), + cancellationToken: cancellationToken + ); + } + + public async Task UpdateTargetQuoteConventionAsync( + string engineId, + string buildId, + string quoteConvention, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + if (engine is null) + return; + var analysis = engine + .ParallelCorpora.Select(pc => new ParallelCorpusAnalysis + { + ParallelCorpusRef = pc.Id, + TargetQuoteConvention = quoteConvention, + }) + .ToList(); + await _builds.UpdateAsync( + b => b.Id == buildId && b.EngineRef == engineId, + u => + { + u.Set(b => b.TargetQuoteConvention, quoteConvention); + u.Set(b => b.Analysis, analysis); + }, + cancellationToken: cancellationToken + ); + } + + public async Task IncrementEngineCorpusSizeAsync( + string engineId, + int count = 1, + CancellationToken cancellationToken = default + ) + { + await _engines.UpdateAsync( + engineId, + u => u.Inc(e => e.CorpusSize, count), + cancellationToken: cancellationToken + ); + } + + public async Task InsertPretranslationsAsync( + string engineId, + IAsyncEnumerable pretranslations, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); + int nextModelRevision = engine.ModelRevision + 1; + + var batch = new List(); + await foreach (PretranslationContract item in pretranslations.WithCancellation(cancellationToken)) + { + batch.Add( + new Pretranslation + { + EngineRef = engineId, + ModelRevision = nextModelRevision, + CorpusRef = item.CorpusId, + TextId = item.TextId, + SourceRefs = item.SourceRefs.ToList(), + TargetRefs = item.TargetRefs.ToList(), + Refs = item.TargetRefs.ToList(), + Translation = item.Translation, + SourceTokens = item.SourceTokens, + TranslationTokens = item.TranslationTokens, + Alignment = item + .Alignment?.Select(a => new AlignedWordPair + { + SourceIndex = a.SourceIndex, + TargetIndex = a.TargetIndex, + }) + .ToList(), + Confidence = item.Confidence, + } + ); + if (batch.Count == PretranslationInsertBatchSize) + { + await _pretranslations.InsertAllAsync(batch, cancellationToken); + batch.Clear(); + } + } + if (batch.Count > 0) + await _pretranslations.InsertAllAsync(batch, CancellationToken.None); + } +} diff --git a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs index c31ac96f9..dca7b2497 100644 --- a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs +++ b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs @@ -1,4 +1,7 @@ using SIL.Machine.Corpora; +using SIL.Machine.PunctuationAnalysis; +using SIL.Machine.Translation; +using SIL.Scripture; namespace Serval.Translation.Services; @@ -6,18 +9,16 @@ public class PretranslationService( IRepository pretranslations, IRepository engines, IRepository builds, - ICorpusMappingService corpusMappingService, - IParallelCorpusService parallelCorpusService + ContractMapper contractMapper ) : EntityServiceBase(pretranslations), IPretranslationService { private readonly IRepository _engines = engines; private readonly IRepository _builds = builds; - private readonly IParallelCorpusService _parallelCorpusService = parallelCorpusService; - private readonly ICorpusMappingService _corpusMappingService = corpusMappingService; + private readonly ContractMapper _contractMapper = contractMapper; private const string AIDisclaimerRemark = "This draft of {0} was generated using AI on {1}. It should be reviewed and edited carefully."; - public async Task> GetAllAsync( + public async Task> GetAllAsync( string engineId, int modelRevision, string corpusId, @@ -109,21 +110,15 @@ public async Task GetUsfmAsync( List remarks = [disclaimerRemark, markerPlacementRemark]; - SIL.ServiceToolkit.Models.ParallelCorpus[] parallelCorpora = _corpusMappingService.Map(build, engine).ToArray(); + ParallelCorpusContract[] parallelCorpora = _contractMapper.Map(build, engine).ToArray(); - IEnumerable pretranslations = ( - await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) - ).Select(p => new SIL.ServiceToolkit.Models.ParallelRow - { - SourceRefs = p.SourceRefs ?? [], - TargetRefs = p.TargetRefs ?? [], - TargetText = p.Translation, - Alignment = p - .Alignment?.Select(wp => new SIL.Machine.Corpora.AlignedWordPair(wp.SourceIndex, wp.TargetIndex)) - .ToArray(), - SourceTokens = p.SourceTokens, - TargetTokens = p.TranslationTokens, - }); + IReadOnlyList pretranslations = await GetAllAsync( + engineId, + modelRevision, + corpusId, + textId, + cancellationToken + ); string? targetQuoteConvention = null; if (quoteNormalizationBehavior == PretranslationNormalizationBehavior.Denormalized) @@ -142,11 +137,11 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) _ => throw new InvalidEnumArgumentException(nameof(textOrigin)), }; - usfm = _parallelCorpusService.UpdateTargetUsfm( + usfm = UpdateTargetUsfm( parallelCorpora, corpusId, textId, - textOrigin == PretranslationUsfmTextOrigin.OnlyExisting ? [] : pretranslations.ToArray(), + textOrigin == PretranslationUsfmTextOrigin.OnlyExisting ? [] : pretranslations, textBehavior, Map(paragraphMarkerBehavior), Map(embedBehavior), @@ -162,11 +157,11 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) ) { // Copy and update the source book if it exists - usfm = _parallelCorpusService.UpdateSourceUsfm( + usfm = UpdateSourceUsfm( parallelCorpora, corpusId, textId, - textOrigin == PretranslationUsfmTextOrigin.OnlyExisting ? [] : pretranslations.ToArray(), + textOrigin == PretranslationUsfmTextOrigin.OnlyExisting ? [] : pretranslations, Map(paragraphMarkerBehavior), Map(embedBehavior), Map(styleMarkerBehavior), @@ -179,6 +174,290 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) return usfm; } + private static string UpdateSourceUsfm( + IReadOnlyList parallelCorpora, + string corpusId, + string bookId, + IReadOnlyList pretranslations, + UpdateUsfmMarkerBehavior paragraphBehavior, + UpdateUsfmMarkerBehavior embedBehavior, + UpdateUsfmMarkerBehavior styleBehavior, + bool placeParagraphMarkers, + IEnumerable? remarks, + string? targetQuoteConvention + ) + { + return UpdateUsfm( + parallelCorpora, + corpusId, + bookId, + pretranslations, + UpdateUsfmTextBehavior.StripExisting, + paragraphBehavior, + embedBehavior, + styleBehavior, + placeParagraphMarkers ? [new PlaceMarkersUsfmUpdateBlockHandler()] : null, + remarks, + targetQuoteConvention, + isSource: true + ); + } + + private static string UpdateTargetUsfm( + IReadOnlyList parallelCorpora, + string corpusId, + string bookId, + IReadOnlyList pretranslations, + UpdateUsfmTextBehavior textBehavior, + UpdateUsfmMarkerBehavior paragraphBehavior, + UpdateUsfmMarkerBehavior embedBehavior, + UpdateUsfmMarkerBehavior styleBehavior, + IEnumerable? remarks, + string? targetQuoteConvention + ) + { + return UpdateUsfm( + parallelCorpora, + corpusId, + bookId, + pretranslations, + textBehavior, + paragraphBehavior, + embedBehavior, + styleBehavior, + updateBlockHandlers: null, + remarks, + targetQuoteConvention, + isSource: false + ); + } + + private static string UpdateUsfm( + IReadOnlyList parallelCorpora, + string corpusId, + string bookId, + IEnumerable pretranslations, + UpdateUsfmTextBehavior textBehavior, + UpdateUsfmMarkerBehavior paragraphBehavior, + UpdateUsfmMarkerBehavior embedBehavior, + UpdateUsfmMarkerBehavior styleBehavior, + IEnumerable? updateBlockHandlers, + IEnumerable? remarks, + string? targetQuoteConvention, + bool isSource + ) + { + CorpusBundle corpusBundle = new(parallelCorpora); + ParallelCorpusContract corpus = corpusBundle.ParallelCorpora.Single(c => c.Id == corpusId); + CorpusFileContract sourceFile = corpus.SourceCorpora[0].Files[0]; + CorpusFileContract targetFile = corpus.TargetCorpora[0].Files[0]; + ParatextProjectSettings? sourceSettings = corpusBundle.GetSettings(sourceFile.Location); + ParatextProjectSettings? targetSettings = corpusBundle.GetSettings(targetFile.Location); + + using Shared.Services.ZipParatextProjectTextUpdater updater = corpusBundle.GetTextUpdater( + isSource ? sourceFile.Location : targetFile.Location + ); + string usfm = + updater.UpdateUsfm( + bookId, + pretranslations + .Select(p => + Map( + p, + isSource, + sourceSettings?.Versification, + targetSettings?.Versification, + paragraphBehavior, + styleBehavior + ) + ) + .Where(row => row.Refs.Any()) + .OrderBy(row => row.Refs[0]) + .ToArray(), + isSource ? sourceSettings?.FullName : targetSettings?.FullName, + textBehavior, + paragraphBehavior, + embedBehavior, + styleBehavior, + updateBlockHandlers: updateBlockHandlers, + remarks: remarks, + errorHandler: (_) => true, + compareSegments: isSource + ) ?? ""; + + if (!string.IsNullOrEmpty(targetQuoteConvention)) + usfm = DenormalizeQuotationMarks(usfm, targetQuoteConvention); + return usfm; + } + + private static UpdateUsfmRow Map( + Pretranslation pretranslation, + bool isSource, + ScrVers? sourceVersification, + ScrVers? targetVersification, + UpdateUsfmMarkerBehavior paragraphBehavior, + UpdateUsfmMarkerBehavior styleBehavior + ) + { + Dictionary? metadata = null; + if (pretranslation.Alignment is not null) + { + metadata = new Dictionary + { + { + PlaceMarkersAlignmentInfo.MetadataKey, + new PlaceMarkersAlignmentInfo( + pretranslation.SourceTokens, + pretranslation.TranslationTokens, + CreateWordAlignmentMatrix(pretranslation), + paragraphBehavior, + styleBehavior + ) + }, + }; + } + + ScriptureRef[] refs; + if (isSource) + { + refs = + [ + .. ( + pretranslation.SourceRefs?.Any() ?? false + ? Map(pretranslation.SourceRefs, sourceVersification) + : Map(pretranslation.TargetRefs ?? [], targetVersification) + ), + ]; + } + else + { + // the pretranslations are generated from the source book and inserted into the target book + // use relaxed references since the USFM structure may not be the same + refs = [.. Map(pretranslation.TargetRefs ?? [], targetVersification).Select(r => r.ToRelaxed())]; + } + + return new UpdateUsfmRow(refs, pretranslation.Translation, metadata); + } + + private static IEnumerable Map(IEnumerable refs, ScrVers? versification) + { + return refs.Select(r => + { + ScriptureRef.TryParse(r, versification, out ScriptureRef sr); + return sr; + }) + .Where(r => !r.IsEmpty); + } + + private static WordAlignmentMatrix? CreateWordAlignmentMatrix(Pretranslation pretranslation) + { + if ( + pretranslation.Alignment is null + || pretranslation.SourceTokens is null + || pretranslation.TranslationTokens is null + ) + { + return null; + } + + var matrix = new WordAlignmentMatrix(pretranslation.SourceTokens.Count, pretranslation.TranslationTokens.Count); + foreach (Shared.Models.AlignedWordPair wordPair in pretranslation.Alignment) + matrix[wordPair.SourceIndex, wordPair.TargetIndex] = true; + + return matrix; + } + + private static string DenormalizeQuotationMarks(string usfm, string quoteConvention) + { + QuoteConvention targetQuoteConvention = QuoteConventions.Standard.GetQuoteConventionByName(quoteConvention); + if (targetQuoteConvention is null) + return usfm; + + QuotationMarkDenormalizationFirstPass quotationMarkDenormalizationFirstPass = new(targetQuoteConvention); + + UsfmParser.Parse(usfm, quotationMarkDenormalizationFirstPass); + List<(int ChapterNumber, QuotationMarkUpdateStrategy Strategy)> bestChapterStrategies = + quotationMarkDenormalizationFirstPass.FindBestChapterStrategies(); + + QuotationMarkDenormalizationUsfmUpdateBlockHandler quotationMarkDenormalizer = new( + targetQuoteConvention, + new QuotationMarkUpdateSettings( + chapterStrategies: bestChapterStrategies.Select(tuple => tuple.Strategy).ToList() + ) + ); + int denormalizableChapterCount = bestChapterStrategies.Count(tup => + tup.Strategy != QuotationMarkUpdateStrategy.Skip + ); + List remarks = []; + string quotationDenormalizationRemark; + if (denormalizableChapterCount == bestChapterStrategies.Count) + { + quotationDenormalizationRemark = + "The quote style in all chapters has been automatically adjusted to match the rest of the project."; + } + else if (denormalizableChapterCount > 0) + { + quotationDenormalizationRemark = + "The quote style in the following chapters has been automatically adjusted to match the rest of the project: " + + GetChapterRangesString( + bestChapterStrategies + .Where(tuple => tuple.Strategy != QuotationMarkUpdateStrategy.Skip) + .Select(tuple => tuple.ChapterNumber) + .ToList() + ) + + "."; + } + else + { + quotationDenormalizationRemark = + "The quote style was not automatically adjusted to match the rest of your project in any chapters."; + } + remarks.Add(quotationDenormalizationRemark); + + var updater = new UpdateUsfmParserHandler(updateBlockHandlers: [quotationMarkDenormalizer], remarks: remarks); + UsfmParser.Parse(usfm, updater); + + usfm = updater.GetUsfm(); + return usfm; + } + + internal static string GetChapterRangesString(List chapterNumbers) + { + chapterNumbers = chapterNumbers.Order().ToList(); + int start = chapterNumbers[0]; + int end = chapterNumbers[0]; + List chapterRangeStrings = []; + foreach (int chapterNumber in chapterNumbers[1..]) + { + if (chapterNumber == end + 1) + { + end = chapterNumber; + } + else + { + if (start == end) + { + chapterRangeStrings.Add(start.ToString(CultureInfo.InvariantCulture)); + } + else + { + chapterRangeStrings.Add($"{start}-{end}"); + } + start = chapterNumber; + end = chapterNumber; + } + } + if (start == end) + { + chapterRangeStrings.Add(start.ToString(CultureInfo.InvariantCulture)); + } + else + { + chapterRangeStrings.Add($"{start}-{end}"); + } + return string.Join(", ", chapterRangeStrings); + } + /// /// Generate a natural sounding remark/comment describing marker placement. /// diff --git a/src/Serval/src/Serval.Translation/Services/ServicesExtensions.cs b/src/Serval/src/Serval.Translation/Services/ServicesExtensions.cs new file mode 100644 index 000000000..a4aea645e --- /dev/null +++ b/src/Serval/src/Serval.Translation/Services/ServicesExtensions.cs @@ -0,0 +1,16 @@ +namespace Serval.Translation.Services; + +public static class ServicesExtensions +{ + public static ITranslationEngineService GetEngineService(this IEngineServiceFactory factory, string engineType) + { + if (factory.TryGetEngineService(engineType, out ITranslationEngineService? service)) + return service; + throw new InvalidOperationException($"No engine registered for type '{engineType}'."); + } + + public static bool EngineTypeExists(this IEngineServiceFactory factory, string engineType) + { + return factory.TryGetEngineService(engineType, out _); + } +} diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs deleted file mode 100644 index cef0c8660..000000000 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ /dev/null @@ -1,407 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace Serval.Translation.Services; - -public class TranslationPlatformServiceV1( - IRepository builds, - IRepository engines, - IRepository pretranslations, - IDataAccessContext dataAccessContext, - IPublishEndpoint publishEndpoint -) : TranslationPlatformApi.TranslationPlatformApiBase -{ - private const int PretranslationInsertBatchSize = 128; - private static readonly Empty Empty = new(); - - private readonly IRepository _builds = builds; - private readonly IRepository _engines = engines; - private readonly IRepository _pretranslations = pretranslations; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IPublishEndpoint _publishEndpoint = publishEndpoint; - - public override async Task BuildStarted(BuildStartedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => u.Set(b => b.State, JobState.Active), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, true), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - await _publishEndpoint.Publish( - new TranslationBuildStarted - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - return Empty; - } - - public override async Task BuildCompleted(BuildCompletedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.State, JobState.Completed) - .Set(b => b.Message, "Completed") - .Set(b => b.DateFinished, DateTime.UtcNow), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => - u.Set(e => e.Confidence, request.Confidence) - .Set(e => e.CorpusSize, request.CorpusSize) - .Set(e => e.IsBuilding, false) - .Inc(e => e.ModelRevision), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete pretranslations created by the previous build - await _pretranslations.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision < engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new TranslationBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildCanceled(BuildCanceledRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.Message, "Canceled") - .Set(b => b.DateFinished, DateTime.UtcNow) - .Set(b => b.State, JobState.Canceled), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, false), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete pretranslations that might have been created during the build - await _pretranslations.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new TranslationBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildFaulted(BuildFaultedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.State, JobState.Faulted) - .Set(b => b.Message, request.Message) - .Set(b => b.DateFinished, DateTime.UtcNow), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, false), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete pretranslations that might have been created during the build - await _pretranslations.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new TranslationBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildRestarting(BuildRestartingRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.Message, "Restarting") - .Set(b => b.Step, 0) - .Set(b => b.Progress, 0) - .Set(b => b.State, JobState.Pending), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.GetAsync(build.EngineRef, ct); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete pretranslations that might have been created during the build - await _pretranslations.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task UpdateBuildStatus(UpdateBuildStatusRequest request, ServerCallContext context) - { - await _builds.UpdateAsync( - b => b.Id == request.BuildId && (b.State == JobState.Active || b.State == JobState.Pending), - u => - { - u.Set(b => b.Step, request.Step); - if (request.HasProgress) - u.Set(b => b.Progress, Math.Round(request.Progress, 4, MidpointRounding.AwayFromZero)); - if (request.HasMessage) - u.Set(b => b.Message, request.Message); - if (request.HasQueueDepth) - u.Set(b => b.QueueDepth, request.QueueDepth); - if (request.Phases.Count > 0) - { - u.Set( - b => b.Phases, - request - .Phases.Select(p => new BuildPhase - { - Stage = (BuildPhaseStage)p.Stage, - Step = p.HasStep ? p.Step : null, - StepCount = p.HasStepCount ? p.StepCount : null, - Started = p.Started?.ToDateTime(), - }) - .ToList() - ); - } - - if (request.Started is not null) - u.Set(b => b.DateStarted, request.Started.ToDateTime()); - if (request.Completed is not null) - u.Set(b => b.DateCompleted, request.Completed.ToDateTime()); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task UpdateBuildExecutionData( - UpdateBuildExecutionDataRequest request, - ServerCallContext context - ) - { - await _builds.UpdateAsync( - b => b.Id == request.BuildId, - u => - u.Set( - b => b.ExecutionData, - new Models.ExecutionData - { - TrainCount = request.ExecutionData.TrainCount, - PretranslateCount = request.ExecutionData.PretranslateCount, - Warnings = [.. request.ExecutionData.Warnings], - EngineSourceLanguageTag = request.ExecutionData.EngineSourceLanguageTag, - EngineTargetLanguageTag = request.ExecutionData.EngineTargetLanguageTag, - ResolvedSourceLanguage = request.ExecutionData.ResolvedSourceLanguage, - ResolvedTargetLanguage = request.ExecutionData.ResolvedTargetLanguage, - } - ), - cancellationToken: context.CancellationToken - ); - - return new Empty(); - } - - public override async Task UpdateTargetQuoteConvention( - UpdateTargetQuoteConventionRequest request, - ServerCallContext context - ) - { - Engine? engine = await _engines.GetAsync(request.EngineId, context.CancellationToken); - if (engine == null) - return Empty; - var analysis = engine - .ParallelCorpora.Select(pc => new ParallelCorpusAnalysis - { - ParallelCorpusRef = pc.Id, - TargetQuoteConvention = request.TargetQuoteConvention, - }) - .ToList(); - await _builds.UpdateAsync( - b => b.Id == request.BuildId && b.EngineRef == request.EngineId, - u => - { - u.Set(b => b.TargetQuoteConvention, request.TargetQuoteConvention); - u.Set(b => b.Analysis, analysis); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task IncrementEngineCorpusSize( - IncrementEngineCorpusSizeRequest request, - ServerCallContext context - ) - { - await _engines.UpdateAsync( - request.EngineId, - u => u.Inc(e => e.CorpusSize, request.Count), - cancellationToken: context.CancellationToken - ); - return Empty; - } - - public override async Task InsertPretranslations( - IAsyncStreamReader requestStream, - ServerCallContext context - ) - { - string engineId = ""; - int nextModelRevision = 0; - - var batch = new List(); - await foreach (InsertPretranslationsRequest request in requestStream.ReadAllAsync(context.CancellationToken)) - { - if (request.EngineId != engineId) - { - Engine? engine = await _engines.GetAsync(request.EngineId, context.CancellationToken); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - nextModelRevision = engine.ModelRevision + 1; - engineId = request.EngineId; - } - batch.Add( - new Pretranslation - { - EngineRef = request.EngineId, - ModelRevision = nextModelRevision, - CorpusRef = request.CorpusId, - TextId = request.TextId, - SourceRefs = request.SourceRefs.ToList(), - TargetRefs = request.TargetRefs.ToList(), - Refs = request.TargetRefs.ToList(), - Translation = request.Translation, - SourceTokens = request.SourceTokens, - TranslationTokens = request.TranslationTokens, - Alignment = request.Alignment.Select(Map).ToList(), - Confidence = request.Confidence, - } - ); - if (batch.Count == PretranslationInsertBatchSize) - { - await _pretranslations.InsertAllAsync(batch, context.CancellationToken); - batch.Clear(); - } - } - if (batch.Count > 0) - await _pretranslations.InsertAllAsync(batch, CancellationToken.None); - - return Empty; - } - - private Models.AlignedWordPair Map(V1.AlignedWordPair alignedWordPair) - { - return new Models.AlignedWordPair() - { - SourceIndex = alignedWordPair.SourceIndex, - TargetIndex = alignedWordPair.TargetIndex, - }; - } -} diff --git a/src/Serval/src/Serval.Translation/Usings.cs b/src/Serval/src/Serval.Translation/Usings.cs index dba08892f..24b8995d6 100644 --- a/src/Serval/src/Serval.Translation/Usings.cs +++ b/src/Serval/src/Serval.Translation/Usings.cs @@ -2,32 +2,34 @@ global using System.Diagnostics.CodeAnalysis; global using System.Globalization; global using System.Linq.Expressions; +global using System.Reflection; +global using System.Runtime.CompilerServices; global using System.Text.Json; global using System.Text.Json.Nodes; global using Asp.Versioning; global using CaseExtensions; -global using Grpc.Core; -global using Grpc.Net.ClientFactory; -global using MassTransit; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.Routing; global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; +global using MongoDB.Bson; +global using MongoDB.Driver; global using NSwag.Annotations; +global using Serval.DataFiles.Contracts; global using Serval.Shared.Configuration; global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; +global using Serval.Shared.Dtos; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; global using Serval.Translation.Configuration; -global using Serval.Translation.Consumers; global using Serval.Translation.Contracts; +global using Serval.Translation.Dtos; global using Serval.Translation.Models; global using Serval.Translation.Services; global using SIL.DataAccess; -global using SIL.ServiceToolkit.Services; -global using SIL.ServiceToolkit.Utils; diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs deleted file mode 100644 index 9e7dc5ef3..000000000 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMediatorRegistrationConfiguratorExtensions -{ - public static IMediatorRegistrationConfigurator AddWebhooksConsumers( - this IMediatorRegistrationConfigurator configurator - ) - { - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMemoryDataAccessConfiguratorExtensions.cs deleted file mode 100644 index 5426d58fa..000000000 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMemoryDataAccessConfiguratorExtensions.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMemoryDataAccessConfiguratorExtensions -{ - public static IMemoryDataAccessConfigurator AddWebhooksRepositories(this IMemoryDataAccessConfigurator configurator) - { - configurator.AddRepository(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs deleted file mode 100644 index 2d3084fba..000000000 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -using MongoDB.Driver; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMongoDataAccessConfiguratorExtensions -{ - public static IMongoDataAccessConfigurator AddWebhooksRepositories(this IMongoDataAccessConfigurator configurator) - { - configurator.AddRepository( - "webhooks.hooks", - init: - [ - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Owner)) - ), - c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) - ), - ] - ); - return configurator; - } -} diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs deleted file mode 100644 index 383e5bafb..000000000 --- a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServalBuilderExtensions -{ - public static IServalBuilder AddWebhooks(this IServalBuilder builder) - { - builder.Services.AddHttpClient(); - builder.Services.AddScoped(); - return builder; - } -} diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..e26ec8e10 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Configuration/IServalConfiguratorExtensions.cs @@ -0,0 +1,38 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddWebhooks(this IServalConfigurator configurator) + { + configurator.Services.AddHttpClient(); + configurator.Services.AddScoped(); + + configurator.AddWebhooksDataAccess(); + + configurator.JobQueues.Add("webhook"); + + configurator.AddHandlers(Assembly.GetExecutingAssembly()); + + return configurator; + } + + public static IServalConfigurator AddWebhooksDataAccess(this IServalConfigurator configurator) + { + configurator.DataAccess.AddRepository( + "webhooks.hooks", + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Owner)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) + ), + ] + ); + + return configurator; + } +} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs deleted file mode 100644 index 769388347..000000000 --- a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace Serval.Webhooks.Consumers; - -public class TranslationBuildFinishedConsumer(IWebhookService webhookService, IUrlService urlService) - : IConsumer -{ - private readonly IWebhookService _webhookService = webhookService; - private readonly IUrlService _urlService = urlService; - - public async Task Consume(ConsumeContext context) - { - await _webhookService.SendEventAsync( - WebhookEvent.TranslationBuildFinished, - context.Message.Owner, - new TranslationBuildFinishedDto - { - Build = new ResourceLinkDto - { - Id = context.Message.BuildId, - Url = _urlService.GetUrl( - Endpoints.GetTranslationBuild, - new { id = context.Message.EngineId, buildId = context.Message.BuildId } - ), - }, - Engine = new ResourceLinkDto - { - Id = context.Message.EngineId, - Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = context.Message.EngineId })!, - }, - BuildState = context.Message.BuildState, - Message = context.Message.Message, - DateFinished = context.Message.DateFinished, - }, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs deleted file mode 100644 index bfdcc5fc1..000000000 --- a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace Serval.Webhooks.Consumers; - -public class TranslationBuildStartedConsumer(IWebhookService webhookService, IUrlService urlService) - : IConsumer -{ - private readonly IWebhookService _webhookService = webhookService; - private readonly IUrlService _urlService = urlService; - - public async Task Consume(ConsumeContext context) - { - await _webhookService.SendEventAsync( - WebhookEvent.TranslationBuildStarted, - context.Message.Owner, - new TranslationBuildStartedDto - { - Build = new ResourceLinkDto - { - Id = context.Message.BuildId, - Url = _urlService.GetUrl( - Endpoints.GetTranslationBuild, - new { id = context.Message.EngineId, buildId = context.Message.BuildId } - ), - }, - Engine = new ResourceLinkDto - { - Id = context.Message.EngineId, - Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = context.Message.EngineId }), - }, - }, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs deleted file mode 100644 index 6647dd5ad..000000000 --- a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace Serval.Webhooks.Consumers; - -public class WordAlignmentBuildFinishedConsumer(IWebhookService webhookService, IUrlService urlService) - : IConsumer -{ - private readonly IWebhookService _webhookService = webhookService; - private readonly IUrlService _urlService = urlService; - - public async Task Consume(ConsumeContext context) - { - await _webhookService.SendEventAsync( - WebhookEvent.WordAlignmentBuildFinished, - context.Message.Owner, - new WordAlignmentBuildFinishedDto - { - Build = new ResourceLinkDto - { - Id = context.Message.BuildId, - Url = _urlService.GetUrl( - Endpoints.GetWordAlignmentBuild, - new { id = context.Message.EngineId, buildId = context.Message.BuildId } - ), - }, - Engine = new ResourceLinkDto - { - Id = context.Message.EngineId, - Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = context.Message.EngineId })!, - }, - BuildState = context.Message.BuildState, - Message = context.Message.Message, - DateFinished = context.Message.DateFinished, - }, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs deleted file mode 100644 index 6d5c939cf..000000000 --- a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace Serval.Webhooks.Consumers; - -public class WordAlignmentBuildStartedConsumer(IWebhookService webhookService, IUrlService urlService) - : IConsumer -{ - private readonly IWebhookService _webhookService = webhookService; - private readonly IUrlService _urlService = urlService; - - public async Task Consume(ConsumeContext context) - { - await _webhookService.SendEventAsync( - WebhookEvent.WordAlignmentBuildStarted, - context.Message.Owner, - new WordAlignmentBuildStartedDto - { - Build = new ResourceLinkDto - { - Id = context.Message.BuildId, - Url = _urlService.GetUrl( - Endpoints.GetWordAlignmentBuild, - new { id = context.Message.EngineId, buildId = context.Message.BuildId } - ), - }, - Engine = new ResourceLinkDto - { - Id = context.Message.EngineId, - Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = context.Message.EngineId }), - }, - }, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs b/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs index e5497c8dc..a0940f86d 100644 --- a/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs +++ b/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs @@ -1,4 +1,6 @@ -namespace Serval.Webhooks.Controllers; +using Serval.Webhooks.Dtos; + +namespace Serval.Webhooks.Controllers; [ApiVersion(1.0)] [Route("api/v{version:apiVersion}/hooks")] diff --git a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs b/src/Serval/src/Serval.Webhooks/Dtos/TranslationBuildFinishedDto.cs similarity index 100% rename from src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs rename to src/Serval/src/Serval.Webhooks/Dtos/TranslationBuildFinishedDto.cs diff --git a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildStartedDto.cs b/src/Serval/src/Serval.Webhooks/Dtos/TranslationBuildStartedDto.cs similarity index 100% rename from src/Serval/src/Serval.Shared/Contracts/TranslationBuildStartedDto.cs rename to src/Serval/src/Serval.Webhooks/Dtos/TranslationBuildStartedDto.cs diff --git a/src/Serval/src/Serval.Webhooks/Contracts/WebhookConfigDto.cs b/src/Serval/src/Serval.Webhooks/Dtos/WebhookConfigDto.cs similarity index 90% rename from src/Serval/src/Serval.Webhooks/Contracts/WebhookConfigDto.cs rename to src/Serval/src/Serval.Webhooks/Dtos/WebhookConfigDto.cs index e3cd35e55..87e6e5417 100644 --- a/src/Serval/src/Serval.Webhooks/Contracts/WebhookConfigDto.cs +++ b/src/Serval/src/Serval.Webhooks/Dtos/WebhookConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Webhooks.Contracts; +namespace Serval.Webhooks.Dtos; public record WebhookConfigDto { diff --git a/src/Serval/src/Serval.Webhooks/Contracts/WebhookDto.cs b/src/Serval/src/Serval.Webhooks/Dtos/WebhookDto.cs similarity index 85% rename from src/Serval/src/Serval.Webhooks/Contracts/WebhookDto.cs rename to src/Serval/src/Serval.Webhooks/Dtos/WebhookDto.cs index 86adfb89d..e02986de9 100644 --- a/src/Serval/src/Serval.Webhooks/Contracts/WebhookDto.cs +++ b/src/Serval/src/Serval.Webhooks/Dtos/WebhookDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Webhooks.Contracts; +namespace Serval.Webhooks.Dtos; public record WebhookDto { diff --git a/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildFinishedHandler.cs b/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildFinishedHandler.cs new file mode 100644 index 000000000..b27debbe2 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildFinishedHandler.cs @@ -0,0 +1,33 @@ +namespace Serval.Webhooks.Handlers; + +public class TranslationBuildFinishedHandler(IWebhookService webhookService, IUrlService urlService) + : IEventHandler +{ + public Task HandleAsync(TranslationBuildFinished message, CancellationToken cancellationToken) + { + return webhookService.SendEventAsync( + WebhookEvent.TranslationBuildFinished, + message.Owner, + new TranslationBuildFinishedDto + { + Build = new ResourceLinkDto + { + Id = message.BuildId, + Url = urlService.GetUrl( + Endpoints.GetTranslationBuild, + new { id = message.EngineId, buildId = message.BuildId } + ), + }, + Engine = new ResourceLinkDto + { + Id = message.EngineId, + Url = urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = message.EngineId })!, + }, + BuildState = message.BuildState, + Message = message.Message, + DateFinished = message.DateFinished, + }, + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildStartedHandler.cs b/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildStartedHandler.cs new file mode 100644 index 000000000..0a624225a --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Handlers/TranslationBuildStartedHandler.cs @@ -0,0 +1,30 @@ +namespace Serval.Webhooks.Handlers; + +public class TranslationBuildStartedHandler(IWebhookService webhookService, IUrlService urlService) + : IEventHandler +{ + public Task HandleAsync(TranslationBuildStarted message, CancellationToken cancellationToken) + { + return webhookService.SendEventAsync( + WebhookEvent.TranslationBuildStarted, + message.Owner, + new TranslationBuildStartedDto + { + Build = new ResourceLinkDto + { + Id = message.BuildId, + Url = urlService.GetUrl( + Endpoints.GetTranslationBuild, + new { id = message.EngineId, buildId = message.BuildId } + ), + }, + Engine = new ResourceLinkDto + { + Id = message.EngineId, + Url = urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = message.EngineId }), + }, + }, + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildFinishedHandler.cs b/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildFinishedHandler.cs new file mode 100644 index 000000000..e73b2d59d --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildFinishedHandler.cs @@ -0,0 +1,33 @@ +namespace Serval.Webhooks.Handlers; + +public class WordAlignmentBuildFinishedHandler(IWebhookService webhookService, IUrlService urlService) + : IEventHandler +{ + public Task HandleAsync(WordAlignmentBuildFinished message, CancellationToken cancellationToken) + { + return webhookService.SendEventAsync( + WebhookEvent.WordAlignmentBuildFinished, + message.Owner, + new WordAlignmentBuildFinishedDto + { + Build = new ResourceLinkDto + { + Id = message.BuildId, + Url = urlService.GetUrl( + Endpoints.GetWordAlignmentBuild, + new { id = message.EngineId, buildId = message.BuildId } + ), + }, + Engine = new ResourceLinkDto + { + Id = message.EngineId, + Url = urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = message.EngineId })!, + }, + BuildState = message.BuildState, + Message = message.Message, + DateFinished = message.DateFinished, + }, + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildStartedHandler.cs b/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildStartedHandler.cs new file mode 100644 index 000000000..46fc2202c --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Handlers/WordAlignmentBuildStartedHandler.cs @@ -0,0 +1,30 @@ +namespace Serval.Webhooks.Handlers; + +public class WordAlignmentBuildStartedHandler(IWebhookService webhookService, IUrlService urlService) + : IEventHandler +{ + public Task HandleAsync(WordAlignmentBuildStarted message, CancellationToken cancellationToken) + { + return webhookService.SendEventAsync( + WebhookEvent.WordAlignmentBuildStarted, + message.Owner, + new WordAlignmentBuildStartedDto + { + Build = new ResourceLinkDto + { + Id = message.BuildId, + Url = urlService.GetUrl( + Endpoints.GetWordAlignmentBuild, + new { id = message.EngineId, buildId = message.BuildId } + ), + }, + Engine = new ResourceLinkDto + { + Id = message.EngineId, + Url = urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = message.EngineId }), + }, + }, + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs b/src/Serval/src/Serval.Webhooks/Models/WebhookEvent.cs similarity index 79% rename from src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs rename to src/Serval/src/Serval.Webhooks/Models/WebhookEvent.cs index 1d6d1e555..e03c4d5dc 100644 --- a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs +++ b/src/Serval/src/Serval.Webhooks/Models/WebhookEvent.cs @@ -1,4 +1,4 @@ -namespace Serval.Webhooks.Contracts; +namespace Serval.Webhooks.Models; public enum WebhookEvent { diff --git a/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj b/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj index 8f0042059..bea66be16 100644 --- a/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj +++ b/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj @@ -10,14 +10,17 @@ $(NoWarn);CS1591;CS1573 + + - + + diff --git a/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs b/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs index f5450c993..d35598b40 100644 --- a/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs +++ b/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs @@ -14,6 +14,11 @@ public async Task SendEventAsync( ) { if (await Entities.ExistsAsync(h => h.Owner == owner && h.Events.Contains(webhookEvent), cancellationToken)) - _jobClient.Enqueue(j => j.RunAsync(webhookEvent, owner, payload, CancellationToken.None)); + { + _jobClient.Enqueue( + "webhook", + j => j.RunAsync(webhookEvent, owner, payload, CancellationToken.None) + ); + } } } diff --git a/src/Serval/src/Serval.Webhooks/Usings.cs b/src/Serval/src/Serval.Webhooks/Usings.cs index 39f9b6a52..445937c1f 100644 --- a/src/Serval/src/Serval.Webhooks/Usings.cs +++ b/src/Serval/src/Serval.Webhooks/Usings.cs @@ -1,22 +1,24 @@ global using System.Diagnostics.CodeAnalysis; +global using System.Reflection; global using System.Security.Cryptography; global using System.Text; global using System.Text.Json; global using System.Text.Json.Nodes; global using Asp.Versioning; global using Hangfire; -global using MassTransit; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.Routing; global using Microsoft.Extensions.Options; +global using MongoDB.Driver; global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; +global using Serval.Shared.Dtos; global using Serval.Shared.Models; global using Serval.Shared.Services; -global using Serval.Webhooks.Consumers; -global using Serval.Webhooks.Contracts; +global using Serval.Translation.Contracts; global using Serval.Webhooks.Models; global using Serval.Webhooks.Services; +global using Serval.WordAlignment.Contracts; global using SIL.DataAccess; diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs new file mode 100644 index 000000000..78e756726 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -0,0 +1,10 @@ +namespace Serval.WordAlignment.Contracts; + +public record ExecutionDataContract +{ + public int? TrainCount { get; init; } + public int? WordAlignCount { get; init; } + public IReadOnlyList? Warnings { get; init; } + public string? EngineSourceLanguageTag { get; init; } + public string? EngineTargetLanguageTag { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/IServalConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment.Contracts/IServalConfiguratorExtensions.cs new file mode 100644 index 000000000..7dea9ed3b --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/IServalConfiguratorExtensions.cs @@ -0,0 +1,16 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalConfiguratorExtensions +{ + public static IServalConfigurator AddWordAlignmentEngine( + this IServalConfigurator configurator, + string engineType + ) + where TEngineService : class, IWordAlignmentEngineService + { + configurator.Services.AddKeyedScoped( + engineType.ToLowerInvariant() + ); + return configurator; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs b/src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentEngineService.cs similarity index 69% rename from src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs rename to src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentEngineService.cs index d5fb2fbb5..537cfc79b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentEngineService.cs @@ -1,21 +1,17 @@ -using Serval.WordAlignment.V1; - -namespace Serval.Machine.Shared.Services; +namespace Serval.WordAlignment.Contracts; public interface IWordAlignmentEngineService { - EngineType Type { get; } - Task CreateAsync( string engineId, - string? engineName, string sourceLanguage, string targetLanguage, + string? engineName = null, CancellationToken cancellationToken = default ); Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - Task AlignAsync( + Task AlignAsync( string engineId, string sourceSegment, string targetSegment, @@ -25,12 +21,12 @@ Task AlignAsync( Task StartBuildAsync( string engineId, string buildId, - string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, + string? options = null, CancellationToken cancellationToken = default ); Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - int GetQueueSize(); + Task GetQueueSizeAsync(CancellationToken cancellationToken = default); } diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentPlatformService.cs b/src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentPlatformService.cs new file mode 100644 index 000000000..3a6b7725f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/IWordAlignmentPlatformService.cs @@ -0,0 +1,37 @@ +namespace Serval.WordAlignment.Contracts; + +public interface IWordAlignmentPlatformService +{ + Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildCompletedAsync( + string buildId, + int corpusSize, + double confidence, + CancellationToken cancellationToken = default + ); + Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); + Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); + Task UpdateBuildStatusAsync( + string buildId, + BuildProgressStatusContract progressStatus, + int? queueDepth = null, + IReadOnlyCollection? phases = null, + DateTime? started = null, + DateTime? completed = null, + CancellationToken cancellationToken = default + ); + Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default); + Task IncrementEngineCorpusSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); + Task InsertWordAlignmentsAsync( + string engineId, + IAsyncEnumerable wordAlignments, + CancellationToken cancellationToken = default + ); + Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + ExecutionDataContract executionData, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/Serval.WordAlignment.Contracts.csproj b/src/Serval/src/Serval.WordAlignment.Contracts/Serval.WordAlignment.Contracts.csproj new file mode 100644 index 000000000..b1422d964 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/Serval.WordAlignment.Contracts.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/Usings.cs b/src/Serval/src/Serval.WordAlignment.Contracts/Usings.cs new file mode 100644 index 000000000..732345fe6 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/Usings.cs @@ -0,0 +1,2 @@ +global using Serval.Shared.Contracts; +global using Serval.WordAlignment.Contracts; diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildFinished.cs b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildFinished.cs new file mode 100644 index 000000000..24f434916 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildFinished.cs @@ -0,0 +1,10 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentBuildFinished( + string BuildId, + string EngineId, + string Owner, + JobState BuildState, + string Message, + DateTime DateFinished +) : IEvent; diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildStarted.cs b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildStarted.cs new file mode 100644 index 000000000..1c5cbcacf --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentBuildStarted.cs @@ -0,0 +1,3 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentBuildStarted(string BuildId, string EngineId, string Owner) : IEvent; diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentContract.cs new file mode 100644 index 000000000..847b89764 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentContract.cs @@ -0,0 +1,12 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentContract +{ + public required string CorpusId { get; init; } + public required string TextId { get; init; } + public required IReadOnlyList SourceRefs { get; init; } + public required IReadOnlyList TargetRefs { get; init; } + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Alignment { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentResultContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentResultContract.cs new file mode 100644 index 000000000..f4e256dd3 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/WordAlignmentResultContract.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentResultContract +{ + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Alignment { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs deleted file mode 100644 index 84d4d6a58..000000000 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Microsoft.AspNetCore.Builder; - -public static class IEndpointRouteBuilderExtensions -{ - public static IEndpointRouteBuilder MapServalWordAlignmentServices(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - - return builder; - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs deleted file mode 100644 index 72db9d994..000000000 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMediatorRegistrationConfiguratorExtensions -{ - public static IMediatorRegistrationConfigurator AddWordAlignmentConsumers( - this IMediatorRegistrationConfigurator configurator - ) - { - configurator.AddConsumer(); - configurator.AddConsumer(); - configurator.AddConsumer(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs deleted file mode 100644 index 3edb081fa..000000000 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMemoryDataAccessConfiguratorExtensions -{ - public static IMemoryDataAccessConfigurator AddWordAlignmentRepositories( - this IMemoryDataAccessConfigurator configurator - ) - { - configurator.AddRepository(); - configurator.AddRepository(); - configurator.AddRepository(); - return configurator; - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs deleted file mode 100644 index e749d087a..000000000 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs +++ /dev/null @@ -1,44 +0,0 @@ -using Serval.Health.V1; -using Serval.WordAlignment.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServalBuilderExtensions -{ - public static IServalBuilder AddWordAlignment(this IServalBuilder builder) - { - builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); - builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - builder.Services.Configure(builder.Configuration.GetSection(WordAlignmentOptions.Key)); - var wordAlignmentOptions = new WordAlignmentOptions(); - builder.Configuration.GetSection(WordAlignmentOptions.Key).Bind(wordAlignmentOptions); - - foreach (EngineInfo engine in wordAlignmentOptions.Engines) - { - builder.Services.AddGrpcClient( - engine.Type, - o => o.Address = new Uri(engine.Address) - ); - builder.Services.AddGrpcClient( - $"{engine.Type}-Health", - o => o.Address = new Uri(engine.Address) - ); - builder.Services.AddHealthChecks().AddCheck(engine.Type); - } - - builder.Services.AddOutbox(x => - { - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - x.AddConsumer(); - }); - - return builder; - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IServalConfiguratorExtensions.cs similarity index 77% rename from src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs rename to src/Serval/src/Serval.WordAlignment/Configuration/IServalConfiguratorExtensions.cs index 8c92b5ccb..994e6747b 100644 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IServalConfiguratorExtensions.cs @@ -1,15 +1,25 @@ -using MongoDB.Bson; -using MongoDB.Driver; - namespace Microsoft.Extensions.DependencyInjection; -public static class IMongoDataAccessConfiguratorExtensions +public static class IServalConfiguratorExtensions { - public static IMongoDataAccessConfigurator AddWordAlignmentRepositories( - this IMongoDataAccessConfigurator configurator - ) + public static IServalConfigurator AddWordAlignment(this IServalConfigurator configurator) { - configurator.AddRepository( + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + configurator.Services.AddScoped(); + + configurator.AddWordAlignmentDataAccess(); + + configurator.AddHandlers(Assembly.GetExecutingAssembly()); + + return configurator; + } + + public static IServalConfigurator AddWordAlignmentDataAccess(this IServalConfigurator configurator) + { + configurator.DataAccess.AddRepository( "word_alignment.engines", init: [ @@ -23,7 +33,7 @@ this IMongoDataAccessConfigurator configurator ), ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "word_alignment.builds", init: [ @@ -46,7 +56,7 @@ this IMongoDataAccessConfigurator configurator ), ] ); - configurator.AddRepository( + configurator.DataAccess.AddRepository( "word_alignment.word_alignments", init: [ @@ -88,6 +98,7 @@ this IMongoDataAccessConfigurator configurator ), ] ); + return configurator; } } diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs deleted file mode 100644 index ac6cf3e3f..000000000 --- a/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.WordAlignment.Configuration; - -public class WordAlignmentOptions -{ - public const string Key = "WordAlignment"; - - public List Engines { get; set; } = new List(); -} - -public class EngineInfo -{ - public string Type { get; set; } = ""; - public string Address { get; set; } = ""; -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/CorpusUpdatedConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/CorpusUpdatedConsumer.cs deleted file mode 100644 index 43633f9e4..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/CorpusUpdatedConsumer.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace Serval.WordAlignment.Consumers; - -public class CorpusUpdatedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.UpdateCorpusFilesAsync( - context.Message.CorpusId, - context.Message.Files.Select(Map).ToList(), - context.CancellationToken - ); - } - - private static CorpusFile Map(CorpusFileResult corpusFile) - { - return new CorpusFile - { - Id = corpusFile.File.DataFileId, - TextId = corpusFile.TextId ?? corpusFile.File.Name, - Filename = corpusFile.File.Filename, - Format = corpusFile.File.Format, - }; - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs deleted file mode 100644 index c6ab6f3a4..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.WordAlignment.Consumers; - -public class DataFileDeletedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.DeleteAllCorpusFilesAsync(context.Message.DataFileId, context.CancellationToken); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/DataFileUpdatedConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/DataFileUpdatedConsumer.cs deleted file mode 100644 index 49145dbfc..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/DataFileUpdatedConsumer.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Serval.WordAlignment.Consumers; - -public class DataFileUpdatedConsumer(IEngineService engineService) : IConsumer -{ - private readonly IEngineService _engineService = engineService; - - public async Task Consume(ConsumeContext context) - { - await _engineService.UpdateDataFileFilenameFilesAsync( - context.Message.DataFileId, - context.Message.Filename, - context.CancellationToken - ); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/EngineCancelBuildConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/EngineCancelBuildConsumer.cs deleted file mode 100644 index 7eedfb681..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/EngineCancelBuildConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.WordAlignment.Consumers; - -public class EngineCancelBuildConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.CancelBuild) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - CancelBuildRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.CancelBuildAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/EngineCreateConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/EngineCreateConsumer.cs deleted file mode 100644 index e2d4ae3de..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/EngineCreateConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.WordAlignment.Consumers; - -public class EngineCreateConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.Create) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - CreateRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.CreateAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/EngineDeleteConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/EngineDeleteConsumer.cs deleted file mode 100644 index e1d70704d..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/EngineDeleteConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.WordAlignment.Consumers; - -public class EngineDeleteConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.Delete) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - DeleteRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.DeleteAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/EngineStartBuildConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/EngineStartBuildConsumer.cs deleted file mode 100644 index 36ef4b92d..000000000 --- a/src/Serval/src/Serval.WordAlignment/Consumers/EngineStartBuildConsumer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Serval.WordAlignment.V1; - -namespace Serval.WordAlignment.Consumers; - -public class EngineStartBuildConsumer(GrpcClientFactory grpcClientFactory) - : OutboxConsumerBase(EngineOutboxConstants.OutboxId, EngineOutboxConstants.StartBuild) -{ - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; - - protected override async Task HandleMessageAsync( - StartBuildRequest content, - Stream? stream, - CancellationToken cancellationToken - ) - { - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(content.EngineType); - await client.StartBuildAsync(content, cancellationToken: cancellationToken); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs index 87162909e..b065d151e 100644 --- a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs +++ b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs @@ -30,9 +30,7 @@ CancellationToken cancellationToken { try { - return Map( - await _engineService.GetQueueAsync(engineType.ToPascalCase(), cancellationToken: cancellationToken) - ); + return Map(await _engineService.GetQueueAsync(engineType, cancellationToken: cancellationToken)); } catch (InvalidOperationException ioe) { diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs index 25e4c7e56..978c04fa0 100644 --- a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs +++ b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs @@ -204,7 +204,7 @@ CancellationToken cancellationToken /// /// The engine id /// The corpus configuration (see remarks) - /// + /// /// /// /// The added corpus @@ -224,7 +224,7 @@ CancellationToken cancellationToken public async Task> AddParallelCorpusAsync( [NotNull] string id, [FromBody] WordAlignmentParallelCorpusConfigDto corpusConfig, - [FromServices] IRequestClient getCorpusClient, + [FromServices] IRequestHandler getCorpusHandler, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken ) @@ -232,7 +232,7 @@ CancellationToken cancellationToken Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); ParallelCorpus corpus = await MapAsync( - getCorpusClient, + getCorpusHandler, idGenerator.GenerateId(), corpusConfig, cancellationToken @@ -251,7 +251,7 @@ CancellationToken cancellationToken /// The engine id /// The parallel corpus id /// The corpus configuration - /// The data file client + /// The get corpus handler /// /// The corpus was updated successfully /// Bad request @@ -271,7 +271,7 @@ public async Task> UpdateParallelCo [NotNull] string id, [NotNull] string parallelCorpusId, [FromBody] WordAlignmentParallelCorpusUpdateConfigDto corpusConfig, - [FromServices] IRequestClient getCorpusClient, + [FromServices] IRequestHandler getCorpusHandler, CancellationToken cancellationToken ) { @@ -281,10 +281,10 @@ CancellationToken cancellationToken parallelCorpusId, corpusConfig.SourceCorpusIds is null ? null - : await MapAsync(getCorpusClient, corpusConfig.SourceCorpusIds, cancellationToken), + : await MapAsync(getCorpusHandler, corpusConfig.SourceCorpusIds, cancellationToken), corpusConfig.TargetCorpusIds is null ? null - : await MapAsync(getCorpusClient, corpusConfig.TargetCorpusIds, cancellationToken), + : await MapAsync(getCorpusHandler, corpusConfig.TargetCorpusIds, cancellationToken), cancellationToken ); return Ok(Map(id, parallelCorpus)); @@ -690,7 +690,7 @@ private async Task AuthorizeAsync(string id, CancellationToken cancellationToken } private async Task MapAsync( - IRequestClient getDataFileClient, + IRequestHandler getCorpusHandler, string corpusId, WordAlignmentParallelCorpusConfigDto source, CancellationToken cancellationToken @@ -699,13 +699,13 @@ CancellationToken cancellationToken return new ParallelCorpus { Id = corpusId, - SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken), - TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken), + SourceCorpora = await MapAsync(getCorpusHandler, source.SourceCorpusIds, cancellationToken), + TargetCorpora = await MapAsync(getCorpusHandler, source.TargetCorpusIds, cancellationToken), }; } private async Task> MapAsync( - IRequestClient getCorpusClient, + IRequestHandler getCorpusHandler, IEnumerable corpusIds, CancellationToken cancellationToken ) @@ -713,13 +713,10 @@ CancellationToken cancellationToken var corpora = new List(); foreach (string corpusId in corpusIds) { - Response response = await getCorpusClient.GetResponse< - CorpusResult, - CorpusNotFound - >(new GetCorpus { CorpusId = corpusId, Owner = Owner }, cancellationToken); - if (response.Is(out Response? result)) + GetCorpusResponse response = await getCorpusHandler.HandleAsync(new(corpusId, Owner), cancellationToken); + if (response.IsFound) { - if (!result.Message.Files.Any()) + if (!response.Corpus.Files.Any()) { throw new InvalidOperationException( $"The corpus {corpusId} does not have any files associated with it." @@ -729,21 +726,22 @@ CancellationToken cancellationToken new MonolingualCorpus { Id = corpusId, - Name = result.Message.Name ?? "", - Language = result.Message.Language, - Files = result - .Message.Files.Select(f => new CorpusFile + Name = response.Corpus.Name ?? "", + Language = response.Corpus.Language, + Files = + [ + .. response.Corpus.Files.Select(f => new CorpusFile { Id = f.File.DataFileId, Filename = f.File.Filename, Format = f.File.Format, TextId = f.TextId, - }) - .ToList(), + }), + ], } ); } - else if (response.Is(out Response? _)) + else { throw new InvalidOperationException($"The corpus {corpusId} cannot be found."); } @@ -1096,11 +1094,11 @@ private Engine Map(WordAlignmentEngineConfigDto source) }; } - private static PhaseDto Map(BuildPhase source) + private static PhaseDto Map(Phase source) { return new PhaseDto { - Stage = (PhaseStage)source.Stage, + Stage = source.Stage, Step = source.Step, StepCount = source.StepCount, Started = source.Started, diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildConfigDto.cs similarity index 89% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildConfigDto.cs index 115c78f84..aaae86c15 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentBuildConfigDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildDto.cs similarity index 96% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildDto.cs index b9da7babc..d46fb0ada 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentBuildDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentBuildDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusConfigDto.cs similarity index 86% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusConfigDto.cs index b35329fc5..e1271ae7f 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentCorpusConfigDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusDto.cs similarity index 86% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusDto.cs index 1ba2db860..28abb3f0b 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentCorpusDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDto.cs similarity index 91% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDto.cs index bd7900b9a..4df2cb4b4 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineConfigDto.cs similarity index 92% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineConfigDto.cs index 54a242b0a..8b6d302cb 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentEngineConfigDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineDto.cs similarity index 92% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineDto.cs index 542b4a3d7..ae0f37630 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentEngineDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentEngineDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs similarity index 88% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentExecutionDataDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 1c5fa8171..8f37edfc6 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentExecutionDataDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusConfigDto.cs similarity index 87% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusConfigDto.cs index ddd7ce7c7..3069a0a34 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusConfigDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentParallelCorpusConfigDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusDto.cs similarity index 89% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusDto.cs index 53f00302a..9aaf69f68 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentParallelCorpusDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusUpdateDto.cs similarity index 94% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusUpdateDto.cs index 5a9669476..ba6f89bd5 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentParallelCorpusUpdateDto.cs @@ -1,6 +1,6 @@ using System.ComponentModel.DataAnnotations; -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentParallelCorpusUpdateConfigDto : IValidatableObject { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentRequestDto.cs similarity index 77% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentRequestDto.cs index 0ad1fa0b8..e2f7c08af 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentRequestDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentRequestDto { diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentResultDto.cs similarity index 85% rename from src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentResultDto.cs index 209bd6ca0..40c302344 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentResultDto.cs @@ -1,4 +1,4 @@ -namespace Serval.WordAlignment.Contracts; +namespace Serval.WordAlignment.Dtos; public record WordAlignmentResultDto { diff --git a/src/Serval/src/Serval.WordAlignment/Handlers/CorpusUpdatedHandler.cs b/src/Serval/src/Serval.WordAlignment/Handlers/CorpusUpdatedHandler.cs new file mode 100644 index 000000000..096273584 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Handlers/CorpusUpdatedHandler.cs @@ -0,0 +1,20 @@ +namespace Serval.WordAlignment.Handlers; + +public class CorpusUpdatedHandler(IEngineService engineService) : IEventHandler +{ + public async Task HandleAsync(CorpusUpdated evt, CancellationToken cancellationToken) + { + await engineService.UpdateCorpusFilesAsync(evt.CorpusId, [.. evt.Files.Select(Map)], cancellationToken); + } + + private static CorpusFile Map(CorpusDataFileContract corpusFile) + { + return new CorpusFile + { + Id = corpusFile.File.DataFileId, + TextId = corpusFile.TextId ?? corpusFile.File.Name, + Filename = corpusFile.File.Filename, + Format = corpusFile.File.Format, + }; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Handlers/DataFileDeletedHandler.cs b/src/Serval/src/Serval.WordAlignment/Handlers/DataFileDeletedHandler.cs new file mode 100644 index 000000000..aa254c447 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Handlers/DataFileDeletedHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Handlers; + +public class DataFileDeletedHandler(IEngineService engineService) : IEventHandler +{ + public Task HandleAsync(DataFileDeleted evt, CancellationToken cancellationToken) + { + return engineService.DeleteAllCorpusFilesAsync(evt.DataFileId, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Handlers/DataFileUpdatedHandler.cs b/src/Serval/src/Serval.WordAlignment/Handlers/DataFileUpdatedHandler.cs new file mode 100644 index 000000000..c35b7b811 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Handlers/DataFileUpdatedHandler.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Handlers; + +public class DataFileUpdatedHandler(IEngineService engineService) : IEventHandler +{ + public Task HandleAsync(DataFileUpdated evt, CancellationToken cancellationToken) + { + return engineService.UpdateDataFileFilenameFilesAsync(evt.DataFileId, evt.Filename, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/Build.cs b/src/Serval/src/Serval.WordAlignment/Models/Build.cs index 115ae57b5..535affa91 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/Build.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/Build.cs @@ -20,5 +20,5 @@ public record Build : IEntity public DateTime? DateCreated { get; set; } public DateTime? DateStarted { get; set; } public DateTime? DateCompleted { get; set; } - public IReadOnlyList? Phases { get; init; } + public IReadOnlyList? Phases { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Models/CorpusFile.cs b/src/Serval/src/Serval.WordAlignment/Models/CorpusFile.cs new file mode 100644 index 000000000..f456f070f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/CorpusFile.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Models; + +public record CorpusFile +{ + public required string Id { get; set; } + public required string Filename { get; set; } + public required FileFormat Format { get; set; } + public required string TextId { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/MonolingualCorpus.cs new file mode 100644 index 000000000..07eac893f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/MonolingualCorpus.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Models; + +public record MonolingualCorpus +{ + public required string Id { get; set; } + public string? Name { get; set; } + public required string Language { get; set; } + public required IReadOnlyList Files { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs new file mode 100644 index 000000000..1ebfd50b3 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record ParallelCorpus +{ + public required string Id { get; set; } + public required IReadOnlyList SourceCorpora { get; set; } + public required IReadOnlyList TargetCorpora { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpusFilter.cs b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpusFilter.cs new file mode 100644 index 000000000..18fe93c97 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpusFilter.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record ParallelCorpusFilter +{ + public required string CorpusRef { get; set; } + public IReadOnlyList? TextIds { get; set; } + public string? ScriptureRange { get; set; } +} diff --git a/src/Serval/src/Serval.Shared/Models/Queue.cs b/src/Serval/src/Serval.WordAlignment/Models/Queue.cs similarity index 75% rename from src/Serval/src/Serval.Shared/Models/Queue.cs rename to src/Serval/src/Serval.WordAlignment/Models/Queue.cs index 7a3272a82..b1c821726 100644 --- a/src/Serval/src/Serval.Shared/Models/Queue.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/Queue.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.WordAlignment.Models; public record Queue { diff --git a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs index 47b5e5f05..3185d3dc0 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs @@ -1,8 +1,8 @@ -namespace Serval.WordAlignment.Models; +namespace Serval.WordAlignment.Models; public record WordAlignmentResult { - public required IReadOnlyList SourceTokens { get; set; } - public required IReadOnlyList TargetTokens { get; set; } - public required IReadOnlyList Alignment { get; set; } + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Alignment { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj b/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj index 21baab6fa..e4badd8cc 100644 --- a/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj +++ b/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj @@ -10,18 +10,18 @@ $(NoWarn);CS1591;CS1573 + + - - - - + + diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineOutboxConstants.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineOutboxConstants.cs deleted file mode 100644 index 622b1ff7a..000000000 --- a/src/Serval/src/Serval.WordAlignment/Services/EngineOutboxConstants.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Serval.WordAlignment.Services; - -public static class EngineOutboxConstants -{ - public const string OutboxId = "WordAlignmentEngine"; - - public const string Create = "Create"; - public const string Update = "Update"; - public const string Delete = "Delete"; - public const string StartBuild = "StartBuild"; - public const string CancelBuild = "CancelBuild"; -} diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs index 5b6addff7..a91c833fc 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs @@ -1,27 +1,21 @@ -using Serval.WordAlignment.V1; - namespace Serval.WordAlignment.Services; public class EngineService( IRepository engines, IRepository builds, IRepository wordAlignments, - GrpcClientFactory grpcClientFactory, + IEngineServiceFactory engineServiceFactory, IOptionsMonitor dataFileOptions, IDataAccessContext dataAccessContext, - ILoggerFactory loggerFactory, - IOutboxService outboxService, - IOptionsMonitor wordAlignmentOptions + ILoggerFactory loggerFactory ) : OwnedEntityServiceBase(engines), IEngineService { private readonly IRepository _builds = builds; private readonly IRepository _wordAlignments = wordAlignments; - private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; + private readonly IEngineServiceFactory _engineServiceFactory = engineServiceFactory; private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = loggerFactory.CreateLogger(); - private readonly IOutboxService _outboxService = outboxService; - private readonly IOptionsMonitor _wordAlignmentOptions = wordAlignmentOptions; public override async Task> GetAllAsync( string owner, @@ -31,7 +25,7 @@ public override async Task> GetAllAsync( return await Entities.GetAllAsync(e => e.Owner == owner, cancellationToken); } - public async Task GetWordAlignmentAsync( + public async Task GetWordAlignmentAsync( string engineId, string sourceSegment, string targetSegment, @@ -42,23 +36,27 @@ public override async Task> GetAllAsync( if (engine.ModelRevision == 0) return null; - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); try { - GetWordAlignmentResponse response = await client.GetWordAlignmentAsync( - new GetWordAlignmentRequest - { - EngineType = engine.Type, - EngineId = engine.Id, - SourceSegment = sourceSegment, - TargetSegment = targetSegment, - }, - cancellationToken: cancellationToken - ); - return Map(response.Result); + WordAlignmentResultContract result = await _engineServiceFactory + .GetEngineService(engine.Type) + .AlignAsync(engine.Id, sourceSegment, targetSegment, cancellationToken); + return new WordAlignmentResult + { + SourceTokens = result.SourceTokens, + TargetTokens = result.TargetTokens, + Alignment = + [ + .. result.Alignment.Select(p => new AlignedWordPair + { + SourceIndex = p.SourceIndex, + TargetIndex = p.TargetIndex, + Score = p.Score, + }), + ], + }; } - catch (RpcException re) when (re.StatusCode is StatusCode.NotFound or StatusCode.FailedPrecondition) + catch (InvalidOperationException) { return null; } @@ -66,7 +64,7 @@ public override async Task> GetAllAsync( public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) { - if (!_wordAlignmentOptions.CurrentValue.Engines.Any(e => e.Type == engine.Type)) + if (!_engineServiceFactory.EngineTypeExists(engine.Type)) throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type."); await _dataAccessContext.WithTransactionAsync( @@ -75,23 +73,9 @@ await _dataAccessContext.WithTransactionAsync( engine.DateCreated = DateTime.UtcNow; await Entities.InsertAsync(engine, cancellationToken); - CreateRequest request = new() - { - EngineType = engine.Type, - EngineId = engine.Id, - SourceLanguage = engine.SourceLanguage, - TargetLanguage = engine.TargetLanguage, - }; - if (engine.Name is not null) - request.EngineName = engine.Name; - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.Create, - engine.Id, - request, - cancellationToken: ct - ); + await _engineServiceFactory + .GetEngineService(engine.Type) + .CreateAsync(engine.Id, engine.SourceLanguage, engine.TargetLanguage, engine.Name, ct); }, cancellationToken ); @@ -110,15 +94,7 @@ await _dataAccessContext.WithTransactionAsync( await _builds.DeleteAllAsync(b => b.EngineRef == engineId, ct); await _wordAlignments.DeleteAllAsync(wa => wa.EngineRef == engineId, ct); - DeleteRequest request = new() { EngineType = engine.Type, EngineId = engine.Id }; - - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.Delete, - engine.Id, - request, - cancellationToken: ct - ); + await _engineServiceFactory.GetEngineService(engine.Type).DeleteAsync(engine.Id, ct); }, cancellationToken ); @@ -167,7 +143,7 @@ await _builds.ExistsAsync( Dictionary? wordAlignOn = build.WordAlignOn?.ToDictionary(c => c.ParallelCorpusRef ); - IReadOnlyList parallelCorpora = engine + IReadOnlyList parallelCorpora = engine .ParallelCorpora.Where(pc => trainOn == null || trainOn.ContainsKey(pc.Id) @@ -176,63 +152,55 @@ await _builds.ExistsAsync( ) .ToList(); - StartBuildRequest request = new() - { - EngineType = engine.Type, - EngineId = engine.Id, - BuildId = build.Id, - Corpora = - { - parallelCorpora.Select(c => - Map( - c, - trainOn?.GetValueOrDefault(c.Id), - wordAlignOn?.GetValueOrDefault(c.Id), - trainOn is null, - wordAlignOn is null - ) - ), - }, - }; + IReadOnlyList corpora = parallelCorpora + .Select(c => + MapToFilteredCorpus( + c, + trainOn?.GetValueOrDefault(c.Id), + wordAlignOn?.GetValueOrDefault(c.Id), + trainOn is null, + wordAlignOn is null + ) + ) + .ToList(); + string? buildOptions = null; if (build.Options is not null) - request.Options = JsonSerializer.Serialize(build.Options); + buildOptions = JsonSerializer.Serialize(build.Options); // Log the build request summary try { - var buildRequestSummary = (JsonObject)JsonNode.Parse(JsonSerializer.Serialize(request))!; - // correct build options parsing - buildRequestSummary.Remove("Options"); + var buildRequestSummary = new JsonObject + { + ["Event"] = "BuildRequest", + ["EngineId"] = engine.Id, + ["BuildId"] = build.Id, + ["CorpusCount"] = corpora.Count, + ["ModelRevision"] = engine.ModelRevision, + ["ClientId"] = engine.Owner, + }; try { - buildRequestSummary.Add("Options", JsonNode.Parse(request.Options)); + buildRequestSummary.Add("Options", JsonNode.Parse(buildOptions ?? "null")); } catch (JsonException) { buildRequestSummary.Add( "Options", - "Build \"Options\" failed parsing: " + (request.Options ?? "null") + "Build \"Options\" failed parsing: " + (buildOptions ?? "null") ); } - buildRequestSummary.Add("Event", "BuildRequest"); - buildRequestSummary.Add("ModelRevision", engine.ModelRevision); - buildRequestSummary.Add("ClientId", engine.Owner); _logger.LogInformation("{request}", buildRequestSummary.ToJsonString()); } catch (JsonException) { _logger.LogInformation("Error parsing build request summary."); - _logger.LogInformation("{request}", JsonSerializer.Serialize(request)); } - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engine.Id, - request, - cancellationToken: ct - ); + await _engineServiceFactory + .GetEngineService(engine.Type) + .StartBuildAsync(engine.Id, build.Id, corpora, buildOptions, ct); return true; }, cancellationToken @@ -247,21 +215,14 @@ await _outboxService.EnqueueMessageAsync( cancellationToken ); - CancelBuildRequest request = new CancelBuildRequest { EngineType = engine.Type, EngineId = engine.Id }; - await _outboxService.EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.CancelBuild, - engine.Id, - request, - cancellationToken: cancellationToken - ); + await _engineServiceFactory.GetEngineService(engine.Type).CancelBuildAsync(engine.Id, cancellationToken); return currentBuild; } public Task AddParallelCorpusAsync( string engineId, - Shared.Models.ParallelCorpus corpus, + ParallelCorpus corpus, CancellationToken cancellationToken = default ) { @@ -272,15 +233,15 @@ public Task AddParallelCorpusAsync( ); } - public async Task UpdateParallelCorpusAsync( + public async Task UpdateParallelCorpusAsync( string engineId, string parallelCorpusId, - IReadOnlyList? sourceCorpora, - IReadOnlyList? targetCorpora, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { - Shared.Models.ParallelCorpus? parallelCorpus = null; + ParallelCorpus? parallelCorpus = null; await _dataAccessContext.WithTransactionAsync( async (ct) => { @@ -450,7 +411,7 @@ await _wordAlignments.DeleteAllAsync( public async Task UpdateCorpusFilesAsync( string corpusId, - IReadOnlyList files, + IReadOnlyList files, CancellationToken cancellationToken = default ) { @@ -482,37 +443,12 @@ await Entities.UpdateAllAsync( public async Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default) { - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(engineType); - GetQueueSizeResponse response = await client.GetQueueSizeAsync( - new GetQueueSizeRequest { EngineType = engineType }, - cancellationToken: cancellationToken - ); - return new Queue { Size = response.Size, EngineType = engineType }; - } - - private Models.WordAlignmentResult Map(V1.WordAlignmentResult source) - { - return new Models.WordAlignmentResult - { - SourceTokens = source.SourceTokens.ToList(), - TargetTokens = source.TargetTokens.ToList(), - Alignment = source.Alignment.Select(Map).ToList(), - }; - } - - private Models.AlignedWordPair Map(V1.AlignedWordPair source) - { - return new Models.AlignedWordPair - { - SourceIndex = source.SourceIndex, - TargetIndex = source.TargetIndex, - Score = source.Score, - }; + int size = await _engineServiceFactory.GetEngineService(engineType).GetQueueSizeAsync(cancellationToken); + return new Queue { Size = size, EngineType = engineType }; } - private V1.ParallelCorpus Map( - Shared.Models.ParallelCorpus source, + private ParallelCorpusContract MapToFilteredCorpus( + ParallelCorpus source, TrainingCorpus? trainingCorpus, WordAlignmentCorpus? wordAlignmentCorpus, bool trainOnAllCorpora, @@ -521,7 +457,7 @@ bool wordAlignOnAllCorpora { string? referenceFileLocation = source.TargetCorpora.Count > 0 && source.TargetCorpora[0].Files.Count > 0 - ? Map(source.TargetCorpora[0].Files[0]).Location + ? Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.TargetCorpora[0].Files[0].Filename) : null; bool trainOnAllSources = @@ -534,13 +470,12 @@ bool wordAlignOnAllCorpora bool wordAlignAllTargets = wordAlignOnAllCorpora || (wordAlignmentCorpus is not null && wordAlignmentCorpus.TargetFilters is null); - return new V1.ParallelCorpus + return new ParallelCorpusContract { Id = source.Id, - SourceCorpora = - { - source.SourceCorpora.Select(sc => - Map( + SourceCorpora = source + .SourceCorpora.Select(sc => + MapToFilteredMonolingualCorpus( sc, trainingCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), wordAlignmentCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), @@ -548,12 +483,11 @@ bool wordAlignOnAllCorpora trainOnAllSources, wordAlignAllSources ) - ), - }, - TargetCorpora = - { - source.TargetCorpora.Select(tc => - Map( + ) + .ToList(), + TargetCorpora = source + .TargetCorpora.Select(tc => + MapToFilteredMonolingualCorpus( tc, trainingCorpus?.TargetFilters?.Where(sf => sf.CorpusRef == tc.Id).FirstOrDefault(), null, @@ -561,13 +495,13 @@ bool wordAlignOnAllCorpora trainOnAllTargets, wordAlignAllTargets ) - ), - }, + ) + .ToList(), }; } - private V1.MonolingualCorpus Map( - Shared.Models.MonolingualCorpus inputCorpus, + private MonolingualCorpusContract MapToFilteredMonolingualCorpus( + MonolingualCorpus inputCorpus, ParallelCorpusFilter? trainingFilter, ParallelCorpusFilter? wordAlignmentFilter, string? referenceFileLocation, @@ -575,7 +509,7 @@ private V1.MonolingualCorpus Map( bool wordAlignOnAll ) { - Dictionary? trainOnChapters = null; + Dictionary>? trainOnChapters = null; if ( trainingFilter is not null && trainingFilter.ScriptureRange is not null @@ -583,18 +517,10 @@ trainingFilter is not null ) { trainOnChapters = GetChapters(referenceFileLocation, trainingFilter.ScriptureRange) - .Select( - (kvp) => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - } - ) - .ToDictionary(); + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToHashSet()); } - Dictionary? wordAlignmentChapters = null; + Dictionary>? wordAlignmentChapters = null; if ( wordAlignmentFilter is not null && wordAlignmentFilter.ScriptureRange is not null @@ -602,24 +528,9 @@ wordAlignmentFilter is not null ) { wordAlignmentChapters = GetChapters(referenceFileLocation, wordAlignmentFilter.ScriptureRange) - .Select( - (kvp) => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - } - ) - .ToDictionary(); + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToHashSet()); } - var returnCorpus = new V1.MonolingualCorpus - { - Id = inputCorpus.Id, - Language = inputCorpus.Language, - Files = { inputCorpus.Files.Select(Map) }, - }; - if ( trainingFilter is not null && trainingFilter.TextIds is not null @@ -631,21 +542,6 @@ trainingFilter is not null ); } - if ( - trainOnAll - || (trainingFilter is not null && trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null) - ) - { - returnCorpus.TrainOnAll = true; - } - else - { - if (trainOnChapters is not null) - returnCorpus.TrainOnChapters.Add(trainOnChapters); - if (trainingFilter?.TextIds is not null) - returnCorpus.TrainOnTextIds.Add(trainingFilter.TextIds); - } - if ( wordAlignmentFilter is not null && wordAlignmentFilter.TextIds is not null @@ -657,35 +553,32 @@ wordAlignmentFilter is not null ); } - if ( - wordAlignOnAll - || ( - wordAlignmentFilter is not null - && wordAlignmentFilter.TextIds is null - && wordAlignmentFilter.ScriptureRange is null - ) - ) - { - returnCorpus.WordAlignOnAll = true; - } - else + var result = new MonolingualCorpusContract { - if (wordAlignmentChapters is not null) - returnCorpus.WordAlignOnChapters.Add(wordAlignmentChapters); - if (wordAlignmentFilter?.TextIds is not null) - returnCorpus.WordAlignOnTextIds.Add(wordAlignmentFilter.TextIds); - } + Id = inputCorpus.Id, + Language = inputCorpus.Language, + Files = inputCorpus + .Files.Select(f => new CorpusFileContract + { + TextId = f.TextId, + Format = f.Format, + Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, f.Filename), + }) + .ToList(), + TrainOnTextIds = trainOnAll || trainingFilter is not null ? null : [], + InferenceTextIds = wordAlignOnAll || wordAlignmentFilter is not null ? null : [], + }; - return returnCorpus; - } + if (trainOnChapters is not null) + result.TrainOnChapters = trainOnChapters; + if (trainingFilter?.TextIds is not null) + result.TrainOnTextIds = trainingFilter.TextIds.ToHashSet(); - private V1.CorpusFile Map(Shared.Models.CorpusFile source) - { - return new V1.CorpusFile - { - TextId = source.TextId, - Format = (V1.FileFormat)source.Format, - Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename), - }; + if (wordAlignmentChapters is not null) + result.InferenceChapters = wordAlignmentChapters; + if (wordAlignmentFilter?.TextIds is not null) + result.InferenceTextIds = wordAlignmentFilter.TextIds.ToHashSet(); + + return result; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineServiceFactory.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineServiceFactory.cs new file mode 100644 index 000000000..7ce6c4222 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/EngineServiceFactory.cs @@ -0,0 +1,20 @@ +namespace Serval.WordAlignment.Services; + +public class EngineServiceFactory(IServiceProvider serviceProvider) : IEngineServiceFactory +{ + private readonly IServiceProvider _serviceProvider = serviceProvider; + + public bool TryGetEngineService(string engineType, [NotNullWhen(true)] out IWordAlignmentEngineService? service) + { + IWordAlignmentEngineService? engine = _serviceProvider.GetKeyedService( + engineType.ToLowerInvariant() + ); + if (engine is null) + { + service = null; + return false; + } + service = engine; + return true; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/IEngineServiceFactory.cs b/src/Serval/src/Serval.WordAlignment/Services/IEngineServiceFactory.cs new file mode 100644 index 000000000..33937e283 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/IEngineServiceFactory.cs @@ -0,0 +1,6 @@ +namespace Serval.WordAlignment.Services; + +public interface IEngineServiceFactory +{ + bool TryGetEngineService(string engineType, [NotNullWhen(true)] out IWordAlignmentEngineService? service); +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs new file mode 100644 index 000000000..a67bc6a17 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs @@ -0,0 +1,369 @@ +namespace Serval.WordAlignment.Services; + +public class PlatformService( + IRepository builds, + IRepository engines, + IRepository wordAlignments, + IDataAccessContext dataAccessContext, + IEventRouter eventRouter +) : IWordAlignmentPlatformService +{ + private const int WordAlignmentInsertBatchSize = 128; + + private readonly IRepository _builds = builds; + private readonly IRepository _engines = engines; + private readonly IRepository _wordAlignments = wordAlignments; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IEventRouter _eventRouter = eventRouter; + + public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => u.Set(b => b.State, JobState.Active), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, true), + cancellationToken: ct + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + + await _eventRouter.PublishAsync( + new WordAlignmentBuildStarted(BuildId: build.Id, EngineId: engine.Id, Owner: engine.Owner), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildCompletedAsync( + string buildId, + int corpusSize, + double confidence, + CancellationToken cancellationToken = default + ) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.State, JobState.Completed) + .Set(b => b.Message, "Completed") + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => + u.Set(e => e.Confidence, confidence) + .Set(e => e.CorpusSize, corpusSize) + .Set(e => e.IsBuilding, false) + .Inc(e => e.ModelRevision), + cancellationToken: ct + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + + // delete alignments created by the previous build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision < engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new WordAlignmentBuildFinished( + BuildId: build.Id, + EngineId: engine.Id, + Owner: engine.Owner, + BuildState: build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.Message, "Canceled") + .Set(b => b.DateFinished, DateTime.UtcNow) + .Set(b => b.State, JobState.Canceled), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + + // delete word alignments that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new WordAlignmentBuildFinished( + BuildId: build.Id, + EngineId: engine.Id, + Owner: engine.Owner, + BuildState: build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.State, JobState.Faulted) + .Set(b => b.Message, message) + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + + // delete word alignments that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _eventRouter.PublishAsync( + new WordAlignmentBuildFinished( + BuildId: build.Id, + EngineId: engine.Id, + Owner: engine.Owner, + BuildState: build.State, + build.Message!, + build.DateFinished!.Value + ), + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + buildId, + u => + u.Set(b => b.Message, "Restarting") + .Set(b => b.Step, 0) + .Set(b => b.Progress, 0) + .Set(b => b.State, JobState.Pending), + cancellationToken: ct + ); + if (build is null) + throw new EntityNotFoundException($"Could not find the Build '{buildId}'."); + + Engine? engine = await _engines.GetAsync(build.EngineRef, ct); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{build.EngineRef}'."); + + // delete word alignments that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + }, + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildStatusAsync( + string buildId, + BuildProgressStatusContract progressStatus, + int? queueDepth = null, + IReadOnlyCollection? phases = null, + DateTime? started = null, + DateTime? completed = null, + CancellationToken cancellationToken = default + ) + { + await _builds.UpdateAsync( + b => b.Id == buildId && (b.State == JobState.Active || b.State == JobState.Pending), + u => + { + u.Set(b => b.Step, progressStatus.Step); + if (progressStatus.PercentCompleted.HasValue) + { + u.Set( + b => b.Progress, + Math.Round(progressStatus.PercentCompleted.Value, 4, MidpointRounding.AwayFromZero) + ); + } + if (progressStatus.Message is not null) + u.Set(b => b.Message, progressStatus.Message); + if (queueDepth.HasValue) + u.Set(b => b.QueueDepth, queueDepth.Value); + if (phases is not null && phases.Count > 0) + { + u.Set( + b => b.Phases, + [ + .. phases.Select(p => new Phase + { + Stage = p.Stage, + Step = p.Step, + Started = p.Started, + StepCount = p.StepCount, + }), + ] + ); + } + if (started.HasValue) + u.Set(b => b.DateStarted, started.Value); + if (completed.HasValue) + u.Set(b => b.DateCompleted, completed.Value); + }, + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) + { + await _builds.UpdateAsync( + b => b.Id == buildId && (b.State == JobState.Active || b.State == JobState.Pending), + u => u.Set(b => b.Step, step), + cancellationToken: cancellationToken + ); + } + + public async Task IncrementEngineCorpusSizeAsync( + string engineId, + int count = 1, + CancellationToken cancellationToken = default + ) + { + await _engines.UpdateAsync( + engineId, + u => u.Inc(e => e.CorpusSize, count), + cancellationToken: cancellationToken + ); + } + + public async Task InsertWordAlignmentsAsync( + string engineId, + IAsyncEnumerable wordAlignments, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); + int nextModelRevision = engine.ModelRevision + 1; + + var batch = new List(); + await foreach (WordAlignmentContract item in wordAlignments.WithCancellation(cancellationToken)) + { + batch.Add( + new Models.WordAlignment + { + EngineRef = engineId, + ModelRevision = nextModelRevision, + CorpusRef = item.CorpusId, + TextId = item.TextId, + SourceRefs = item.SourceRefs.ToList(), + TargetRefs = item.TargetRefs.ToList(), + Refs = item.TargetRefs.ToList(), + SourceTokens = item.SourceTokens.ToList(), + TargetTokens = item.TargetTokens.ToList(), + Alignment = item + .Alignment.Select(a => new AlignedWordPair + { + SourceIndex = a.SourceIndex, + TargetIndex = a.TargetIndex, + Score = a.Score, + }) + .ToList(), + } + ); + if (batch.Count == WordAlignmentInsertBatchSize) + { + await _wordAlignments.InsertAllAsync(batch, cancellationToken); + batch.Clear(); + } + } + if (batch.Count > 0) + await _wordAlignments.InsertAllAsync(batch, CancellationToken.None); + } + + public async Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + ExecutionDataContract executionData, + CancellationToken cancellationToken = default + ) + { + await _builds.UpdateAsync( + b => b.Id == buildId, + u => + u.Set( + b => b.ExecutionData, + new ExecutionData + { + TrainCount = executionData.TrainCount, + WordAlignCount = executionData.WordAlignCount, + Warnings = executionData.Warnings?.ToList() ?? [], + EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, + EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, + } + ), + cancellationToken: cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/ServicesExtensions.cs b/src/Serval/src/Serval.WordAlignment/Services/ServicesExtensions.cs new file mode 100644 index 000000000..dd9f0f7b9 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/ServicesExtensions.cs @@ -0,0 +1,16 @@ +namespace Serval.WordAlignment.Services; + +public static class ServicesExtensions +{ + public static IWordAlignmentEngineService GetEngineService(this IEngineServiceFactory factory, string engineType) + { + if (factory.TryGetEngineService(engineType, out IWordAlignmentEngineService? service)) + return service; + throw new InvalidOperationException($"No engine registered for type '{engineType}'."); + } + + public static bool EngineTypeExists(this IEngineServiceFactory factory, string engineType) + { + return factory.TryGetEngineService(engineType, out _); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs b/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs deleted file mode 100644 index 690b935e2..000000000 --- a/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs +++ /dev/null @@ -1,373 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.WordAlignment.V1; - -namespace Serval.WordAlignment.Services; - -public class WordAlignmentPlatformServiceV1( - IRepository builds, - IRepository engines, - IRepository wordAlignments, - IDataAccessContext dataAccessContext, - IPublishEndpoint publishEndpoint -) : WordAlignmentPlatformApi.WordAlignmentPlatformApiBase -{ - private const int WordAlignmentInsertBatchSize = 128; - private static readonly Empty Empty = new(); - - private readonly IRepository _builds = builds; - private readonly IRepository _engines = engines; - private readonly IRepository _wordAlignments = wordAlignments; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IPublishEndpoint _publishEndpoint = publishEndpoint; - - public override async Task BuildStarted(BuildStartedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => u.Set(b => b.State, JobState.Active), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, true), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - await _publishEndpoint.Publish( - new WordAlignmentBuildStarted - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - return Empty; - } - - public override async Task BuildCompleted(BuildCompletedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.State, JobState.Completed) - .Set(b => b.Message, "Completed") - .Set(b => b.DateFinished, DateTime.UtcNow), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => - u.Set(e => e.Confidence, request.Confidence) - .Set(e => e.CorpusSize, request.CorpusSize) - .Set(e => e.IsBuilding, false) - .Inc(e => e.ModelRevision), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete alignments created by the previous build - await _wordAlignments.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision < engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new WordAlignmentBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildCanceled(BuildCanceledRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.Message, "Canceled") - .Set(b => b.DateFinished, DateTime.UtcNow) - .Set(b => b.State, JobState.Canceled), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, false), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete word alignments that might have been created during the build - await _wordAlignments.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new WordAlignmentBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildFaulted(BuildFaultedRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.State, JobState.Faulted) - .Set(b => b.Message, request.Message) - .Set(b => b.DateFinished, DateTime.UtcNow), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.UpdateAsync( - build.EngineRef, - u => u.Set(e => e.IsBuilding, false), - cancellationToken: ct - ); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete word alignments that might have been created during the build - await _wordAlignments.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - - await _publishEndpoint.Publish( - new WordAlignmentBuildFinished - { - BuildId = build.Id, - EngineId = engine.Id, - Owner = engine.Owner, - BuildState = build.State, - Message = build.Message!, - DateFinished = build.DateFinished!.Value, - }, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task BuildRestarting(BuildRestartingRequest request, ServerCallContext context) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - Build? build = await _builds.UpdateAsync( - request.BuildId, - u => - u.Set(b => b.Message, "Restarting") - .Set(b => b.Step, 0) - .Set(b => b.Progress, 0) - .Set(b => b.State, JobState.Pending), - cancellationToken: ct - ); - if (build is null) - throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); - - Engine? engine = await _engines.GetAsync(build.EngineRef, ct); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - - // delete word alignments that might have been created during the build - await _wordAlignments.DeleteAllAsync( - p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, - ct - ); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task UpdateBuildStatus(UpdateBuildStatusRequest request, ServerCallContext context) - { - await _builds.UpdateAsync( - b => b.Id == request.BuildId && (b.State == JobState.Active || b.State == JobState.Pending), - u => - { - u.Set(b => b.Step, request.Step); - if (request.HasProgress) - u.Set(b => b.Progress, Math.Round(request.Progress, 4, MidpointRounding.AwayFromZero)); - if (request.HasMessage) - u.Set(b => b.Message, request.Message); - if (request.HasQueueDepth) - u.Set(b => b.QueueDepth, request.QueueDepth); - if (request.Phases.Count > 0) - { - u.Set( - b => b.Phases, - request - .Phases.Select(p => new BuildPhase - { - Stage = (BuildPhaseStage)p.Stage, - Step = p.HasStep ? p.Step : null, - StepCount = p.HasStepCount ? p.StepCount : null, - Started = p.Started?.ToDateTime(), - }) - .ToList() - ); - } - - if (request.Started is not null) - u.Set(b => b.DateStarted, request.Started.ToDateTime()); - if (request.Completed is not null) - u.Set(b => b.DateCompleted, request.Completed.ToDateTime()); - }, - cancellationToken: context.CancellationToken - ); - - return Empty; - } - - public override async Task IncrementEngineCorpusSize( - IncrementEngineCorpusSizeRequest request, - ServerCallContext context - ) - { - await _engines.UpdateAsync( - request.EngineId, - u => u.Inc(e => e.CorpusSize, request.Count), - cancellationToken: context.CancellationToken - ); - return Empty; - } - - public override async Task InsertWordAlignments( - IAsyncStreamReader requestStream, - ServerCallContext context - ) - { - string engineId = ""; - int nextModelRevision = 0; - - var batch = new List(); - await foreach (InsertWordAlignmentsRequest request in requestStream.ReadAllAsync(context.CancellationToken)) - { - if (request.EngineId != engineId) - { - Engine? engine = await _engines.GetAsync(request.EngineId, context.CancellationToken); - if (engine is null) - throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); - nextModelRevision = engine.ModelRevision + 1; - engineId = request.EngineId; - } - batch.Add( - new Models.WordAlignment - { - EngineRef = request.EngineId, - ModelRevision = nextModelRevision, - CorpusRef = request.CorpusId, - TextId = request.TextId, - SourceRefs = request.SourceRefs.ToList(), - TargetRefs = request.TargetRefs.ToList(), - Refs = request.TargetRefs.ToList(), - SourceTokens = request.SourceTokens.ToList(), - TargetTokens = request.TargetTokens.ToList(), - Alignment = request - .Alignment.Select(a => new Models.AlignedWordPair - { - SourceIndex = a.SourceIndex, - TargetIndex = a.TargetIndex, - Score = a.Score, - }) - .ToList(), - } - ); - if (batch.Count == WordAlignmentInsertBatchSize) - { - await _wordAlignments.InsertAllAsync(batch, context.CancellationToken); - batch.Clear(); - } - } - if (batch.Count > 0) - await _wordAlignments.InsertAllAsync(batch, CancellationToken.None); - - return Empty; - } - - public override async Task UpdateBuildExecutionData( - UpdateBuildExecutionDataRequest request, - ServerCallContext context - ) - { - await _builds.UpdateAsync( - b => b.Id == request.BuildId, - u => - u.Set( - b => b.ExecutionData, - new Models.ExecutionData - { - TrainCount = request.ExecutionData.TrainCount, - WordAlignCount = request.ExecutionData.WordAlignCount, - Warnings = [.. request.ExecutionData.Warnings], - EngineSourceLanguageTag = request.ExecutionData.EngineSourceLanguageTag, - EngineTargetLanguageTag = request.ExecutionData.EngineTargetLanguageTag, - } - ), - cancellationToken: context.CancellationToken - ); - - return new Empty(); - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Usings.cs b/src/Serval/src/Serval.WordAlignment/Usings.cs index 389a1a698..dabe86cb2 100644 --- a/src/Serval/src/Serval.WordAlignment/Usings.cs +++ b/src/Serval/src/Serval.WordAlignment/Usings.cs @@ -1,31 +1,31 @@ global using System.Diagnostics.CodeAnalysis; global using System.Linq.Expressions; +global using System.Reflection; global using System.Text.Json; global using System.Text.Json.Nodes; global using Asp.Versioning; global using CaseExtensions; -global using Grpc.Core; -global using Grpc.Net.ClientFactory; -global using MassTransit; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.Routing; global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; +global using MongoDB.Bson; +global using MongoDB.Driver; global using NSwag.Annotations; +global using Serval.DataFiles.Contracts; global using Serval.Shared.Configuration; global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; +global using Serval.Shared.Dtos; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; -global using Serval.WordAlignment.Configuration; -global using Serval.WordAlignment.Consumers; global using Serval.WordAlignment.Contracts; +global using Serval.WordAlignment.Dtos; global using Serval.WordAlignment.Models; global using Serval.WordAlignment.Services; global using SIL.DataAccess; -global using SIL.ServiceToolkit.Services; -global using SIL.ServiceToolkit.Utils; diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/DataFilesTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/DataFilesTests.cs index 374b56265..9e0d25a5a 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/DataFilesTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/DataFilesTests.cs @@ -140,7 +140,7 @@ public async Task CreateAsync(IEnumerable scope, int expectedStatusCode) fp = new FileParameter(fs); ServalApiException? ex = Assert.ThrowsAsync(async () => { - await client.CreateAsync(fp, FileFormat.Text); + await client.CreateAsync(fp, Client.FileFormat.Text); }); Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/ServalWebApplicationFactory.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/ServalWebApplicationFactory.cs index a95d9bb66..1685c9f67 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/ServalWebApplicationFactory.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/ServalWebApplicationFactory.cs @@ -1,12 +1,22 @@ -using Serval.Translation.Configuration; -using Serval.WordAlignment.Configuration; - -namespace Serval.ApiServer; +namespace Serval.ApiServer; public class ServalWebApplicationFactory : WebApplicationFactory { protected override void ConfigureWebHost(IWebHostBuilder builder) { + builder.ConfigureAppConfiguration( + (_, config) => + { + config.AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:Mongo"] = "mongodb://localhost:27017/serval_test", + ["ConnectionStrings:Hangfire"] = "mongodb://localhost:27017/serval_test_jobs", + } + ); + } + ); + builder.ConfigureServices(services => { services @@ -17,48 +27,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) }) .AddScheme("TestScheme", options => { }); - services.Configure(options => - options.Url = new MongoUrl("mongodb://localhost:27017/serval_test") - ); - services.Configure(options => options.LongPollTimeout = TimeSpan.FromSeconds(1)); - - services.Configure(options => - { - options.Engines = - [ - new Translation.Configuration.EngineInfo { Type = "Echo" }, - new Translation.Configuration.EngineInfo { Type = "Nmt" }, - ]; - }); - - services.Configure(options => - { - options.Engines = - [ - new WordAlignment.Configuration.EngineInfo { Type = "EchoWordAlignment" }, - new WordAlignment.Configuration.EngineInfo { Type = "Statistical" }, - ]; - }); - - services.AddHangfire(c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - "mongodb://localhost:27017/serval_test_jobs", - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy(), - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); }); } } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/StatusTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/StatusTests.cs index da173d376..617c21f66 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/StatusTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/StatusTests.cs @@ -4,7 +4,7 @@ namespace Serval.ApiServer; [Category("Integration")] public class StatusTests { - TestEnvironment? _env; + TestEnvironment _env; [SetUp] public void SetUp() @@ -18,7 +18,7 @@ public void SetUp() [TestCase(new[] { Scopes.CreateTranslationEngines }, 403)] public async Task GetHealthAsync(IEnumerable? scope, int expectedStatusCode) { - StatusClient client = _env!.CreateClient(scope); + StatusClient client = _env.CreateClient(scope); ServalApiException ex; switch (expectedStatusCode) { @@ -27,7 +27,7 @@ public async Task GetHealthAsync(IEnumerable? scope, int expectedStatusC HealthReport healthReport = await client.GetHealthAsync(); Assert.That(healthReport, Is.Not.Null); Assert.That(healthReport.Status.ToString(), Is.Not.EqualTo("Healthy")); - Assert.That(healthReport.Results, Has.Count.EqualTo(9)); + Assert.That(healthReport.Results, Has.Count.EqualTo(7)); break; case 401: @@ -49,7 +49,7 @@ public async Task GetHealthAsync(IEnumerable? scope, int expectedStatusC [Test] public async Task GetDeploymentAsync() { - StatusClient client = _env!.CreateClient(); + StatusClient client = _env.CreateClient(); DeploymentInfo deploymentInfo = await client.GetDeploymentInfoAsync(); Assert.That(deploymentInfo, Is.Not.Null); Assert.That(deploymentInfo.DeploymentVersion, Is.EqualTo("Unknown")); @@ -59,7 +59,7 @@ public async Task GetDeploymentAsync() [Test] public async Task GetPingAsync() { - StatusClient client = _env!.CreateClient(); + StatusClient client = _env.CreateClient(); HealthReport healthReport = await client.GetPingAsync(); Assert.That(healthReport, Is.Not.Null); Assert.That(healthReport.Status.ToString(), Is.Not.EqualTo("Healthy")); @@ -69,7 +69,7 @@ public async Task GetPingAsync() [TearDown] public void TearDown() { - _env!.Dispose(); + _env.Dispose(); } private class TestEnvironment : DisposableBase diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 24ee437e4..db66794ed 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -1,12 +1,5 @@ -using System.IO.Compression; -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.Configuration; +using Serval.Translation.Contracts; using Serval.Translation.Models; -using Serval.Translation.V1; -using static Serval.ApiServer.Utils; -using Phase = Serval.Client.Phase; -using PhaseStage = Serval.Client.PhaseStage; -using Queue = Serval.Client.Queue; namespace Serval.ApiServer; @@ -402,7 +395,7 @@ public async Task DeleteEngineByIdAsync(IEnumerable scope, int expectedS [Test] [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 200, ECHO_ENGINE1_ID)] [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] - [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE1_ID)] + [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE2_ID)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task TranslateSegmentWithEngineByIdAsync( IEnumerable scope, @@ -432,8 +425,6 @@ await _env.Builds.InsertAsync( break; case 409: { - _env.EchoClient.TranslateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { await client.TranslateAsync(engineId, "This is a test ."); @@ -460,7 +451,7 @@ await _env.Builds.InsertAsync( [Test] [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 200, ECHO_ENGINE1_ID)] [TestCase(new[] { Scopes.ReadTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] - [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE1_ID)] + [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE2_ID)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task TranslateNSegmentWithEngineByIdAsync( IEnumerable scope, @@ -495,8 +486,6 @@ await _env.Builds.InsertAsync( break; case 409: { - _env.EchoClient.TranslateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { await client.TranslateNAsync(engineId, 1, "This is a test ."); @@ -523,7 +512,7 @@ await _env.Builds.InsertAsync( [Test] [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 200, ECHO_ENGINE1_ID)] [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] - [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE1_ID)] + [TestCase(new[] { Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE2_ID)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task GetWordGraphForSegmentByIdAsync( IEnumerable scope, @@ -553,13 +542,6 @@ await _env.Builds.InsertAsync( break; case 409: { - _env.EchoClient.GetWordGraphAsync( - Arg.Any(), - null, - null, - Arg.Any() - ) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { await client.GetWordGraphAsync(engineId, "This is a test ."); @@ -586,7 +568,7 @@ await _env.Builds.InsertAsync( [Test] [TestCase(new[] { Scopes.UpdateTranslationEngines }, 200, ECHO_ENGINE1_ID)] [TestCase(new[] { Scopes.UpdateTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] - [TestCase(new[] { Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE1_ID)] + [TestCase(new[] { Scopes.UpdateTranslationEngines }, 409, ECHO_ENGINE2_ID)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task TrainEngineByIdOnSegmentPairAsync( IEnumerable scope, @@ -617,13 +599,6 @@ await _env.Builds.InsertAsync( break; case 409: { - _env.EchoClient.TrainSegmentPairAsync( - Arg.Any(), - null, - null, - Arg.Any() - ) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { await client.TrainSegmentAsync(engineId, sp); @@ -1361,9 +1336,8 @@ bool buildOwnedByClient switch (expectedStatusCode) { case 200: - ICollection results = await client.GetAllBuildsCreatedAfterAsync( - DateTime.UtcNow.AddHours(-1) - ); + var createdAfter = DateTime.UtcNow.AddHours(-1); + ICollection results = await client.GetAllBuildsCreatedAfterAsync(createdAfter); if (buildOwnedByClient) { Assert.That(results, Is.Not.Empty); @@ -1371,7 +1345,7 @@ bool buildOwnedByClient { Assert.That(results.First().Revision, Is.EqualTo(1)); Assert.That(results.First().Id, Is.EqualTo(build?.Id)); - Assert.That(results.First().State, Is.EqualTo(JobState.Pending)); + Assert.That(results.First().State, Is.EqualTo(Client.JobState.Pending)); }); } else @@ -1418,7 +1392,7 @@ public async Task GetAllBuildsForEngineByIdAsync( { Assert.That(results.First().Revision, Is.EqualTo(1)); Assert.That(results.First().Id, Is.EqualTo(build?.Id)); - Assert.That(results.First().State, Is.EqualTo(JobState.Pending)); + Assert.That(results.First().State, Is.EqualTo(Client.JobState.Pending)); }); break; case 403: @@ -1465,7 +1439,7 @@ public async Task GetBuildByIdForEngineByIdAsync( { Assert.That(result.Revision, Is.EqualTo(1)); Assert.That(result.Id, Is.EqualTo(build.Id)); - Assert.That(result.State, Is.EqualTo(JobState.Pending)); + Assert.That(result.State, Is.EqualTo(Client.JobState.Pending)); }); break; } @@ -1519,7 +1493,7 @@ public async Task GetNextFinishedBuildAsync(IEnumerable scope, int expec { Assert.That(result.Revision, Is.EqualTo(1)); Assert.That(result.Id, Is.EqualTo(build?.Id)); - Assert.That(result.State, Is.EqualTo(JobState.Completed)); + Assert.That(result.State, Is.EqualTo(Client.JobState.Completed)); }); break; case 403: @@ -1907,9 +1881,9 @@ public async Task GetCurrentBuildForEngineByIdAsync( Owner = "client1", Phases = [ - new BuildPhase + new Shared.Models.Phase { - Stage = BuildPhaseStage.Train, + Stage = Shared.Contracts.PhaseStage.Train, Step = 1, StepCount = 2, }, @@ -1928,9 +1902,9 @@ public async Task GetCurrentBuildForEngineByIdAsync( Assert.That( result.Phases![0], Is.EqualTo( - new Phase + new Client.Phase { - Stage = PhaseStage.Train, + Stage = Client.PhaseStage.Train, Step = 1, StepCount = 2, } @@ -1982,13 +1956,8 @@ public async Task CancelCurrentBuildForEngineByIdAsync( string buildId = "b00000000000000000000000"; if (addBuild) { - _env.EchoClient.CancelBuildAsync( - Arg.Is(new CancelBuildRequest() { EngineId = engineId, EngineType = "Echo" }), - null, - null, - Arg.Any() - ) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse() { BuildId = buildId })); + _env.EchoService.CancelBuildAsync(engineId, Arg.Any()) + .Returns(Task.FromResult(buildId)); var build = new Build { Id = buildId, @@ -2276,8 +2245,6 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() var ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] }; var tbc = new TranslationBuildConfig { Pretranslate = [ptcc] }; TranslationBuild build = await client.StartBuildAsync(engineId, tbc); - _env.NmtClient.StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { build = await client.StartBuildAsync(engineId, tbc); @@ -2513,12 +2480,12 @@ await _env.Builds.InsertAsync( Owner = "client1", Analysis = [ - new Shared.Models.ParallelCorpusAnalysis() + new Translation.Models.ParallelCorpusAnalysis() { ParallelCorpusRef = "111111111111111111111112", TargetQuoteConvention = "", }, - new Shared.Models.ParallelCorpusAnalysis() + new Translation.Models.ParallelCorpusAnalysis() { ParallelCorpusRef = "111111111111111111111113", TargetQuoteConvention = "standard_english", @@ -2576,172 +2543,135 @@ public TestEnvironment() >(); Builds = _scope.ServiceProvider.GetRequiredService>(); - EchoClient = Substitute.For(); - EchoClient - .CreateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .DeleteAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - EchoClient - .GetModelDownloadUrlAsync( - Arg.Any(), - null, - null, - Arg.Any() - ) - .Returns( - CreateAsyncUnaryCall( - new GetModelDownloadUrlResponse - { - Url = "http://example.com", - ModelRevision = 1, - ExpiresAt = DateTime.UtcNow.AddHours(1).ToTimestamp(), - } - ) - ); - var wg = new Translation.V1.WordGraph + var sources = new HashSet + { + Translation.Contracts.TranslationSource.Primary, + Translation.Contracts.TranslationSource.Secondary, + }; + var translationResult = new TranslationResultContract + { + Translation = "This is a test .", + SourceTokens = "This is a test .".Split(), + TargetTokens = "This is a test .".Split(), + Confidences = [1.0, 1.0, 1.0, 1.0, 1.0], + Sources = [sources, sources, sources, sources, sources], + Alignment = + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + new AlignedWordPairContract { SourceIndex = 2, TargetIndex = 2 }, + new AlignedWordPairContract { SourceIndex = 3, TargetIndex = 3 }, + new AlignedWordPairContract { SourceIndex = 4, TargetIndex = 4 }, + ], + Phrases = + [ + new PhraseContract + { + SourceSegmentStart = 0, + SourceSegmentEnd = 5, + TargetSegmentCut = 5, + }, + ], + }; + var wordGraph = new WordGraphContract { - SourceTokens = { "This is a test .".Split() }, - FinalStates = { 4 }, + SourceTokens = "This is a test .".Split(), + InitialStateScore = 0.0, + FinalStates = new HashSet { 4 }, Arcs = - { - new Translation.V1.WordGraphArc + [ + new WordGraphArcContract { PrevState = 0, NextState = 1, Score = 1.0, + TargetTokens = [], + Confidences = [], + SourceSegmentStart = 0, + SourceSegmentEnd = 1, + Alignment = [], + Sources = [], }, - new Translation.V1.WordGraphArc + new WordGraphArcContract { PrevState = 1, NextState = 2, Score = 1.0, + TargetTokens = [], + Confidences = [], + SourceSegmentStart = 1, + SourceSegmentEnd = 2, + Alignment = [], + Sources = [], }, - new Translation.V1.WordGraphArc + new WordGraphArcContract { PrevState = 2, NextState = 3, Score = 1.0, + TargetTokens = [], + Confidences = [], + SourceSegmentStart = 2, + SourceSegmentEnd = 3, + Alignment = [], + Sources = [], }, - new Translation.V1.WordGraphArc + new WordGraphArcContract { PrevState = 3, NextState = 4, Score = 1.0, - }, - }, - }; - var wgr = new GetWordGraphResponse { WordGraph = wg }; - EchoClient - .TrainSegmentPairAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .GetWordGraphAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(wgr)); - - var translationResult = new Translation.V1.TranslationResult - { - Translation = "This is a test .", - SourceTokens = { "This is a test .".Split() }, - TargetTokens = { "This is a test .".Split() }, - Confidences = { 1.0, 1.0, 1.0, 1.0, 1.0 }, - Sources = - { - new TranslationSources - { - Values = - { - Translation.V1.TranslationSource.Primary, - Translation.V1.TranslationSource.Secondary, - }, - }, - new TranslationSources - { - Values = - { - Translation.V1.TranslationSource.Primary, - Translation.V1.TranslationSource.Secondary, - }, - }, - new TranslationSources - { - Values = - { - Translation.V1.TranslationSource.Primary, - Translation.V1.TranslationSource.Secondary, - }, - }, - new TranslationSources - { - Values = - { - Translation.V1.TranslationSource.Primary, - Translation.V1.TranslationSource.Secondary, - }, - }, - new TranslationSources - { - Values = - { - Translation.V1.TranslationSource.Primary, - Translation.V1.TranslationSource.Secondary, - }, - }, - }, - Alignment = - { - new Translation.V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new Translation.V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - new Translation.V1.AlignedWordPair { SourceIndex = 2, TargetIndex = 2 }, - new Translation.V1.AlignedWordPair { SourceIndex = 3, TargetIndex = 3 }, - new Translation.V1.AlignedWordPair { SourceIndex = 4, TargetIndex = 4 }, - }, - Phrases = - { - new Translation.V1.Phrase - { - SourceSegmentStart = 0, + TargetTokens = [], + Confidences = [], + SourceSegmentStart = 3, SourceSegmentEnd = 5, - TargetSegmentCut = 5, + Alignment = [], + Sources = [], }, - }, + ], }; - var translateResponse = new TranslateResponse { Results = { translationResult } }; - EchoClient - .TranslateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(translateResponse)); - EchoClient - .GetQueueSizeAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new GetQueueSizeResponse() { Size = 0 })); - - NmtClient = Substitute.For(); - NmtClient - .CreateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - NmtClient - .DeleteAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - NmtClient - .StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - NmtClient - .CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - NmtClient - .GetWordGraphAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.Unimplemented)); - NmtClient - .TranslateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.Unimplemented)); - - SmtClient = Substitute.For(); + var languageInfo = new LanguageInfoContract { IsNative = true, InternalCode = "abc_123" }; + + EchoService = Substitute.For(); + EchoService + .TranslateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([translationResult])); + EchoService + .GetWordGraphAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(wordGraph)); + EchoService + .GetModelDownloadUrlAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult( + new ModelDownloadUrlContract + { + Url = "http://example.com", + ModelRevision = 1, + ExpiresAt = DateTime.UtcNow.AddHours(1), + } + ) + ); + EchoService.GetQueueSizeAsync(Arg.Any()).Returns(Task.FromResult(0)); + EchoService + .GetLanguageInfoAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(languageInfo)); + + NmtService = Substitute.For(); + NmtService + .GetWordGraphAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException(new NotSupportedException())); + NmtService + .TranslateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException>(new NotSupportedException())); + NmtService.GetQueueSizeAsync(Arg.Any()).Returns(Task.FromResult(0)); + NmtService + .GetLanguageInfoAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(languageInfo)); + + SmtService = Substitute.For(); + SmtService + .GetLanguageInfoAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(languageInfo)); _dataFileOptions = _scope.ServiceProvider.GetRequiredService>(); ZipParatextProject(FILE3_FILENAME); ZipParatextProject(FILE4_FILENAME); @@ -2753,9 +2683,9 @@ public TestEnvironment() public IRepository Corpora { get; } public IRepository Pretranslations { get; } public IRepository Builds { get; } - public TranslationEngineApi.TranslationEngineApiClient EchoClient { get; } - public TranslationEngineApi.TranslationEngineApiClient NmtClient { get; } - public TranslationEngineApi.TranslationEngineApiClient SmtClient { get; } + public ITranslationEngineService EchoService { get; } + public ITranslationEngineService NmtService { get; } + public ITranslationEngineService SmtService { get; } public TranslationBuildsClient CreateTranslationBuildsClient(IEnumerable? scope = null) { @@ -2763,17 +2693,7 @@ public TranslationBuildsClient CreateTranslationBuildsClient(IEnumerable HttpClient httpClient = Factory .WithWebHostBuilder(builder => { - builder.ConfigureTestServices(services => - { - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("Echo") - .Returns(EchoClient); - grpcClientFactory - .CreateClient("Nmt") - .Returns(NmtClient); - services.AddSingleton(grpcClientFactory); - }); + builder.ConfigureTestServices(ConfigureEngineServices); }) .CreateClient(); httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); @@ -2792,17 +2712,7 @@ public TranslationEnginesClient CreateTranslationEnginesClient(IEnumerable { - builder.ConfigureTestServices(services => - { - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("Echo") - .Returns(EchoClient); - grpcClientFactory - .CreateClient("Nmt") - .Returns(NmtClient); - services.AddSingleton(grpcClientFactory); - }); + builder.ConfigureTestServices(ConfigureEngineServices); }) .CreateClient(); httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); @@ -2821,44 +2731,20 @@ public TranslationEngineTypesClient CreateTranslationEngineTypesClient(IEnumerab HttpClient httpClient = Factory .WithWebHostBuilder(builder => { - builder.ConfigureTestServices(services => - { - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("Echo") - .Returns(EchoClient); - grpcClientFactory - .CreateClient("Nmt") - .Returns(NmtClient); - grpcClientFactory - .CreateClient("SmtTransfer") - .Returns(SmtClient); - services.AddSingleton(grpcClientFactory); - }); + builder.ConfigureTestServices(ConfigureEngineServices); }) .CreateClient(); - NmtClient - .GetQueueSizeAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new GetQueueSizeResponse() { Size = 0 })); - NmtClient - .GetLanguageInfoAsync(Arg.Any(), null, null, Arg.Any()) - .Returns( - CreateAsyncUnaryCall(new GetLanguageInfoResponse() { InternalCode = "abc_123", IsNative = true }) - ); - SmtClient - .GetLanguageInfoAsync(Arg.Any(), null, null, Arg.Any()) - .Returns( - CreateAsyncUnaryCall(new GetLanguageInfoResponse() { InternalCode = "abc_123", IsNative = true }) - ); - EchoClient - .GetLanguageInfoAsync(Arg.Any(), null, null, Arg.Any()) - .Returns( - CreateAsyncUnaryCall(new GetLanguageInfoResponse() { InternalCode = "abc_123", IsNative = true }) - ); httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); return new TranslationEngineTypesClient(httpClient); } + private void ConfigureEngineServices(IServiceCollection services) + { + services.AddKeyedScoped("echo", (_, _) => EchoService); + services.AddKeyedScoped("nmt", (_, _) => NmtService); + services.AddKeyedScoped("smttransfer", (_, _) => SmtService); + } + public DataFilesClient CreateDataFilesClient() { IEnumerable scope = [Scopes.DeleteFiles, Scopes.ReadFiles, Scopes.UpdateFiles, Scopes.CreateFiles]; diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs index 6eb916516..78d5b08c1 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs @@ -1,13 +1,8 @@ +global using System.IO.Compression; global using System.Security.Claims; global using System.Text; global using System.Text.Encodings.Web; global using System.Xml.Linq; -global using Grpc.Core; -global using Grpc.Net.ClientFactory; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; global using Microsoft.AspNetCore.Authentication; global using Microsoft.AspNetCore.Mvc.Testing; global using Microsoft.AspNetCore.TestHost; @@ -17,7 +12,9 @@ global using NUnit.Framework; global using Serval.Client; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; -global using Serval.Shared.Models; +global using Serval.Shared.Services; +global using Serval.Translation.Configuration; global using SIL.DataAccess; global using SIL.ObjectModel; diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs deleted file mode 100644 index fd571f51e..000000000 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs +++ /dev/null @@ -1,27 +0,0 @@ -namespace Serval.ApiServer; - -public static class Utils -{ - public static AsyncUnaryCall CreateAsyncUnaryCall(StatusCode statusCode) - { - var status = new Status(statusCode, string.Empty); - return new AsyncUnaryCall( - Task.FromException(new RpcException(status)), - Task.FromResult(new Metadata()), - () => status, - () => [], - () => { } - ); - } - - public static AsyncUnaryCall CreateAsyncUnaryCall(TResponse response) - { - return new AsyncUnaryCall( - Task.FromResult(response), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => [], - () => { } - ); - } -} diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/WebhooksTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/WebhooksTests.cs index 9c5dcfe1c..71491c6ab 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/WebhooksTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/WebhooksTests.cs @@ -19,7 +19,7 @@ public async Task Setup() Owner = "client1", Url = "/a/url", Secret = "s3CreT#", - Events = [Webhooks.Contracts.WebhookEvent.TranslationBuildStarted], + Events = [Webhooks.Models.WebhookEvent.TranslationBuildStarted], }; await _env.Webhooks.InsertAsync(webhook); } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/WordAlignmentEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/WordAlignmentEngineTests.cs index cd6e689b7..38eb10029 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/WordAlignmentEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/WordAlignmentEngineTests.cs @@ -1,10 +1,5 @@ -using Google.Protobuf.WellKnownTypes; +using Serval.WordAlignment.Contracts; using Serval.WordAlignment.Models; -using Serval.WordAlignment.V1; -using SIL.ServiceToolkit.Services; -using static Serval.ApiServer.Utils; -using Phase = Serval.Client.Phase; -using PhaseStage = Serval.Client.PhaseStage; namespace Serval.ApiServer; @@ -363,7 +358,7 @@ public async Task DeleteEngineByIdAsync(IEnumerable scope, int expectedS 404, DOES_NOT_EXIST_ENGINE_ID )] - [TestCase(new[] { Scopes.ReadWordAlignmentEngines, Scopes.UpdateWordAlignmentEngines }, 409, ECHO_ENGINE1_ID)] + [TestCase(new[] { Scopes.ReadWordAlignmentEngines, Scopes.UpdateWordAlignmentEngines }, 409, ECHO_ENGINE2_ID)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task GetWordAlignmentForSegmentPairWithEngineByIdAsync( IEnumerable scope, @@ -387,13 +382,6 @@ await _env.Builds.InsertAsync( break; case 409: { - _env.EchoClient.GetWordAlignmentAsync( - Arg.Any(), - null, - null, - Arg.Any() - ) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { await client.AlignAsync( @@ -901,7 +889,7 @@ public async Task GetAllBuildsForEngineByIdAsync( { Assert.That(results.First().Revision, Is.EqualTo(1)); Assert.That(results.First().Id, Is.EqualTo(build?.Id)); - Assert.That(results.First().State, Is.EqualTo(JobState.Pending)); + Assert.That(results.First().State, Is.EqualTo(Client.JobState.Pending)); }); break; case 403: @@ -948,7 +936,7 @@ public async Task GetBuildByIdForEngineByIdAsync( { Assert.That(result.Revision, Is.EqualTo(1)); Assert.That(result.Id, Is.EqualTo(build.Id)); - Assert.That(result.State, Is.EqualTo(JobState.Pending)); + Assert.That(result.State, Is.EqualTo(Client.JobState.Pending)); }); break; } @@ -1243,9 +1231,9 @@ public async Task GetCurrentBuildForEngineByIdAsync( EngineRef = engineId, Phases = [ - new BuildPhase + new Shared.Models.Phase { - Stage = BuildPhaseStage.Train, + Stage = Shared.Contracts.PhaseStage.Train, Step = 1, StepCount = 2, }, @@ -1264,9 +1252,9 @@ public async Task GetCurrentBuildForEngineByIdAsync( Assert.That( result.Phases![0], Is.EqualTo( - new Phase + new Client.Phase { - Stage = PhaseStage.Train, + Stage = Client.PhaseStage.Train, Step = 1, StepCount = 2, } @@ -1318,13 +1306,6 @@ public async Task CancelCurrentBuildForEngineByIdAsync( string buildId = "b00000000000000000000000"; if (addBuild) { - _env.EchoClient.CancelBuildAsync( - Arg.Is(new CancelBuildRequest() { EngineId = engineId, EngineType = "EchoWordAlignment" }), - null, - null, - Arg.Any() - ) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse() { BuildId = buildId })); var build = new Build { Id = buildId, EngineRef = engineId }; await _env.Builds.InsertAsync(build); } @@ -1526,8 +1507,6 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() WordAlignmentCorpusConfig wacc = new() { ParallelCorpusId = addedCorpus.Id }; var tbc = new WordAlignmentBuildConfig { WordAlignOn = [wacc] }; WordAlignmentBuild build = await client.StartBuildAsync(engineId, tbc); - _env.StatisticalClient.StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.FailedPrecondition)); ServalApiException? ex = Assert.ThrowsAsync(async () => { build = await client.StartBuildAsync(engineId, tbc); @@ -1628,12 +1607,12 @@ public void TearDown() _env.Dispose(); } - private static IReadOnlyList CreateNAlignedWordPair(int numberOfAlignedWords) + private static IReadOnlyList CreateNAlignedWordPair(int numberOfAlignedWords) { - var alignedWordPairs = new List(); + var alignedWordPairs = new List(); for (int i = 0; i < numberOfAlignedWords; i++) { - alignedWordPairs.Add(new WordAlignment.Models.AlignedWordPair { SourceIndex = i, TargetIndex = i }); + alignedWordPairs.Add(new Serval.Shared.Models.AlignedWordPair { SourceIndex = i, TargetIndex = i }); } return alignedWordPairs; } @@ -1658,56 +1637,31 @@ public TestEnvironment() >(); Builds = _scope.ServiceProvider.GetRequiredService>(); - EchoClient = Substitute.For(); - EchoClient - .CreateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .DeleteAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - EchoClient - .CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - var wordAlignmentResult = new WordAlignment.V1.WordAlignmentResult + var wordAlignmentResult = new WordAlignmentResultContract { - SourceTokens = { "This is a test .".Split() }, - TargetTokens = { "This is a test .".Split() }, + SourceTokens = "This is a test .".Split(), + TargetTokens = "This is a test .".Split(), Alignment = - { - new WordAlignment.V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new WordAlignment.V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - new WordAlignment.V1.AlignedWordPair { SourceIndex = 2, TargetIndex = 2 }, - new WordAlignment.V1.AlignedWordPair { SourceIndex = 3, TargetIndex = 3 }, - new WordAlignment.V1.AlignedWordPair { SourceIndex = 4, TargetIndex = 4 }, - }, + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + new AlignedWordPairContract { SourceIndex = 2, TargetIndex = 2 }, + new AlignedWordPairContract { SourceIndex = 3, TargetIndex = 3 }, + new AlignedWordPairContract { SourceIndex = 4, TargetIndex = 4 }, + ], }; - var wordAlignmentResponse = new GetWordAlignmentResponse { Result = wordAlignmentResult }; - EchoClient - .GetWordAlignmentAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(wordAlignmentResponse)); - EchoClient - .GetQueueSizeAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new GetQueueSizeResponse() { Size = 0 })); - - StatisticalClient = Substitute.For(); - StatisticalClient - .CreateAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - StatisticalClient - .DeleteAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - StatisticalClient - .StartBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - StatisticalClient - .CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - StatisticalClient - .GetWordAlignmentAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(StatusCode.Unimplemented)); + + EchoService = Substitute.For(); + EchoService + .AlignAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(wordAlignmentResult)); + EchoService.GetQueueSizeAsync(Arg.Any()).Returns(Task.FromResult(0)); + + StatisticalService = Substitute.For(); + StatisticalService + .AlignAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException(new NotSupportedException())); + StatisticalService.GetQueueSizeAsync(Arg.Any()).Returns(Task.FromResult(0)); } public ServalWebApplicationFactory Factory { get; } @@ -1716,8 +1670,8 @@ public TestEnvironment() public IRepository Corpora { get; } public IRepository WordAlignments { get; } public IRepository Builds { get; } - public WordAlignmentEngineApi.WordAlignmentEngineApiClient EchoClient { get; } - public WordAlignmentEngineApi.WordAlignmentEngineApiClient StatisticalClient { get; } + public IWordAlignmentEngineService EchoService { get; } + public IWordAlignmentEngineService StatisticalService { get; } public WordAlignmentEnginesClient CreateWordAlignmentEnginesClient(IEnumerable? scope = null) { @@ -1727,14 +1681,7 @@ public WordAlignmentEnginesClient CreateWordAlignmentEnginesClient(IEnumerable { - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("EchoWordAlignment") - .Returns(EchoClient); - grpcClientFactory - .CreateClient("Statistical") - .Returns(StatisticalClient); - services.AddSingleton(grpcClientFactory); + ConfigureEngineServices(services); services.AddTransient(CreateFileSystem); }); }) @@ -1743,6 +1690,12 @@ public WordAlignmentEnginesClient CreateWordAlignmentEnginesClient(IEnumerable EchoService); + services.AddKeyedScoped("statistical", (_, _) => StatisticalService); + } + public DataFilesClient CreateDataFilesClient() { IEnumerable scope = [Scopes.DeleteFiles, Scopes.ReadFiles, Scopes.UpdateFiles, Scopes.CreateFiles]; diff --git a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs index d842653e5..bd398b09d 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs +++ b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs @@ -56,7 +56,7 @@ public TestEnvironment() Corpora, Substitute.For>(), DataAccessContext, - Substitute.For() + Substitute.For() ); } diff --git a/src/Serval/test/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs b/src/Serval/test/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs index 4f6387365..32651106b 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs +++ b/src/Serval/test/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs @@ -89,7 +89,7 @@ public void UpdateAsync_GetAsyncFails() // We will use the mediator to cancel the token, which will cause GetAsync() to fail // What we are testing for is GetAsync() failing due to network or other connectivity issues, token cancellation being one source var cts = new CancellationTokenSource(); - env.Mediator.When(x => x.Publish(Arg.Any(), Arg.Any())) + env.EventRouter.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) .Do(_ => cts.Cancel()); // Set up a valid existing file @@ -137,7 +137,7 @@ public async Task DeleteAsync_Exists() Assert.That(env.DataFiles.Contains(DataFileId), Is.False); DeletedFile deletedFile = env.DeletedFiles.Entities.Single(); Assert.That(deletedFile.Filename, Is.EqualTo("file1.txt")); - await env.Mediator.Received().Publish(Arg.Any(), Arg.Any()); + await env.EventRouter.Received().PublishAsync(Arg.Any(), Arg.Any()); } [Test] @@ -154,14 +154,14 @@ public TestEnvironment() DataFiles = new MemoryRepository(); IOptionsMonitor options = Substitute.For>(); options.CurrentValue.Returns(new DataFileOptions()); - Mediator = Substitute.For(); + EventRouter = Substitute.For(); DeletedFiles = new MemoryRepository(); FileSystem = Substitute.For(); Service = new DataFileService( DataFiles, new MemoryDataAccessContext(), options, - Mediator, + EventRouter, DeletedFiles, FileSystem ); @@ -169,7 +169,7 @@ public TestEnvironment() public IFileSystem FileSystem { get; } public MemoryRepository DeletedFiles { get; } - public IScopedMediator Mediator { get; } + public IEventRouter EventRouter { get; } public MemoryRepository DataFiles { get; } public DataFileService Service { get; } } diff --git a/src/Serval/test/Serval.DataFiles.Tests/Usings.cs b/src/Serval/test/Serval.DataFiles.Tests/Usings.cs index d625c262b..a7f99fa21 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Usings.cs +++ b/src/Serval/test/Serval.DataFiles.Tests/Usings.cs @@ -1,13 +1,13 @@ global using System.Text; -global using MassTransit.Mediator; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; +global using Serval.DataFiles.Contracts; global using Serval.DataFiles.Models; global using Serval.Shared.Configuration; global using Serval.Shared.Contracts; +global using Serval.Shared.Services; global using Serval.Shared.Utils; global using SIL.DataAccess; -global using SIL.ServiceToolkit.Services; diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index 1ac3ed8a5..157b5876a 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -83,15 +83,15 @@ public async Task Echo_ParallelCorpus(bool paratext) [TestCase(false)] public async Task Echo_WordAlignment(bool paratext) { - string engineId = await _helperClient.CreateNewEngineAsync("EchoWordAlignment", "es", "en", "Echo4"); + string engineId = await _helperClient.CreateNewEngineAsync("EchoWordAlignment", "es", "es", "Echo4"); if (paratext) { - await _helperClient.AddParatextCorpusToEngineAsync(engineId, "es", "en", false); + await _helperClient.AddParatextCorpusToEngineAsync(engineId, "es", "es", false); } else { string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; - ParallelCorpusConfig trainCorpus = await _helperClient.MakeParallelTextCorpus(books, "es", "en", false); + ParallelCorpusConfig trainCorpus = await _helperClient.MakeParallelTextCorpus(books, "es", "es", false); await _helperClient.AddParallelTextCorpusToEngineAsync(engineId, trainCorpus, false); } diff --git a/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs b/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs index a0ddd8678..4f36caed2 100644 --- a/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs +++ b/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs @@ -16,14 +16,16 @@ public void SetUp() public async Task InitializesRepositories() { // Setup - IServalBuilder servalBuilder = new ServalBuilder(_env.Services, _env.Configuration); - servalBuilder.AddMongoDataAccess(cfg => - { - cfg.AddTranslationRepositories(); - cfg.AddWordAlignmentRepositories(); - cfg.AddDataFilesRepositories(); - cfg.AddWebhooksRepositories(); - }); + _env.Services.AddServal( + _env.Configuration, + c => + { + c.AddTranslationDataAccess(); + c.AddWordAlignmentDataAccess(); + c.AddDataFilesDataAccess(); + c.AddWebhooksDataAccess(); + } + ); // SUT await _env.InitializeDatabaseAsync(); @@ -37,11 +39,7 @@ public async Task InitializesRepositories() public async Task Migrates_TranslationEngines_ParallelCorpora() { // Setup - IServalBuilder servalBuilder = new ServalBuilder(_env.Services, _env.Configuration); - servalBuilder.AddMongoDataAccess(cfg => - { - cfg.AddTranslationRepositories(); - }); + _env.Services.AddServal(_env.Configuration, c => c.AddTranslation()); // Populate pre-migration-data await _env.SetupSchemaAsync("translation.engines", 2); diff --git a/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj b/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj index 0e931be4a..fc91f32d2 100644 --- a/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj +++ b/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj @@ -18,6 +18,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/CorpusBundleTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs similarity index 90% rename from src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/CorpusBundleTests.cs rename to src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs index 4db628d07..efb1e907a 100644 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/CorpusBundleTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs @@ -1,6 +1,4 @@ -using SIL.ServiceToolkit.Utils; - -namespace SIL.ServiceToolkit.Services; +namespace Serval.Shared.Services; public class CorpusBundleTests { @@ -124,24 +122,24 @@ public TestEnvironment(bool addParatext, bool addText) ); private readonly TempDirectory _tempDir = new(name: "CorpusBundleTests"); - public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) + public ParallelCorpusContract[] GetCorpora(bool addParatext, bool addText) { - List parallelCorpora = []; + List parallelCorpora = []; if (addParatext) { parallelCorpora.AddRange( - new ParallelCorpus + new ParallelCorpusContract { Id = "corpus1", SourceCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "pt-source1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Paratext, @@ -153,13 +151,13 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) ], TargetCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "pt-target1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Paratext, @@ -169,18 +167,18 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) }, ], }, - new ParallelCorpus + new ParallelCorpusContract { Id = "corpus2", SourceCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "pt-source1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Paratext, @@ -192,13 +190,13 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) ], TargetCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "pt-target1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Paratext, @@ -214,18 +212,18 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) if (addText) { parallelCorpora.AddRange( - new ParallelCorpus + new ParallelCorpusContract { Id = "corpus1", SourceCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "source-corpus1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Text, @@ -233,13 +231,13 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) }, ], }, - new MonolingualCorpus + new MonolingualCorpusContract { Id = "source-corpus2", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Text, @@ -250,13 +248,13 @@ public ParallelCorpus[] GetCorpora(bool addParatext, bool addText) ], TargetCorpora = [ - new MonolingualCorpus + new MonolingualCorpusContract { Id = "target-corpus1", Language = "en", Files = [ - new CorpusFile + new CorpusFileContract { TextId = "textId1", Format = FileFormat.Text, diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs new file mode 100644 index 000000000..6368829df --- /dev/null +++ b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs @@ -0,0 +1,1011 @@ +namespace Serval.Shared.Services; + +[TestFixture] +public class ParallelCorpusServiceTests +{ + [Test] + public void AnalyzeTargetQuoteConvention_FileFormatParatext() + { + using var env = new TestEnvironment(); + ParallelCorpusContract parallelCorpus = env.GetCorpora(paratextProject: true).First(); + const string ExpectedTargetName = "typewriter_english"; + + string targetQuotationConvention = env.Processor.AnalyzeTargetQuoteConvention([parallelCorpus]); + + Assert.That(targetQuotationConvention, Is.EqualTo(ExpectedTargetName)); + } + + [Test] + public void AnalyzeTargetQuoteConvention_FileFormatText() + { + using var env = new TestEnvironment(); + ParallelCorpusContract parallelCorpus = env.GetCorpora(paratextProject: false).First(); + + string targetQuotationConvention = env.Processor.AnalyzeTargetQuoteConvention([parallelCorpus]); + + Assert.That(targetQuotationConvention, Is.Empty); + } + + [Test] + public async Task PreprocessAsync_FileFormatText() + { + using var env = new TestEnvironment(); + IReadOnlyList corpora = env.GetCorpora(paratextProject: false); + int trainCount = 0; + int inferenceCount = 0; + await env.Processor.PreprocessAsync( + corpora, + (row, _) => + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + trainCount++; + return Task.CompletedTask; + }, + (row, isInTrainingData, _) => + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + inferenceCount++; + } + + return Task.CompletedTask; + }, + false + ); + + Assert.Multiple(() => + { + Assert.That(trainCount, Is.EqualTo(2)); + Assert.That(inferenceCount, Is.EqualTo(3)); + }); + } + + [Test] + public async Task PreprocessAsync_FileFormatParatext() + { + using var env = new TestEnvironment(); + IReadOnlyList corpora = env.GetCorpora(paratextProject: true); + int trainCount = 0; + int inferenceCount = 0; + var trainRefs = new List(); + var inferenceRefs = new List(); + await env.Processor.PreprocessAsync( + corpora, + (row, _) => + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { + trainCount++; + trainRefs.Add(row.TargetRefs[0].ToString() ?? ""); + } + return Task.CompletedTask; + }, + (row, isInTrainingData, _) => + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + inferenceCount++; + inferenceRefs.Add(row.TargetRefs[0].ToString() ?? ""); + } + + return Task.CompletedTask; + }, + false, + ["mt"] + ); + + Assert.Multiple(() => + { + Assert.That(trainCount, Is.EqualTo(5)); + Assert.That(inferenceCount, Is.EqualTo(17)); + }); + } + + [Test] + public void FindMissingParentProjects() + { + using var env = new TestEnvironment(); + ParallelCorpusContract parallelCorpus = env.GetCorpora(paratextProject: true).First(); + + IReadOnlyList<( + string ParallelCorpusId, + string MonolingualCorpusId, + MissingParentProjectErrorContract Error + )> errors = env.Processor.FindMissingParentProjects([parallelCorpus]); + + Assert.That(errors, Has.Count.EqualTo(0)); + } + + [Test] + public void FindMissingParentProjects_MissingParent() + { + using var env = new TestEnvironment(); + ParallelCorpusContract parallelCorpus = env.GetCorpora(paratextProject: true).Last(); + + IReadOnlyList<( + string ParallelCorpusId, + string MonolingualCorpusId, + MissingParentProjectErrorContract Error + )> errors = env.Processor.FindMissingParentProjects([parallelCorpus]); + + Assert.That(errors, Has.Count.EqualTo(1)); + Assert.That(errors[0].Error.ProjectName, Is.EqualTo("Te2")); + Assert.That(errors[0].Error.ParentProjectName, Is.EqualTo("Te1")); + } + + [Test] + public void AnalyzeUsfmVersification() + { + using var env = new TestEnvironment(); + ParallelCorpusContract parallelCorpus = env.GetCorpora(paratextProject: true).First(); + + IReadOnlyList<( + string ParallelCorpusId, + string MonolingualCorpusId, + IReadOnlyList UsfmErrors + )> errors = env.Processor.AnalyzeUsfmVersification([parallelCorpus]); + + Assert.That(errors, Has.Count.EqualTo(1)); + Assert.That(errors[0].UsfmErrors, Has.Count.EqualTo(3)); + Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(Contracts.UsfmVersificationErrorType.MissingVerse)); + Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(Contracts.UsfmVersificationErrorType.MissingVerse)); + Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(Contracts.UsfmVersificationErrorType.MissingVerse)); + } + + [Test] + public async Task PreprocessAsync_FilterOutEverything() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultTextFileCorpus with { }; + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(0)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_TrainOnAll() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: []); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(4)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_TrainOnTextIds() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus( + trainOnTextIds: ["textId1"], + inferenceTextIds: [] + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(4)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_TrainAndPretranslateAll() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus(trainOnTextIds: null, inferenceTextIds: null); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.That(result.Pretranslations.Count, Is.EqualTo(2)); + } + + [Test] + public async Task PreprocessAsync_PretranslateAll() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus(trainOnTextIds: [], inferenceTextIds: null); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.That(result.Pretranslations.Count, Is.EqualTo(4)); + } + + [Test] + public async Task PreprocessAsync_InferenceTextIds() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus( + inferenceTextIds: ["textId1"], + trainOnTextIds: null + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.That(result.Pretranslations.Count, Is.EqualTo(2)); + } + + [Test] + public async Task PreprocessAsync_InferenceTextIdsOverlapWithTrainOnTextIds() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = TestEnvironment.TextFileCorpus( + inferenceTextIds: ["textId1"], + trainOnTextIds: ["textId1"] + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.Multiple(() => + { + Assert.That(result.GetTrainCount().Source1Count, Is.EqualTo(4)); + Assert.That(result.Pretranslations.Count, Is.EqualTo(2)); + }); + } + + [Test] + public async Task PreprocessAsync_EnableKeyTerms() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultParatextCorpus; + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: true); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(14)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(3652)); + }); + } + + [Test] + public async Task PreprocessAsync_EnableKeyTermsNoTrainingData() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultParatextCorpus; + corpus.SourceCorpora[0].TrainOnTextIds = []; + corpus.TargetCorpora[0].TrainOnTextIds = []; + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: true); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(0)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_DisableKeyTerms() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultParatextCorpus; + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(14)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_InferenceChapters() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.ParatextCorpus( + trainOnChapters: [], + inferenceChapters: new Dictionary> { { "1CH", [12] } } + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.That(result.Pretranslations.Count, Is.EqualTo(4)); + } + + [Test] + public async Task PreprocessAsync_DoNotPretranslateRemark() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultParatextCorpus; + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + Assert.That(result.Pretranslations.Count, Is.EqualTo(20)); + } + + [Test] + public async Task PreprocessAsync_TrainOnChapters() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.ParatextCorpus( + trainOnChapters: new Dictionary> { { "MAT", [1] } }, + inferenceChapters: [] + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(5)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task PreprocessAsync_MixedSource_Paratext() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultMixedSourceParatextCorpus; + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(7)); + Assert.That(src2Count, Is.EqualTo(14)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + Assert.That(result.Pretranslations.Count, Is.EqualTo(21)); + } + + [Test] + public async Task PreprocessAsync_MixedSource_Text() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.DefaultMixedSourceTextFileCorpus; + + PreprocessResult result = await env.RunPreprocessAsync([corpus]); + + (int src1Count, int src2Count, int trgCount, int termCount) = result.GetTrainCount(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(1)); + Assert.That(src2Count, Is.EqualTo(4)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + Assert.That(result.Pretranslations.Count, Is.EqualTo(3)); + } + + [Test] + public async Task PreprocessAsync_RemoveFreestandingEllipses() + { + using var env = new TestEnvironment(); + ParallelCorpusContract corpus = env.ParatextCorpus( + trainOnChapters: new Dictionary> { { "MAT", [2] } }, + inferenceChapters: new Dictionary> { { "MAT", [2] } } + ); + + PreprocessResult result = await env.RunPreprocessAsync([corpus], useKeyTerms: false); + + string sourceExtract = result.GetSourceExtract(); + Assert.That( + sourceExtract, + Is.EqualTo( + "Source one, chapter two, verse one.\nSource one, chapter two, verse two. \u201ca quotation\u201d\n\n" + ), + sourceExtract + ); + string targetExtract = result.GetTargetExtract(); + Assert.That( + targetExtract, + Is.EqualTo( + "Target one, chapter two, verse one.\n\nTarget one, chapter two, verse three. \"a quotation\"\n" + ), + targetExtract + ); + Assert.That(result.Pretranslations.Count, Is.EqualTo(1)); + } + + [Test] + public async Task PreprocessAsync_ParallelCorpus() + { + using var env = new TestEnvironment(); + List corpora = + [ + new ParallelCorpusContract() + { + Id = "1", + SourceCorpora = + [ + new() + { + Id = "_1", + Language = "en", + Files = [env.ParatextFile("pt-source1")], + TrainOnChapters = new() { { "MAT", [1] }, { "LEV", [] } }, + InferenceChapters = new() { { "1CH", [] } }, + }, + new() + { + Id = "_1", + Language = "en", + Files = [env.ParatextFile("pt-source2")], + TrainOnChapters = new() { { "MAT", [1] }, { "MRK", [] } }, + InferenceChapters = new() { { "1CH", [] } }, + }, + ], + TargetCorpora = + [ + new() + { + Id = "_1", + Language = "en", + Files = [env.ParatextFile("pt-target1")], + TrainOnChapters = new() { { "MAT", [1] }, { "MRK", [] } }, + }, + new() + { + Id = "_2", + Language = "en", + Files = [env.ParatextFile("pt-target2")], + TrainOnChapters = new() + { + { "MAT", [1] }, + { "MRK", [] }, + { "LEV", [] }, + }, + }, + ], + }, + ]; + + PreprocessResult result = await env.RunPreprocessAsync(corpora, useKeyTerms: false); + + Assert.Multiple(() => + { + string src = result.GetSourceExtract(); + Assert.That( + src, + Is.EqualTo( + @"Source one, chapter fourteen, verse fifty-five. Segment b. +Source one, chapter fourteen, verse fifty-six. +Source one, chapter one, verse one. +Source one, chapter one, verse two and three. +Source one, chapter one, verse four. +Source one, chapter one, verse five. Source two, chapter one, verse six. +Source two, chapter one, verse seven. Source two, chapter one, verse eight. +Source two, chapter one, verse nine. Source one, chapter one, verse ten. +Source two, chapter one, verse one. +" + ) + .IgnoreLineEndings(), + src + ); + string trg = result.GetTargetExtract(); + Assert.That( + trg, + Is.EqualTo( + @"Target two, chapter fourteen, verse fifty-five. +Target two, chapter fourteen, verse fifty-six. +Target one, chapter one, verse one. +Target one, chapter one, verse two. Target one, chapter one, verse three. + +Target one, chapter one, verse five and six. +Target one, chapter one, verse seven and eight. +Target one, chapter one, verse nine and ten. + +" + ) + .IgnoreLineEndings(), + trg + ); + Assert.That(result.Pretranslations.Count, Is.EqualTo(7)); + Assert.That(result.Pretranslations[2].Translation, Is.EqualTo("Source one, chapter twelve, verse one.")); + }); + } + + private record PretranslationEntry(string CorpusId, string TextId, IReadOnlyList Refs, string Translation); + + private class PreprocessResult + { + public List SourceLines { get; } = []; + public List TargetLines { get; } = []; + public int TermCount { get; set; } + public List Pretranslations { get; } = []; + + public (int Source1Count, int Source2Count, int TargetOnlyCount, int TermCount) GetTrainCount() + { + int src1 = 0, + src2 = 0, + trgOnly = 0; + foreach (string line in SourceLines) + { + string trimmed = line.Trim(); + if (trimmed.StartsWith("Source one")) + src1++; + else if (trimmed.StartsWith("Source two")) + src2++; + else if (trimmed.Length == 0) + trgOnly++; + } + return (src1, src2, trgOnly, TermCount); + } + + public string GetSourceExtract() + { + return string.Join("\n", SourceLines) + "\n"; + } + + public string GetTargetExtract() + { + return string.Join("\n", TargetLines) + "\n"; + } + } + + private class TestEnvironment : DisposableBase + { + private static readonly string TestDataPath = Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "Services", + "data" + ); + private readonly TempDirectory _tempDir = new TempDirectory(name: "ParallelCorpusServiceTests"); + + public IParallelCorpusService Processor { get; } = new ParallelCorpusService(); + + public ParallelCorpusContract DefaultTextFileCorpus { get; } = + new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [TextFile("source1")], + TrainOnTextIds = [], + InferenceTextIds = [], + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [TextFile("target1")], + TrainOnTextIds = [], + }, + ], + }; + + public ParallelCorpusContract DefaultMixedSourceTextFileCorpus { get; } = + new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [TextFile("source1"), TextFile("source2")], + TrainOnTextIds = null, + TrainOnChapters = null, + InferenceTextIds = null, + InferenceChapters = null, + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [TextFile("target1")], + TrainOnChapters = null, + TrainOnTextIds = null, + }, + ], + }; + + public ParallelCorpusContract DefaultParatextCorpus => + new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [ParatextFile("pt-source1")], + TrainOnTextIds = null, + InferenceTextIds = null, + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [ParatextFile("pt-target1")], + TrainOnTextIds = null, + }, + ], + }; + + public ParallelCorpusContract DefaultMixedSourceParatextCorpus => + new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [ParatextFile("pt-source1")], + TrainOnTextIds = null, + InferenceTextIds = null, + }, + new() + { + Id = "src_1", + Language = "es", + Files = [ParatextFile("pt-source2")], + TrainOnTextIds = null, + InferenceTextIds = null, + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [ParatextFile("pt-target1")], + TrainOnTextIds = null, + }, + ], + }; + + public static ParallelCorpusContract TextFileCorpus( + HashSet? trainOnTextIds, + HashSet? inferenceTextIds + ) + { + return new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [TextFile("source1")], + TrainOnTextIds = trainOnTextIds, + InferenceTextIds = inferenceTextIds, + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [TextFile("target1")], + TrainOnTextIds = trainOnTextIds, + }, + ], + }; + } + + public ParallelCorpusContract ParatextCorpus( + Dictionary>? trainOnChapters, + Dictionary>? inferenceChapters + ) + { + return new() + { + Id = "corpusId1", + SourceCorpora = + [ + new() + { + Id = "src_1", + Language = "es", + Files = [ParatextFile("pt-source1")], + TrainOnChapters = trainOnChapters, + InferenceChapters = inferenceChapters, + }, + ], + TargetCorpora = + [ + new() + { + Id = "trg_1", + Language = "en", + Files = [ParatextFile("pt-target1")], + TrainOnChapters = trainOnChapters, + }, + ], + }; + } + + public async Task RunPreprocessAsync( + IEnumerable corpora, + bool useKeyTerms = true, + HashSet? ignoreUsfmMarkers = null + ) + { + var result = new PreprocessResult(); + await Processor.PreprocessAsync( + corpora, + (row, dataType) => + { + if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0) + { + if (dataType == TrainingDataType.KeyTerm) + { + result.TermCount++; + } + else + { + result.SourceLines.Add(row.SourceSegment); + result.TargetLines.Add(row.TargetSegment); + } + } + return Task.CompletedTask; + }, + (row, isInTrainingData, corpusId) => + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + result.Pretranslations.Add( + new PretranslationEntry(corpusId, row.TextId, row.TargetRefs, row.SourceSegment) + ); + } + return Task.CompletedTask; + }, + useKeyTerms, + ignoreUsfmMarkers ?? ["rem", "r"] + ); + return result; + } + + public ParallelCorpusContract[] GetCorpora(bool paratextProject) + { + if (paratextProject) + { + return + [ + new ParallelCorpusContract + { + Id = "corpus1", + SourceCorpora = + [ + new MonolingualCorpusContract + { + Id = "pt-source1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-source1"), + }, + ], + InferenceTextIds = [], + }, + ], + TargetCorpora = + [ + new MonolingualCorpusContract + { + Id = "pt-target1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-target1"), + }, + ], + }, + ], + }, + new ParallelCorpusContract + { + Id = "corpus2", + SourceCorpora = + [ + new MonolingualCorpusContract + { + Id = "pt-source1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-source1"), + }, + ], + TrainOnTextIds = [], + }, + ], + TargetCorpora = + [ + new MonolingualCorpusContract + { + Id = "pt-target1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-target1"), + }, + ], + TrainOnTextIds = [], + }, + ], + }, + new ParallelCorpusContract + { + Id = "corpus3", + SourceCorpora = [], + TargetCorpora = + [ + new MonolingualCorpusContract + { + Id = "pt-target2", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-target1"), + }, + ], + TrainOnTextIds = [], + }, + ], + }, + ]; + } + + return + [ + new ParallelCorpusContract + { + Id = "corpus1", + SourceCorpora = + [ + new MonolingualCorpusContract + { + Id = "source-corpus1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Text, + Location = Path.Combine(TestDataPath, "source1.txt"), + }, + ], + }, + new MonolingualCorpusContract + { + Id = "source-corpus2", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Text, + Location = Path.Combine(TestDataPath, "source2.txt"), + }, + ], + }, + ], + TargetCorpora = + [ + new MonolingualCorpusContract + { + Id = "target-corpus1", + Language = "en", + Files = + [ + new CorpusFileContract + { + TextId = "textId1", + Format = FileFormat.Text, + Location = Path.Combine(TestDataPath, "target1.txt"), + }, + ], + }, + ], + }, + ]; + } + + public CorpusFileContract ParatextFile(string name) + { + return new() + { + TextId = name, + Format = FileFormat.Paratext, + Location = ZipParatextProject(name), + }; + } + + private static CorpusFileContract TextFile(string name) + { + return new() + { + TextId = "textId1", + Format = FileFormat.Text, + Location = Path.Combine(TestDataPath, $"{name}.txt"), + }; + } + + protected override void DisposeManagedResources() + { + _tempDir.Dispose(); + } + + private string ZipParatextProject(string name) + { + string fileName = Path.Combine(_tempDir.Path, $"{name}.zip"); + if (!File.Exists(fileName)) + ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), fileName); + return fileName; + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/Settings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/Settings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/TermRenderings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/TermRenderings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/custom.vrs similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source1/custom.vrs diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/Settings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/Settings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/TermRenderings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/TermRenderings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/custom.vrs similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-source2/custom.vrs diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/Settings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/Settings.xml similarity index 100% rename from src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/Settings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/Settings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/TermRenderings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/TermRenderings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/custom.vrs similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target1/custom.vrs diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/04LEVTe4.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/04LEVTe4.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/04LEVTe4.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/04LEVTe4.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/41MATTe4.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/41MATTe4.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/41MATTe4.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/41MATTe4.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/42MRKTe4.SFM b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/42MRKTe4.SFM similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/42MRKTe4.SFM rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/42MRKTe4.SFM diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/Settings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/Settings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/Settings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/Settings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/TermRenderings.xml b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/TermRenderings.xml similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/TermRenderings.xml rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/TermRenderings.xml diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/custom.vrs b/src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/custom.vrs similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target2/custom.vrs rename to src/Serval/test/Serval.Shared.Tests/Services/data/pt-target2/custom.vrs diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt b/src/Serval/test/Serval.Shared.Tests/Services/data/source1.txt similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt rename to src/Serval/test/Serval.Shared.Tests/Services/data/source1.txt diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt b/src/Serval/test/Serval.Shared.Tests/Services/data/source2.txt similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt rename to src/Serval/test/Serval.Shared.Tests/Services/data/source2.txt diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt b/src/Serval/test/Serval.Shared.Tests/Services/data/target1.txt similarity index 100% rename from src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt rename to src/Serval/test/Serval.Shared.Tests/Services/data/target1.txt diff --git a/src/Serval/test/Serval.Shared.Tests/Usings.cs b/src/Serval/test/Serval.Shared.Tests/Usings.cs index 86da99b25..16611e992 100644 --- a/src/Serval/test/Serval.Shared.Tests/Usings.cs +++ b/src/Serval/test/Serval.Shared.Tests/Usings.cs @@ -1,4 +1,9 @@ +global using System.IO.Compression; global using System.Text.Json; global using NUnit.Framework; global using NUnit.Framework.Constraints; -global using SIL.ServiceToolkit.Utils; +global using Serval.Shared.Contracts; +global using Serval.Shared.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; diff --git a/src/Serval/test/Serval.Shared.Tests/Utils/ArgEx.cs b/src/Serval/test/Serval.Shared.Tests/Utils/ArgEx.cs new file mode 100644 index 000000000..3ecaa6e4d --- /dev/null +++ b/src/Serval/test/Serval.Shared.Tests/Utils/ArgEx.cs @@ -0,0 +1,11 @@ +using NSubstitute.Core.Arguments; + +namespace Serval.Shared.Utils; + +public static class ArgEx +{ + public static ref T IsEquivalentTo(T value) + { + return ref ArgumentMatcher.Enqueue(new DeepEqualArgumentMatcher(value))!; + } +} diff --git a/src/Serval/test/Serval.Shared.Tests/Utils/DeepEqualArgumentMatcher.cs b/src/Serval/test/Serval.Shared.Tests/Utils/DeepEqualArgumentMatcher.cs new file mode 100644 index 000000000..050b44c52 --- /dev/null +++ b/src/Serval/test/Serval.Shared.Tests/Utils/DeepEqualArgumentMatcher.cs @@ -0,0 +1,23 @@ +using DeepEqual.Syntax; +using NSubstitute.Core; +using NSubstitute.Core.Arguments; + +namespace Serval.Shared.Utils; + +public class DeepEqualArgumentMatcher(T value) : IArgumentMatcher, IDescribeNonMatches +{ + public bool IsSatisfiedBy(T? argument) => argument.IsDeepEqual(value); + + public string DescribeFor(object? argument) + { + try + { + argument.ShouldDeepEqual(value); + return string.Empty; + } + catch (DeepEqualException ex) + { + return ex.Message; + } + } +} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/IgnoreLineEndingsStringComparer.cs b/src/Serval/test/Serval.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs similarity index 93% rename from src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/IgnoreLineEndingsStringComparer.cs rename to src/Serval/test/Serval.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs index 0ac35307b..898a9c0d5 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/IgnoreLineEndingsStringComparer.cs +++ b/src/Serval/test/Serval.Shared.Tests/Utils/IgnoreLineEndingsStringComparer.cs @@ -1,4 +1,4 @@ -namespace SIL.ServiceToolkit.Utils; +namespace Serval.Shared.Utils; public sealed class IgnoreLineEndingsStringComparer : StringComparer { diff --git a/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs index f18a07068..da61e0d07 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs @@ -85,13 +85,13 @@ public async Task GetNextFinishedBuildAsync_Insert() Id = BUILD1_ID, EngineRef = "engine1", Owner = "user1", - State = Shared.Contracts.JobState.Completed, + State = JobState.Completed, DateFinished = DateTime.UtcNow, }; await builds.InsertAsync(build); EntityChange change = await task; Assert.That(change.Type, Is.EqualTo(EntityChangeType.Insert)); - Assert.That(change.Entity?.State, Is.EqualTo(Shared.Contracts.JobState.Completed)); + Assert.That(change.Entity?.State, Is.EqualTo(JobState.Completed)); } [Test] @@ -111,13 +111,13 @@ await builds.UpdateAsync( build, u => { - u.Set(b => b.State, Shared.Contracts.JobState.Completed); + u.Set(b => b.State, JobState.Completed); u.Set(b => b.DateFinished, DateTime.UtcNow); } ); EntityChange change = await task; Assert.That(change.Type, Is.EqualTo(EntityChangeType.Update)); - Assert.That(change.Entity?.State, Is.EqualTo(Shared.Contracts.JobState.Completed)); + Assert.That(change.Entity?.State, Is.EqualTo(JobState.Completed)); } [Test] diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs deleted file mode 100644 index 98dd9c46e..000000000 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ /dev/null @@ -1,2896 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using MassTransit.Mediator; -using Serval.Translation.Configuration; -using Serval.Translation.V1; - -namespace Serval.Translation.Services; - -[TestFixture] -public class EngineServiceTests -{ - const string BUILD1_ID = "b00000000000000000000001"; - - [Test] - public void TranslateAsync_EngineDoesNotExist() - { - var env = new TestEnvironment(); - Assert.ThrowsAsync(() => env.Service.TranslateAsync("engine1", "esto es una prueba.")); - } - - [Test] - public async Task TranslateAsync_EngineExists() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - Models.TranslationResult? result = await env.Service.TranslateAsync(engineId, "esto es una prueba."); - Assert.That(result, Is.Not.Null); - Assert.That(result!.Translation, Is.EqualTo("this is a test.")); - } - - [Test] - public void GetWordGraphAsync_EngineDoesNotExist() - { - var env = new TestEnvironment(); - Assert.ThrowsAsync(() => - env.Service.GetWordGraphAsync("engine1", "esto es una prueba.") - ); - } - - [Test] - public async Task GetWordGraphAsync_EngineExists() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - Models.WordGraph? result = await env.Service.GetWordGraphAsync(engineId, "esto es una prueba."); - Assert.That(result, Is.Not.Null); - Assert.That(result!.Arcs.SelectMany(a => a.TargetTokens), Is.EqualTo("this is a test .".Split())); - } - - [Test] - public void TrainSegmentAsync_EngineDoesNotExist() - { - var env = new TestEnvironment(); - Assert.ThrowsAsync(() => - env.Service.TrainSegmentPairAsync("engine1", "esto es una prueba.", "this is a test.", true) - ); - } - - [Test] - public async Task TrainSegmentAsync_EngineExists() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - Assert.DoesNotThrowAsync(() => - env.Service.TrainSegmentPairAsync(engineId, "esto es una prueba.", "this is a test.", true) - ); - } - - [Test] - public async Task CreateAsync() - { - var env = new TestEnvironment(); - Engine engine = new() - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - Corpora = [], - }; - await env.Service.CreateAsync(engine); - - engine = (await env.Engines.GetAsync("engine1"))!; - Assert.That(engine.SourceLanguage, Is.EqualTo("es")); - Assert.That(engine.TargetLanguage, Is.EqualTo("en")); - } - - [Test] - public async Task DeleteAsync_EngineExists() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.DeleteAsync("engine1"); - Engine? engine = await env.Engines.GetAsync(engineId); - Assert.That(engine, Is.Null); - } - - [Test] - public async Task DeleteAsync_ProjectDoesNotExist() - { - var env = new TestEnvironment(); - await env.CreateEngineWithTextFilesAsync(); - Assert.ThrowsAsync(() => env.Service.DeleteAsync("engine3")); - } - - [Test] - public async Task StartBuildAsync_TrainOnNotSpecified() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextIdsEmpty() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1", TextIds = [] }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - TrainOnTextIds = { }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - TrainOnTextIds = { }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextIdsPopulated() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1", TextIds = ["text1"] }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - TrainOnTextIds = { "text1" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - TrainOnTextIds = { "text1" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextIdsNotSpecified() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_OneOfMultipleCorpora() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateMultipleCorporaEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1" }], - Pretranslate = [new PretranslateCorpus { CorpusRef = "corpus1" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TrainOnOnePretranslateTheOther() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateMultipleCorporaEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1" }], - Pretranslate = [new PretranslateCorpus { CorpusRef = "corpus2" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = false, - TrainOnAll = true, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = false, - TrainOnAll = true, - }, - }, - }, - }, - new V1.ParallelCorpus - { - Id = "corpus2", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus2", - - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus2", - - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextFilesScriptureRangeSpecified() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - Assert.ThrowsAsync(() => - env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1", ScriptureRange = "MAT" }], - } - ) - ); - } - - [Test] - public async Task StartBuildAsync_ScriptureRangeSpecified() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1", ScriptureRange = "MAT 1;MRK" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { } } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { } } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_ScriptureRangeEmptyString() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { CorpusRef = "corpus1", ScriptureRange = "" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "corpus1", - - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_ParallelCorpus_TextFiles() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = [new() { CorpusRef = "parallel-corpus1-source1", TextIds = ["MAT"] }], - TargetFilters = [new() { CorpusRef = "parallel-corpus1-target1", TextIds = ["MAT"] }], - }, - ], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-source1", - Language = "es", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-target1", - Language = "en", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_ParallelCorpus_OneOfMultipleCorpora() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateMultipleParallelCorpusEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = [new() { CorpusRef = "parallel-corpus1-source1", TextIds = ["MAT"] }], - TargetFilters = [new() { CorpusRef = "parallel-corpus1-target1", TextIds = ["MAT"] }], - }, - ], - Pretranslate = [new PretranslateCorpus { ParallelCorpusRef = "parallel-corpus1" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-source1", - Language = "es", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-target1", - Language = "en", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_ParallelCorpus_TrainOnOnePretranslateTheOther() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateMultipleParallelCorpusEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = [new() { CorpusRef = "parallel-corpus1-source1", TextIds = ["MAT"] }], - TargetFilters = [new() { CorpusRef = "parallel-corpus1-target1", TextIds = ["MAT"] }], - }, - ], - Pretranslate = [new PretranslateCorpus { ParallelCorpusRef = "parallel-corpus2" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-source1", - Language = "es", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = false, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "parallel-corpus1-target1", - Language = "en", - TrainOnTextIds = { "MAT" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - PretranslateAll = false, - TrainOnAll = false, - }, - }, - }, - }, - new V1.ParallelCorpus - { - Id = "parallel-corpus2", - SourceCorpora = - { - new List - { - new() - { - Id = "parallel-corpus2-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List - { - new() - { - Id = "parallel-corpus2-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextIds_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = [new() { CorpusRef = "parallel-corpus1-source1", TextIds = ["MAT", "MRK"] }], - TargetFilters = [new() { CorpusRef = "parallel-corpus1-target1", TextIds = ["MAT", "MRK"] }], - }, - ], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new List() - { - new() - { - Id = "parallel-corpus1-source1", - Language = "es", - TrainOnTextIds = { "MAT", "MRK" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List() - { - new() - { - Id = "parallel-corpus1-target1", - Language = "en", - TrainOnTextIds = { "MAT", "MRK" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_ScriptureRange_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = - [ - new() { CorpusRef = "parallel-corpus1-source1", ScriptureRange = "MAT 1;MRK" }, - ], - TargetFilters = - [ - new() { CorpusRef = "parallel-corpus1-target1", ScriptureRange = "MAT 1;MRK" }, - ], - }, - ], - Pretranslate = - [ - new PretranslateCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = [new() { CorpusRef = "parallel-corpus1-source1", ScriptureRange = "MAT 2" }], - }, - ], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new List() - { - new() - { - Id = "parallel-corpus1-source1", - Language = "es", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { } } - }, - }, - PretranslateChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 2 } } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = false, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - PretranslateAll = false, - TrainOnAll = false, - }, - }, - }, - TargetCorpora = - { - new List() - { - new() - { - Id = "parallel-corpus1-target1", - Language = "en", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_MixedSourceAndTarget_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = - [ - new() { CorpusRef = "parallel-corpus1-source1", ScriptureRange = "MAT 1-2;MRK 1-2" }, - new() { CorpusRef = "parallel-corpus1-source2", ScriptureRange = "MAT 3;MRK 1" }, - ], - TargetFilters = - [ - new() { CorpusRef = "parallel-corpus1-target1", ScriptureRange = "MAT 2-3;MRK 2" }, - new() { CorpusRef = "parallel-corpus1-target2", ScriptureRange = "MAT 1;MRK 1-2" }, - ], - }, - ], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1, 2 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1, 2 } } - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 3 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1 } } - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - TargetCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 2, 3 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 2 } } - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1, 2 } } - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TextFilesScriptureRangeSpecified_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - Assert.ThrowsAsync(() => - env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = - [ - new() - { - CorpusRef = "parallel-corpus1-source1", - ScriptureRange = "MAT", - TextIds = [], - }, - ], - TargetFilters = - [ - new() - { - CorpusRef = "parallel-corpus1-target1", - ScriptureRange = "MAT", - TextIds = [], - }, - ], - }, - ], - } - ) - ); - } - - [Test] - public async Task StartBuildAsync_NoFilters_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = [new TrainingCorpus { ParallelCorpusRef = "parallel-corpus1" }], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - TargetCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - TargetCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task StartBuildAsync_NoTargetFilter_ParallelCorpus() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build - { - Id = BUILD1_ID, - EngineRef = engineId, - Owner = "owner1", - TrainOn = - [ - new TrainingCorpus - { - ParallelCorpusRef = "parallel-corpus1", - SourceFilters = - [ - new() { CorpusRef = "parallel-corpus1-source1", ScriptureRange = "MAT 1;MRK" }, - ], - }, - ], - } - ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, - engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Smt", - Corpora = - { - new V1.ParallelCorpus - { - Id = "parallel-corpus1", - SourceCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { } - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-source2", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = false, - }, - }, - TargetCorpora = - { - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "parallel-corpus1-target2", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, - }, - PretranslateAll = true, - TrainOnAll = true, - }, - }, - }, - }, - }, - cancellationToken: Arg.Any() - ); - } - - [Test] - public async Task CancelBuildAsync_EngineExistsNotBuilding() - { - var env = new TestEnvironment(); - string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.CancelBuildAsync(engineId); - } - - [Test] - public async Task UpdateCorpusAsync() - { - var env = new TestEnvironment(); - Engine engine = await env.CreateEngineWithTextFilesAsync(); - string corpusId = engine.Corpora[0].Id; - - Models.Corpus? corpus = await env.Service.UpdateCorpusAsync( - engine.Id, - corpusId, - sourceFiles: - [ - new() - { - Id = "file1", - Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - new() - { - Id = "file3", - Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text2", - }, - ], - null - ); - - Assert.That(corpus, Is.Not.Null); - Assert.That(corpus.SourceFiles, Has.Count.EqualTo(2)); - Assert.That(corpus.SourceFiles[0].Id, Is.EqualTo("file1")); - Assert.That(corpus.SourceFiles[1].Id, Is.EqualTo("file3")); - Assert.That(corpus.TargetFiles, Has.Count.EqualTo(1)); - } - - [Test] - public async Task UpdateAsync_ShouldUpdateLanguages_WhenRequestIsValid() - { - var env = new TestEnvironment(); - var engine = await env.CreateEngineWithTextFilesAsync(); - - var request = new TranslationEngineUpdateConfigDto { SourceLanguage = "en", TargetLanguage = "fr" }; - - await env.Service.UpdateAsync(engine.Id, request.SourceLanguage, request.TargetLanguage); - - engine = await env.Engines.GetAsync(engine.Id); - - Assert.That(engine, Is.Not.Null); - Assert.That(engine.SourceLanguage, Is.Not.Null); - Assert.That(engine.SourceLanguage, Is.EqualTo("en")); - Assert.That(engine.TargetLanguage, Is.Not.Null); - Assert.That(engine.TargetLanguage, Is.EqualTo("fr")); - } - - [Test] - public async Task UpdateAsync_ShouldNotUpdateSourceLanguage_WhenSourceLanguageNotProvided() - { - var env = new TestEnvironment(); - Engine engine = await env.CreateEngineWithTextFilesAsync(); - - await env.Service.UpdateAsync(engine.Id, sourceLanguage: null, targetLanguage: "fr"); - - Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); - - Assert.That(updatedEngine, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.EqualTo(engine.SourceLanguage)); - Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); - Assert.That(updatedEngine.TargetLanguage, Is.EqualTo("fr")); - } - - [Test] - public async Task UpdateAsync_ShouldNotUpdateTargetLanguage_WhenTargetLanguageNotProvided() - { - var env = new TestEnvironment(); - Engine engine = await env.CreateEngineWithTextFilesAsync(); - - await env.Service.UpdateAsync(engine.Id, sourceLanguage: "en", targetLanguage: null); - - Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); - - Assert.That(updatedEngine, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.EqualTo("en")); - Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); - Assert.That(updatedEngine.TargetLanguage, Is.EqualTo(engine.TargetLanguage)); - } - - [Test] - public async Task UpdateAsync_ShouldNotUpdate_WhenSourceAndTargetLanguagesNotProvided() - { - var env = new TestEnvironment(); - Engine engine = await env.CreateEngineWithTextFilesAsync(); - - await env.Service.UpdateAsync(engine.Id, sourceLanguage: null, targetLanguage: null); - - Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); - - Assert.That(updatedEngine, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); - Assert.That(updatedEngine.SourceLanguage, Is.EqualTo(engine.SourceLanguage)); - Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); - Assert.That(updatedEngine.TargetLanguage, Is.EqualTo(engine.TargetLanguage)); - } - - [Test] - public async Task DeletePretranslationsWhenCorpusIsUpdatedAsync() - { - var env = new TestEnvironment(); - Pretranslation pretranslation = new() - { - Id = "pretranslation1", - EngineRef = "engine1", - CorpusRef = "corpus1", - Refs = ["ref1"], - SourceRefs = ["ref1"], - TargetRefs = ["ref1"], - TextId = "textId1", - Translation = "translation", - }; - var engine = await env.CreateEngineWithTextFilesAsync(); - await env.Pretranslations.InsertAsync(pretranslation); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); - await env.Service.UpdateCorpusAsync(engine.Id, "corpus1", sourceFiles: [], targetFiles: []); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); - } - - [Test] - public async Task DeletePretranslationsWhenParallelCorpusIsUpdatedAsync() - { - var env = new TestEnvironment(); - Pretranslation pretranslation = new() - { - Id = "pretranslation1", - EngineRef = "engine1", - CorpusRef = "parallel-corpus1", - Refs = ["ref1"], - SourceRefs = ["ref1"], - TargetRefs = ["ref1"], - TextId = "textId1", - Translation = "translation", - }; - var engine = await env.CreateParallelCorpusEngineWithTextFilesAsync(); - await env.Pretranslations.InsertAsync(pretranslation); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); - await env.Service.UpdateParallelCorpusAsync( - engine.Id, - "parallel-corpus1", - sourceCorpora: [], - targetCorpora: [] - ); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); - } - - [Test] - public async Task DeletePretranslationsWhenCorpusFilesAreDeletedAsync() - { - var env = new TestEnvironment(); - Pretranslation pretranslation = new() - { - Id = "pretranslation1", - EngineRef = "engine1", - CorpusRef = "corpus1", - Refs = ["ref1"], - SourceRefs = ["ref1"], - TargetRefs = ["ref1"], - TextId = "textId1", - Translation = "translation", - }; - await env.CreateEngineWithTextFilesAsync(); - await env.Pretranslations.InsertAsync(pretranslation); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); - await env.Service.DeleteAllCorpusFilesAsync("file1"); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); - } - - [Test] - public async Task DeletePretranslationsWhenCorpusFilesAreUpdatedAsync() - { - var env = new TestEnvironment(); - Pretranslation pretranslation = new() - { - Id = "pretranslation1", - EngineRef = "engine1", - CorpusRef = "corpus1", - Refs = ["ref1"], - SourceRefs = ["ref1"], - TargetRefs = ["ref1"], - TextId = "textId1", - Translation = "translation", - }; - await env.CreateEngineWithTextFilesAsync(); - await env.Pretranslations.InsertAsync(pretranslation); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); - await env.Service.UpdateCorpusFilesAsync( - "corpus1", - [ - new() - { - Id = "file1", - Filename = "newfilename", - TextId = "text1", - Format = Shared.Contracts.FileFormat.Text, - }, - ] - ); - Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Engines = new MemoryRepository(); - TranslationServiceClient = Substitute.For(); - var translationResult = new V1.TranslationResult - { - Translation = "this is a test.", - SourceTokens = { "esto es una prueba .".Split() }, - TargetTokens = { "this is a test .".Split() }, - Confidences = { 1.0, 1.0, 1.0, 1.0, 1.0 }, - Sources = - { - new TranslationSources { Values = { V1.TranslationSource.Primary } }, - new TranslationSources { Values = { V1.TranslationSource.Primary } }, - new TranslationSources { Values = { V1.TranslationSource.Primary } }, - new TranslationSources { Values = { V1.TranslationSource.Primary } }, - new TranslationSources { Values = { V1.TranslationSource.Primary } }, - }, - Alignment = - { - new V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - new V1.AlignedWordPair { SourceIndex = 2, TargetIndex = 2 }, - new V1.AlignedWordPair { SourceIndex = 3, TargetIndex = 3 }, - new V1.AlignedWordPair { SourceIndex = 4, TargetIndex = 4 }, - }, - Phrases = - { - new V1.Phrase - { - SourceSegmentStart = 0, - SourceSegmentEnd = 5, - TargetSegmentCut = 5, - }, - }, - }; - var translateResponse = new TranslateResponse { Results = { translationResult } }; - TranslationServiceClient - .TranslateAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(translateResponse)); - var wordGraph = new V1.WordGraph - { - SourceTokens = { "esto es una prueba .".Split() }, - FinalStates = { 3 }, - Arcs = - { - new V1.WordGraphArc - { - PrevState = 0, - NextState = 1, - Score = 1.0, - TargetTokens = { "this is".Split() }, - Alignment = - { - new V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - }, - SourceSegmentStart = 0, - SourceSegmentEnd = 2, - Sources = { GetSources(2, false) }, - Confidences = { 1.0, 1.0 }, - }, - new V1.WordGraphArc - { - PrevState = 1, - NextState = 2, - Score = 1.0, - TargetTokens = { "a test".Split() }, - Alignment = - { - new V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - }, - SourceSegmentStart = 2, - SourceSegmentEnd = 4, - Sources = { GetSources(2, false) }, - Confidences = { 1.0, 1.0 }, - }, - new V1.WordGraphArc - { - PrevState = 2, - NextState = 3, - Score = 1.0, - TargetTokens = { ".".Split() }, - Alignment = - { - new V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - }, - SourceSegmentStart = 4, - SourceSegmentEnd = 5, - Sources = { GetSources(1, false) }, - Confidences = { 1.0 }, - }, - }, - }; - var getWordGraphResponse = new GetWordGraphResponse { WordGraph = wordGraph }; - TranslationServiceClient - .GetWordGraphAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(getWordGraphResponse)); - TranslationServiceClient - .CancelBuildAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - TranslationServiceClient.CreateAsync(Arg.Any()).Returns(CreateAsyncUnaryCall(new Empty())); - TranslationServiceClient.DeleteAsync(Arg.Any()).Returns(CreateAsyncUnaryCall(new Empty())); - TranslationServiceClient - .StartBuildAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - TranslationServiceClient.UpdateAsync(Arg.Any()).Returns(CreateAsyncUnaryCall(new Empty())); - TranslationServiceClient - .TrainSegmentPairAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("Smt") - .Returns(TranslationServiceClient); - IOptionsMonitor dataFileOptions = Substitute.For>(); - dataFileOptions.CurrentValue.Returns(new DataFileOptions()); - - Pretranslations = new MemoryRepository(); - OutboxService = Substitute.For(); - IOptionsMonitor translationOptions = Substitute.For< - IOptionsMonitor - >(); - translationOptions.CurrentValue.Returns( - new TranslationOptions { Engines = [new EngineInfo { Type = "Smt" }] } - ); - var parallelCorpusService = Substitute.For(); - parallelCorpusService - .GetChapters( - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(callInfo => - { - return ScriptureRangeParser.GetChapters(callInfo.ArgAt(2)); - }); - - Service = new EngineService( - Engines, - new MemoryRepository(), - Pretranslations, - Substitute.For(), - grpcClientFactory, - new MemoryDataAccessContext(), - new LoggerFactory(), - OutboxService, - translationOptions, - new CorpusMappingService(dataFileOptions, parallelCorpusService) - ); - } - - public EngineService Service { get; } - public IRepository Engines { get; } - public IRepository Pretranslations { get; } - public TranslationEngineApi.TranslationEngineApiClient TranslationServiceClient { get; } - public IOutboxService OutboxService { get; } - - public async Task CreateEngineWithTextFilesAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - Corpora = - [ - new() - { - Id = "corpus1", - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = - [ - new() - { - Id = "file1", - Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - TargetFiles = - [ - new() - { - Id = "file2", - Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - }, - ], - ModelRevision = 1, - }; - await Engines.InsertAsync(engine); - return engine; - } - - public async Task CreateMultipleCorporaEngineWithTextFilesAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - Corpora = - [ - new() - { - Id = "corpus1", - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = - [ - new() - { - Id = "file1", - Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - TargetFiles = - [ - new() - { - Id = "file2", - Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - }, - new() - { - Id = "corpus2", - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = - [ - new() - { - Id = "file3", - Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - TargetFiles = - [ - new() - { - Id = "file4", - Filename = "file4.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "text1", - }, - ], - }, - ], - }; - await Engines.InsertAsync(engine); - return engine; - } - - public async Task CreateEngineWithParatextProjectAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - Corpora = - [ - new() - { - Id = "corpus1", - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = - [ - new() - { - Id = "file1", - Filename = "file1.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file1.zip", - }, - ], - TargetFiles = - [ - new() - { - Id = "file2", - Filename = "file2.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file2.zip", - }, - ], - }, - ], - }; - await Engines.InsertAsync(engine); - return engine; - } - - public async Task CreateParallelCorpusEngineWithTextFilesAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - ParallelCorpora = - [ - new() - { - Id = "parallel-corpus1", - SourceCorpora = - [ - new() - { - Id = "parallel-corpus1-source1", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file1", - Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MAT", - }, - ], - }, - new() - { - Id = "parallel-corpus1-source2", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file3", - Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MRK", - }, - ], - }, - ], - TargetCorpora = new List() - { - new() - { - Id = "parallel-corpus1-target1", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file2", - Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MAT", - }, - ], - }, - new() - { - Id = "parallel-corpus1-target2", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file4", - Filename = "file4.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MRK", - }, - ], - }, - }, - }, - ], - }; - await Engines.InsertAsync(engine); - return engine; - } - - public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - ParallelCorpora = - [ - new() - { - Id = "parallel-corpus1", - SourceCorpora = new List() - { - new() - { - Id = "parallel-corpus1-source1", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file1", - Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MAT", - }, - ], - }, - }, - TargetCorpora = new List() - { - new() - { - Id = "parallel-corpus1-target1", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file2", - Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MAT", - }, - ], - }, - }, - }, - new() - { - Id = "parallel-corpus2", - SourceCorpora = new List() - { - new() - { - Id = "parallel-corpus2-source1", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file3", - Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MRK", - }, - ], - }, - }, - TargetCorpora = new List() - { - new() - { - Id = "parallel-corpus2-target1", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file4", - Filename = "file4.txt", - Format = Shared.Contracts.FileFormat.Text, - TextId = "MRK", - }, - ], - }, - }, - }, - ], - }; - await Engines.InsertAsync(engine); - return engine; - } - - public async Task CreateParallelCorpusEngineWithParatextProjectAsync() - { - var engine = new Engine - { - Id = "engine1", - Owner = "owner1", - SourceLanguage = "es", - TargetLanguage = "en", - Type = "Smt", - ParallelCorpora = - [ - new() - { - Id = "parallel-corpus1", - SourceCorpora = new List() - { - new() - { - Id = "parallel-corpus1-source1", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file1", - Filename = "file1.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file1.zip", - }, - ], - }, - new() - { - Id = "parallel-corpus1-source2", - Name = "", - Language = "es", - Files = - [ - new() - { - Id = "file3", - Filename = "file3.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file3.zip", - }, - ], - }, - }, - TargetCorpora = new List() - { - new() - { - Id = "parallel-corpus1-target1", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file2", - Filename = "file2.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file2.zip", - }, - ], - }, - new() - { - Id = "parallel-corpus1-target2", - Name = "", - Language = "en", - Files = - [ - new() - { - Id = "file4", - Filename = "file4.zip", - Format = Shared.Contracts.FileFormat.Paratext, - TextId = "file4.zip", - }, - ], - }, - }, - }, - ], - }; - await Engines.InsertAsync(engine); - return engine; - } - - private static TranslationSources[] GetSources(int count, bool isUnknown) - { - var sources = new TranslationSources[count]; - for (int i = 0; i < count; i++) - { - sources[i] = new TranslationSources(); - if (!isUnknown) - sources[i].Values.Add(V1.TranslationSource.Primary); - } - return sources; - } - - private static AsyncUnaryCall CreateAsyncUnaryCall(TResponse response) - { - return new AsyncUnaryCall( - Task.FromResult(response), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - } - } -} diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EnginesFeatureTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EnginesFeatureTests.cs new file mode 100644 index 000000000..896d325b6 --- /dev/null +++ b/src/Serval/test/Serval.Translation.Tests/Services/EnginesFeatureTests.cs @@ -0,0 +1,2804 @@ +namespace Serval.Translation.Services; + +#pragma warning disable CS0612 // Type or member is obsolete + +[TestFixture] +public class EnginesFeatureTests +{ + const string OWNER = "owner1"; + + [Test] + public void Translate_EngineDoesNotExist() + { + var env = new TestEnvironment(); + TranslateHandler handler = new(env.Engines, env.EngineServiceFactory); + Assert.ThrowsAsync(() => + handler.HandleAsync(new Translate(OWNER, "engine1", "esto es una prueba.")) + ); + } + + [Test] + public async Task Translate_EngineExists() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + TranslateHandler handler = new(env.Engines, env.EngineServiceFactory); + TranslateResponse response = await handler.HandleAsync(new Translate(OWNER, engineId, "esto es una prueba.")); + using (Assert.EnterMultipleScope()) + { + Assert.That(response.IsAvailable, Is.True); + Assert.That(response.Results!.First().Translation, Is.EqualTo("this is a test.")); + } + } + + [Test] + public void GetWordGraph_EngineDoesNotExist() + { + var env = new TestEnvironment(); + GetWordGraphHandler handler = new(env.Engines, env.EngineServiceFactory); + Assert.ThrowsAsync(() => + handler.HandleAsync(new GetWordGraph(OWNER, "engine1", "esto es una prueba.")) + ); + } + + [Test] + public async Task GetWordGraph_EngineExists() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + GetWordGraphHandler handler = new(env.Engines, env.EngineServiceFactory); + GetWordGraphResponse response = await handler.HandleAsync( + new GetWordGraph(OWNER, engineId, "esto es una prueba.") + ); + using (Assert.EnterMultipleScope()) + { + Assert.That(response.IsAvailable, Is.True); + Assert.That( + response.WordGraph!.Arcs.SelectMany(a => a.TargetTokens), + Is.EqualTo("this is a test .".Split()) + ); + } + } + + [Test] + public void TrainSegment_EngineDoesNotExist() + { + var env = new TestEnvironment(); + TrainSegmentHandler handler = new(env.Engines, env.EngineServiceFactory); + Assert.ThrowsAsync(() => + handler.HandleAsync( + new TrainSegment( + OWNER, + "engine1", + new SegmentPairDto + { + SourceSegment = "esto es una prueba.", + TargetSegment = "this is a test.", + SentenceStart = true, + } + ) + ) + ); + } + + [Test] + public async Task TrainSegment_EngineExists() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + TrainSegmentHandler handler = new(env.Engines, env.EngineServiceFactory); + Assert.DoesNotThrowAsync(() => + handler.HandleAsync( + new TrainSegment( + OWNER, + engineId, + new SegmentPairDto + { + SourceSegment = "esto es una prueba.", + TargetSegment = "this is a test.", + SentenceStart = true, + } + ) + ) + ); + } + + [Test] + public async Task CreateEngine() + { + var env = new TestEnvironment(); + CreateEngineHandler handler = new(env.DataAccessContext, env.Engines, env.EngineServiceFactory, env.DtoMapper); + CreateEngineResponse response = await handler.HandleAsync( + new CreateEngine( + OWNER, + new TranslationEngineConfigDto + { + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + } + ) + ); + + Engine? engine = await env.Engines.GetAsync(response.Engine.Id); + Assert.That(engine, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(engine.SourceLanguage, Is.EqualTo("es")); + Assert.That(engine.TargetLanguage, Is.EqualTo("en")); + } + } + + [Test] + public async Task DeleteEngine_EngineExists() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + DeleteEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.Pretranslations, + env.EngineServiceFactory + ); + await handler.HandleAsync(new DeleteEngine(OWNER, engineId)); + Engine? engine = await env.Engines.GetAsync(engineId); + Assert.That(engine, Is.Null); + } + + [Test] + public async Task DeleteEngine_ProjectDoesNotExist() + { + var env = new TestEnvironment(); + await env.CreateEngineWithTextFilesAsync(); + DeleteEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.Pretranslations, + env.EngineServiceFactory + ); + Assert.ThrowsAsync(() => handler.HandleAsync(new DeleteEngine(OWNER, "engine3"))); + } + + [Test] + public async Task StartBuild_TrainOnNotSpecified() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild(OWNER, engineId, new TranslationBuildConfigDto()) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextIdsEmpty() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1", TextIds = [] }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextIdsPopulated() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1", TextIds = ["text1"] }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + TrainOnTextIds = ["text1"], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + TrainOnTextIds = ["text1"], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextIdsNotSpecified() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto { TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1" }] } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_OneOfMultipleCorpora() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateMultipleCorporaEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1" }], + Pretranslate = [new PretranslateCorpusConfigDto { CorpusId = "corpus1" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TrainOnOnePretranslateTheOther() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateMultipleCorporaEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1" }], + Pretranslate = [new PretranslateCorpusConfigDto { CorpusId = "corpus2" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + InferenceTextIds = [], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + InferenceTextIds = [], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + new() + { + Id = "corpus2", + SourceCorpora = + [ + new() + { + Id = "corpus2", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file3.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus2", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file4.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextFilesScriptureRangeSpecified() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + Assert.ThrowsAsync(() => + handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1", ScriptureRange = "MAT" }], + } + ) + ) + ); + } + + [Test] + public async Task StartBuild_ScriptureRangeSpecified() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1", ScriptureRange = "MAT 1;MRK" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_ScriptureRangeEmptyString() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { CorpusId = "corpus1", ScriptureRange = "" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "corpus1", + SourceCorpora = + [ + new() + { + Id = "corpus1", + Language = "es", + TrainOnChapters = [], + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "corpus1", + Language = "en", + TrainOnChapters = [], + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_ParallelCorpus_TextFiles() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = [new() { CorpusId = "parallel-corpus1-source1", TextIds = ["MAT"] }], + TargetFilters = [new() { CorpusId = "parallel-corpus1-target1", TextIds = ["MAT"] }], + }, + ], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnTextIds = ["MAT"], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file3.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnTextIds = ["MAT"], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file4.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_ParallelCorpus_OneOfMultipleCorpora() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateMultipleParallelCorpusEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = [new() { CorpusId = "parallel-corpus1-source1", TextIds = ["MAT"] }], + TargetFilters = [new() { CorpusId = "parallel-corpus1-target1", TextIds = ["MAT"] }], + }, + ], + Pretranslate = [new PretranslateCorpusConfigDto { ParallelCorpusId = "parallel-corpus1" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnTextIds = ["MAT"], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnTextIds = ["MAT"], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_ParallelCorpus_TrainOnOnePretranslateTheOther() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateMultipleParallelCorpusEngineWithTextFilesAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = [new() { CorpusId = "parallel-corpus1-source1", TextIds = ["MAT"] }], + TargetFilters = [new() { CorpusId = "parallel-corpus1-target1", TextIds = ["MAT"] }], + }, + ], + Pretranslate = [new PretranslateCorpusConfigDto { ParallelCorpusId = "parallel-corpus2" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnTextIds = ["MAT"], + InferenceTextIds = [], + Files = + [ + new() + { + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnTextIds = ["MAT"], + InferenceTextIds = [], + Files = + [ + new() + { + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + ], + }, + new() + { + Id = "parallel-corpus2", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus2-source1", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file3.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus2-target1", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file4.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextIds_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = [new() { CorpusId = "parallel-corpus1-source1", TextIds = ["MAT", "MRK"] }], + TargetFilters = [new() { CorpusId = "parallel-corpus1-target1", TextIds = ["MAT", "MRK"] }], + }, + ], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnTextIds = ["MAT", "MRK"], + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnTextIds = ["MAT", "MRK"], + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_ScriptureRange_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = + [ + new() { CorpusId = "parallel-corpus1-source1", ScriptureRange = "MAT 1;MRK" }, + ], + TargetFilters = + [ + new() { CorpusId = "parallel-corpus1-target1", ScriptureRange = "MAT 1;MRK" }, + ], + }, + ], + Pretranslate = + [ + new PretranslateCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = [new() { CorpusId = "parallel-corpus1-source1", ScriptureRange = "MAT 2" }], + }, + ], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [] }, + InferenceChapters = new() { ["MAT"] = [2] }, + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + TrainOnTextIds = [], + InferenceTextIds = [], + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_MixedSourceAndTarget_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = + [ + new() { CorpusId = "parallel-corpus1-source1", ScriptureRange = "MAT 1-2;MRK 1-2" }, + new() { CorpusId = "parallel-corpus1-source2", ScriptureRange = "MAT 3;MRK 1" }, + ], + TargetFilters = + [ + new() { CorpusId = "parallel-corpus1-target1", ScriptureRange = "MAT 2-3;MRK 2" }, + new() { CorpusId = "parallel-corpus1-target2", ScriptureRange = "MAT 1;MRK 1-2" }, + ], + }, + ], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnChapters = new() { ["MAT"] = [1, 2], ["MRK"] = [1, 2] }, + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + TrainOnChapters = new() { ["MAT"] = [3], ["MRK"] = [1] }, + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + TrainOnChapters = new() { ["MAT"] = [2, 3], ["MRK"] = [2] }, + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [1, 2] }, + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TextFilesScriptureRangeSpecified_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + Assert.ThrowsAsync(() => + handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = + [ + new() + { + CorpusId = "parallel-corpus1-source1", + ScriptureRange = "MAT", + TextIds = [], + }, + ], + TargetFilters = + [ + new() + { + CorpusId = "parallel-corpus1-target1", + ScriptureRange = "MAT", + TextIds = [], + }, + ], + }, + ], + } + ) + ) + ); + } + + [Test] + public async Task StartBuild_NoFilters_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = [new TrainingCorpusConfigDto { ParallelCorpusId = "parallel-corpus1" }], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_TrainOnNotSpecified_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild(OWNER, engineId, new TranslationBuildConfigDto()) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task StartBuild_NoTargetFilter_ParallelCorpus() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateParallelCorpusEngineWithParatextProjectAsync()).Id; + StartBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.ContractMapper, + env.EngineServiceFactory, + Substitute.For>(), + env.DtoMapper, + Substitute.For() + ); + StartBuildResponse response = await handler.HandleAsync( + new StartBuild( + OWNER, + engineId, + new TranslationBuildConfigDto + { + TrainOn = + [ + new TrainingCorpusConfigDto + { + ParallelCorpusId = "parallel-corpus1", + SourceFilters = + [ + new() { CorpusId = "parallel-corpus1-source1", ScriptureRange = "MAT 1;MRK" }, + ], + }, + ], + } + ) + ); + Assert.That(response.IsBuildRunning, Is.False); + await env + .TranslationEngineService.Received() + .StartBuildAsync( + engineId, + response.Build!.Id, + ArgEx.IsEquivalentTo>([ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Language = "es", + TrainOnChapters = new() { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ + new() + { + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() + { + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + ], + TargetCorpora = + [ + new() + { + Id = "parallel-corpus1-target1", + Language = "en", + Files = + [ + new() + { + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Language = "en", + Files = + [ + new() + { + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + ], + }, + ]), + null, + Arg.Any() + ); + } + + [Test] + public async Task CancelBuildAsync_EngineExistsNotBuilding() + { + var env = new TestEnvironment(); + string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; + CancelBuildHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Builds, + env.EngineServiceFactory, + env.DtoMapper + ); + await handler.HandleAsync(new CancelBuild(OWNER, engineId), CancellationToken.None); + } + + [Test] + public async Task UpdateCorpusAsync() + { + var env = new TestEnvironment(); + Engine engine = await env.CreateEngineWithTextFilesAsync(); + string corpusId = engine.Corpora[0].Id; + + Corpus? corpus = await env.Service.UpdateCorpusAsync( + engine.Id, + corpusId, + sourceFiles: + [ + new() + { + Id = "file1", + Filename = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + new() + { + Id = "file3", + Filename = "file3.txt", + Format = FileFormat.Text, + TextId = "text2", + }, + ], + null + ); + + Assert.That(corpus, Is.Not.Null); + Assert.That(corpus.SourceFiles, Has.Count.EqualTo(2)); + Assert.That(corpus.SourceFiles[0].Id, Is.EqualTo("file1")); + Assert.That(corpus.SourceFiles[1].Id, Is.EqualTo("file3")); + Assert.That(corpus.TargetFiles, Has.Count.EqualTo(1)); + } + + [Test] + public async Task UpdateEngine_ShouldUpdateLanguages_WhenRequestIsValid() + { + var env = new TestEnvironment(); + var engine = await env.CreateEngineWithTextFilesAsync(); + + var request = new TranslationEngineUpdateConfigDto { SourceLanguage = "en", TargetLanguage = "fr" }; + + UpdateEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Pretranslations, + env.EngineServiceFactory + ); + await handler.HandleAsync(new UpdateEngine(OWNER, engine.Id, request), CancellationToken.None); + + engine = await env.Engines.GetAsync(engine.Id); + + Assert.That(engine, Is.Not.Null); + Assert.That(engine.SourceLanguage, Is.Not.Null); + Assert.That(engine.SourceLanguage, Is.EqualTo("en")); + Assert.That(engine.TargetLanguage, Is.Not.Null); + Assert.That(engine.TargetLanguage, Is.EqualTo("fr")); + } + + [Test] + public async Task UpdateEngine_ShouldNotUpdateSourceLanguage_WhenSourceLanguageNotProvided() + { + var env = new TestEnvironment(); + Engine engine = await env.CreateEngineWithTextFilesAsync(); + + UpdateEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Pretranslations, + env.EngineServiceFactory + ); + await handler.HandleAsync( + new UpdateEngine(OWNER, engine.Id, new TranslationEngineUpdateConfigDto { TargetLanguage = "fr" }), + CancellationToken.None + ); + + Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); + + Assert.That(updatedEngine, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.EqualTo(engine.SourceLanguage)); + Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); + Assert.That(updatedEngine.TargetLanguage, Is.EqualTo("fr")); + } + + [Test] + public async Task UpdateEngine_ShouldNotUpdateTargetLanguage_WhenTargetLanguageNotProvided() + { + var env = new TestEnvironment(); + Engine engine = await env.CreateEngineWithTextFilesAsync(); + + UpdateEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Pretranslations, + env.EngineServiceFactory + ); + await handler.HandleAsync( + new UpdateEngine(OWNER, engine.Id, new TranslationEngineUpdateConfigDto { SourceLanguage = "en" }), + CancellationToken.None + ); + + Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); + + Assert.That(updatedEngine, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.EqualTo("en")); + Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); + Assert.That(updatedEngine.TargetLanguage, Is.EqualTo(engine.TargetLanguage)); + } + + [Test] + public async Task UpdateEngine_ShouldNotUpdate_WhenSourceAndTargetLanguagesNotProvided() + { + var env = new TestEnvironment(); + Engine engine = await env.CreateEngineWithTextFilesAsync(); + + UpdateEngineHandler handler = new( + env.DataAccessContext, + env.Engines, + env.Pretranslations, + env.EngineServiceFactory + ); + await handler.HandleAsync( + new UpdateEngine(OWNER, engine.Id, new TranslationEngineUpdateConfigDto()), + CancellationToken.None + ); + + Engine? updatedEngine = await env.Engines.GetAsync(engine.Id); + + Assert.That(updatedEngine, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.Not.Null); + Assert.That(updatedEngine.SourceLanguage, Is.EqualTo(engine.SourceLanguage)); + Assert.That(updatedEngine.TargetLanguage, Is.Not.Null); + Assert.That(updatedEngine.TargetLanguage, Is.EqualTo(engine.TargetLanguage)); + } + + [Test] + public async Task DeletePretranslationsWhenCorpusIsUpdatedAsync() + { + var env = new TestEnvironment(); + Pretranslation pretranslation = new() + { + Id = "pretranslation1", + EngineRef = "engine1", + CorpusRef = "corpus1", + Refs = ["ref1"], + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + TextId = "textId1", + Translation = "translation", + }; + var engine = await env.CreateEngineWithTextFilesAsync(); + await env.Pretranslations.InsertAsync(pretranslation); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); + await env.Service.UpdateCorpusAsync(engine.Id, "corpus1", sourceFiles: [], targetFiles: []); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); + } + + [Test] + public async Task DeletePretranslationsWhenParallelCorpusIsUpdatedAsync() + { + var env = new TestEnvironment(); + Pretranslation pretranslation = new() + { + Id = "pretranslation1", + EngineRef = "engine1", + CorpusRef = "parallel-corpus1", + Refs = ["ref1"], + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + TextId = "textId1", + Translation = "translation", + }; + var engine = await env.CreateParallelCorpusEngineWithTextFilesAsync(); + await env.Pretranslations.InsertAsync(pretranslation); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); + await env.Service.UpdateParallelCorpusAsync( + engine.Id, + "parallel-corpus1", + sourceCorpora: [], + targetCorpora: [] + ); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); + } + + [Test] + public async Task DeletePretranslationsWhenCorpusFilesAreDeletedAsync() + { + var env = new TestEnvironment(); + Pretranslation pretranslation = new() + { + Id = "pretranslation1", + EngineRef = "engine1", + CorpusRef = "corpus1", + Refs = ["ref1"], + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + TextId = "textId1", + Translation = "translation", + }; + await env.CreateEngineWithTextFilesAsync(); + await env.Pretranslations.InsertAsync(pretranslation); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); + await env.Service.DeleteAllCorpusFilesAsync("file1"); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); + } + + [Test] + public async Task DeletePretranslationsWhenCorpusFilesAreUpdatedAsync() + { + var env = new TestEnvironment(); + Pretranslation pretranslation = new() + { + Id = "pretranslation1", + EngineRef = "engine1", + CorpusRef = "corpus1", + Refs = ["ref1"], + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + TextId = "textId1", + Translation = "translation", + }; + await env.CreateEngineWithTextFilesAsync(); + await env.Pretranslations.InsertAsync(pretranslation); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Not.Null); + await env.Service.UpdateCorpusFilesAsync( + "corpus1", + [ + new() + { + Id = "file1", + Filename = "newfilename", + TextId = "text1", + Format = FileFormat.Text, + }, + ] + ); + Assert.That(await env.Pretranslations.GetAsync(pretranslation.Id), Is.Null); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Engines = new MemoryRepository(); + TranslationEngineService = Substitute.For(); + + var translationResult = new TranslationResultContract + { + Translation = "this is a test.", + SourceTokens = ["esto", "es", "una", "prueba", "."], + TargetTokens = ["this", "is", "a", "test", "."], + Confidences = [1.0, 1.0, 1.0, 1.0, 1.0], + Sources = + [ + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + ], + Alignment = + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + new AlignedWordPairContract { SourceIndex = 2, TargetIndex = 2 }, + new AlignedWordPairContract { SourceIndex = 3, TargetIndex = 3 }, + new AlignedWordPairContract { SourceIndex = 4, TargetIndex = 4 }, + ], + Phrases = + [ + new PhraseContract + { + SourceSegmentStart = 0, + SourceSegmentEnd = 5, + TargetSegmentCut = 5, + }, + ], + }; + TranslationEngineService + .TranslateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([translationResult])); + + var wordGraph = new WordGraphContract + { + SourceTokens = ["esto", "es", "una", "prueba", "."], + InitialStateScore = 0.0, + FinalStates = new HashSet { 3 }, + Arcs = + [ + new WordGraphArcContract + { + PrevState = 0, + NextState = 1, + Score = 1.0, + TargetTokens = ["this", "is"], + Alignment = + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + ], + SourceSegmentStart = 0, + SourceSegmentEnd = 2, + Sources = + [ + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + ], + Confidences = [1.0, 1.0], + }, + new WordGraphArcContract + { + PrevState = 1, + NextState = 2, + Score = 1.0, + TargetTokens = ["a", "test"], + Alignment = + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + ], + SourceSegmentStart = 2, + SourceSegmentEnd = 4, + Sources = + [ + new HashSet { TranslationSource.Primary }, + new HashSet { TranslationSource.Primary }, + ], + Confidences = [1.0, 1.0], + }, + new WordGraphArcContract + { + PrevState = 2, + NextState = 3, + Score = 1.0, + TargetTokens = ["."], + Alignment = [new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }], + SourceSegmentStart = 4, + SourceSegmentEnd = 5, + Sources = [new HashSet { TranslationSource.Primary }], + Confidences = [1.0], + }, + ], + }; + TranslationEngineService + .GetWordGraphAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(wordGraph)); + TranslationEngineService + .CancelBuildAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(null)); + TranslationEngineService + .CreateAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.CompletedTask); + TranslationEngineService + .DeleteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + TranslationEngineService + .StartBuildAsync( + Arg.Any(), + Arg.Any(), + Arg.Any>(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.CompletedTask); + TranslationEngineService + .UpdateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + TranslationEngineService + .TrainSegmentPairAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.CompletedTask); + + IOptionsMonitor dataFileOptions = Substitute.For>(); + dataFileOptions.CurrentValue.Returns(new DataFileOptions()); + + Pretranslations = new MemoryRepository(); + Builds = new MemoryRepository(); + var parallelCorpusService = Substitute.For(); + parallelCorpusService + .GetChapters(Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + return ScriptureRangeParser.GetChapters(callInfo.ArgAt(2)); + }); + ContractMapper = new ContractMapper(dataFileOptions, parallelCorpusService); + DataAccessContext = new MemoryDataAccessContext(); + Service = new EngineService( + Engines, + Pretranslations, + Substitute.For>(), + DataAccessContext + ); + EngineServiceFactory = Substitute.For(); + EngineServiceFactory + .TryGetEngineService("Smt", out Arg.Any()) + .Returns(callInfo => + { + callInfo[1] = TranslationEngineService; + return true; + }); + DtoMapper = new DtoMapper(Substitute.For()); + } + + public EngineService Service { get; } + public IRepository Engines { get; } + public IRepository Builds { get; } + public IRepository Pretranslations { get; } + public ITranslationEngineService TranslationEngineService { get; } + public ContractMapper ContractMapper { get; } + public IEngineServiceFactory EngineServiceFactory { get; } + public IDataAccessContext DataAccessContext { get; } + public DtoMapper DtoMapper { get; } + + public async Task CreateEngineWithTextFilesAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + Corpora = + [ + new() + { + Id = "corpus1", + SourceLanguage = "es", + TargetLanguage = "en", + SourceFiles = + [ + new() + { + Id = "file1", + Filename = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + TargetFiles = + [ + new() + { + Id = "file2", + Filename = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + ModelRevision = 1, + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task CreateMultipleCorporaEngineWithTextFilesAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + Corpora = + [ + new() + { + Id = "corpus1", + SourceLanguage = "es", + TargetLanguage = "en", + SourceFiles = + [ + new() + { + Id = "file1", + Filename = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + TargetFiles = + [ + new() + { + Id = "file2", + Filename = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + new() + { + Id = "corpus2", + SourceLanguage = "es", + TargetLanguage = "en", + SourceFiles = + [ + new() + { + Id = "file3", + Filename = "file3.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + TargetFiles = + [ + new() + { + Id = "file4", + Filename = "file4.txt", + Format = FileFormat.Text, + TextId = "text1", + }, + ], + }, + ], + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task CreateEngineWithParatextProjectAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + Corpora = + [ + new() + { + Id = "corpus1", + SourceLanguage = "es", + TargetLanguage = "en", + SourceFiles = + [ + new() + { + Id = "file1", + Filename = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + TargetFiles = + [ + new() + { + Id = "file2", + Filename = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + ], + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task CreateParallelCorpusEngineWithTextFilesAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + ParallelCorpora = + [ + new() + { + Id = "parallel-corpus1", + SourceCorpora = + [ + new() + { + Id = "parallel-corpus1-source1", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file1", + Filename = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file3", + Filename = "file3.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + ], + TargetCorpora = new List() + { + new() + { + Id = "parallel-corpus1-target1", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file2", + Filename = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file4", + Filename = "file4.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + }, + }, + ], + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + ParallelCorpora = + [ + new() + { + Id = "parallel-corpus1", + SourceCorpora = new List() + { + new() + { + Id = "parallel-corpus1-source1", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file1", + Filename = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + }, + TargetCorpora = new List() + { + new() + { + Id = "parallel-corpus1-target1", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file2", + Filename = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", + }, + ], + }, + }, + }, + new() + { + Id = "parallel-corpus2", + SourceCorpora = new List() + { + new() + { + Id = "parallel-corpus2-source1", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file3", + Filename = "file3.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + }, + TargetCorpora = new List() + { + new() + { + Id = "parallel-corpus2-target1", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file4", + Filename = "file4.txt", + Format = FileFormat.Text, + TextId = "MRK", + }, + ], + }, + }, + }, + ], + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task CreateParallelCorpusEngineWithParatextProjectAsync() + { + var engine = new Engine + { + Id = "engine1", + Owner = OWNER, + SourceLanguage = "es", + TargetLanguage = "en", + Type = "Smt", + ParallelCorpora = + [ + new() + { + Id = "parallel-corpus1", + SourceCorpora = new List() + { + new() + { + Id = "parallel-corpus1-source1", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file1", + Filename = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-source2", + Name = "", + Language = "es", + Files = + [ + new() + { + Id = "file3", + Filename = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", + }, + ], + }, + }, + TargetCorpora = new List() + { + new() + { + Id = "parallel-corpus1-target1", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file2", + Filename = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", + }, + ], + }, + new() + { + Id = "parallel-corpus1-target2", + Name = "", + Language = "en", + Files = + [ + new() + { + Id = "file4", + Filename = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", + }, + ], + }, + }, + }, + ], + }; + await Engines.InsertAsync(engine); + return engine; + } + } +} + +#pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 944dbb56d..8a0704a7f 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -1,8 +1,3 @@ -using Serval.Translation.V1; -using ExecutionData = Serval.Translation.Models.ExecutionData; -using ParallelCorpus = Serval.Shared.Models.ParallelCorpus; -using PhaseStage = Serval.Translation.V1.PhaseStage; - namespace Serval.Translation.Services; [TestFixture] @@ -31,41 +26,35 @@ await env.Builds.InsertAsync( Owner = "owner1", } ); - await env.PlatformService.BuildStarted(new BuildStartedRequest() { BuildId = "b0" }, env.ServerCallContext); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Active)); + await env.PlatformService.BuildStartedAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Active)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.True); - await env.PlatformService.BuildCanceled(new BuildCanceledRequest() { BuildId = "b0" }, env.ServerCallContext); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Canceled)); + await env.PlatformService.BuildCanceledAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Canceled)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); - await env.PlatformService.BuildRestarting( - new BuildRestartingRequest() { BuildId = "b0" }, - env.ServerCallContext - ); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Pending)); + await env.PlatformService.BuildRestartingAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Pending)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); Assert.That(env.Pretranslations.Count, Is.EqualTo(0)); - await env.PlatformService.InsertPretranslations(new MockAsyncStreamReader("e0"), env.ServerCallContext); + await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); - await env.PlatformService.BuildFaulted(new BuildFaultedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildFaultedAsync("b0", ""); Assert.That(env.Pretranslations.Count, Is.EqualTo(0)); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Faulted)); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Faulted)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); - await env.PlatformService.BuildRestarting( - new BuildRestartingRequest() { BuildId = "b0" }, - env.ServerCallContext - ); - await env.PlatformService.InsertPretranslations(new MockAsyncStreamReader("e0"), env.ServerCallContext); + await env.PlatformService.BuildRestartingAsync("b0"); + await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); - await env.PlatformService.BuildCompleted(new BuildCompletedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); - await env.PlatformService.BuildStarted(new BuildStartedRequest() { BuildId = "b0" }, env.ServerCallContext); - await env.PlatformService.InsertPretranslations(new MockAsyncStreamReader("e0"), env.ServerCallContext); - await env.PlatformService.BuildCompleted(new BuildCompletedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildStartedAsync("b0"); + await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); + await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); } @@ -94,24 +83,23 @@ await env.Builds.InsertAsync( ); Assert.That(env.Builds.Get("b0").QueueDepth, Is.Null); Assert.That(env.Builds.Get("b0").Progress, Is.Null); - var request = new UpdateBuildStatusRequest - { - BuildId = "b0", - QueueDepth = 1, - Progress = 0.5, - }; - request.Phases.Add( - new Phase - { - Stage = PhaseStage.Train, - Step = 2, - StepCount = 3, - } + await env.PlatformService.UpdateBuildStatusAsync( + "b0", + new() { Step = 0, PercentCompleted = 0.5 }, + queueDepth: 1, + phases: + [ + new() + { + Stage = PhaseStage.Train, + Step = 2, + StepCount = 3, + }, + ] ); - await env.PlatformService.UpdateBuildStatus(request, env.ServerCallContext); Assert.That(env.Builds.Get("b0").QueueDepth, Is.EqualTo(1)); Assert.That(env.Builds.Get("b0").Progress, Is.EqualTo(0.5)); - Assert.That(env.Builds.Get("b0").Phases![0].Stage, Is.EqualTo(BuildPhaseStage.Train)); + Assert.That(env.Builds.Get("b0").Phases![0].Stage, Is.EqualTo(PhaseStage.Train)); Assert.That(env.Builds.Get("b0").Phases![0].Step, Is.EqualTo(2)); Assert.That(env.Builds.Get("b0").Phases![0].StepCount, Is.EqualTo(3)); } @@ -137,7 +125,7 @@ public async Task UpdateBuildExecutionData() Id = "123", EngineRef = "e0", Owner = "owner1", - ExecutionData = new ExecutionData { TrainCount = 0, PretranslateCount = 0 }, + ExecutionData = new() { TrainCount = 0, PretranslateCount = 0 }, }; await env.Builds.InsertAsync(build); @@ -148,14 +136,11 @@ public async Task UpdateBuildExecutionData() Assert.That(executionData.TrainCount, Is.EqualTo(0)); Assert.That(executionData.PretranslateCount, Is.EqualTo(0)); - var updateRequest = new UpdateBuildExecutionDataRequest() - { - BuildId = "123", - EngineId = engine.Id, - ExecutionData = new V1.ExecutionData { TrainCount = 4, PretranslateCount = 5 }, - }; - - await env.PlatformService.UpdateBuildExecutionData(updateRequest, env.ServerCallContext); + await env.PlatformService.UpdateBuildExecutionDataAsync( + engine.Id, + "123", + new() { TrainCount = 4, PretranslateCount = 5 } + ); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -180,7 +165,7 @@ public async Task UpdateTargetQuoteConventionAsync() TargetLanguage = "es", ParallelCorpora = [ - new ParallelCorpus + new() { Id = "parallelCorpus01", SourceCorpora = [], @@ -200,14 +185,7 @@ public async Task UpdateTargetQuoteConventionAsync() string expected = "typewriter_english"; - var updateRequest = new UpdateTargetQuoteConventionRequest - { - BuildId = "123", - EngineId = engine.Id, - TargetQuoteConvention = "typewriter_english", - }; - - await env.PlatformService.UpdateTargetQuoteConvention(updateRequest, env.ServerCallContext); + await env.PlatformService.UpdateTargetQuoteConventionAsync(engine.Id, "123", "typewriter_english"); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -232,8 +210,7 @@ public async Task UpdateTargetQuoteConventionAsync_NoEngine() }; await env.Builds.InsertAsync(build); - var updateRequest = new UpdateTargetQuoteConventionRequest { BuildId = "123", EngineId = "e0" }; - await env.PlatformService.UpdateTargetQuoteConvention(updateRequest, env.ServerCallContext); + await env.PlatformService.UpdateTargetQuoteConventionAsync("e0", "123", ""); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -268,8 +245,7 @@ public async Task UpdateTargetQuoteConventionAsync_NoParallelCorpora() }; await env.Builds.InsertAsync(build); - var updateRequest = new UpdateTargetQuoteConventionRequest { BuildId = "123", EngineId = engine.Id }; - await env.PlatformService.UpdateTargetQuoteConvention(updateRequest, env.ServerCallContext); + await env.PlatformService.UpdateTargetQuoteConventionAsync(engine.Id, "123", ""); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -296,13 +272,23 @@ await env.Engines.InsertAsync( } ); Assert.That(env.Engines.Get("e0").CorpusSize, Is.EqualTo(0)); - await env.PlatformService.IncrementEngineCorpusSize( - new IncrementEngineCorpusSizeRequest() { EngineId = "e0", Count = 1 }, - env.ServerCallContext - ); + await env.PlatformService.IncrementEngineCorpusSizeAsync("e0", 1); Assert.That(env.Engines.Get("e0").CorpusSize, Is.EqualTo(1)); } + private static async IAsyncEnumerable GetTestPretranslations() + { + yield return new PretranslationContract + { + CorpusId = "e0", + TextId = "text1", + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + Translation = "test", + }; + await Task.CompletedTask; + } + private class TestEnvironment { public TestEnvironment() @@ -311,8 +297,6 @@ public TestEnvironment() Engines = new MemoryRepository(); Pretranslations = new MemoryRepository(); DataAccessContext = Substitute.For(); - PublishEndpoint = Substitute.For(); - ServerCallContext = Substitute.For(); DataAccessContext .WithTransactionAsync(Arg.Any>(), Arg.Any()) @@ -327,12 +311,12 @@ public TestEnvironment() return ((Func)x[0])((CancellationToken)x[1]); }); - PlatformService = new TranslationPlatformServiceV1( + PlatformService = new PlatformService( Builds, Engines, Pretranslations, DataAccessContext, - PublishEndpoint + Substitute.For() ); } @@ -340,23 +324,6 @@ public TestEnvironment() public MemoryRepository Engines { get; } public MemoryRepository Pretranslations { get; } public IDataAccessContext DataAccessContext { get; } - public IPublishEndpoint PublishEndpoint { get; } - public ServerCallContext ServerCallContext { get; } - public TranslationPlatformServiceV1 PlatformService { get; } - } - - private class MockAsyncStreamReader(string engineId) : IAsyncStreamReader - { - private bool _endOfStream = false; - - public string EngineId { get; } = engineId; - public InsertPretranslationsRequest Current => new() { EngineId = EngineId }; - - public Task MoveNext(CancellationToken cancellationToken) - { - var ret = Task.FromResult(!_endOfStream); - _endOfStream = true; - return ret; - } + public PlatformService PlatformService { get; } } } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index 941714a53..32ee38d47 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -1,7 +1,4 @@ -using System.IO.Compression; -using SIL.Machine.Utils; - -namespace Serval.Translation.Services; +namespace Serval.Translation.Services; [TestFixture] public class PretranslationServiceTests @@ -278,7 +275,7 @@ public async Task GetUsfmAsync_Target_OnlyExisting() \v 3 TRG - Chapter one, verse three. "; - List lines = targetUsfm.Split('\n').ToList(); + List lines = [.. targetUsfm.Split('\n')]; lines.Insert( 1, @@ -444,7 +441,7 @@ await env.Service.GetUsfmAsync( [TestCase(new int[] { 1 }, "1")] public void GetChapterRanges(int[] chapterNumbers, string expectedRangeString) { - string actualRangeString = ParallelCorpusService.GetChapterRangesString(chapterNumbers.ToList()); + string actualRangeString = PretranslationService.GetChapterRangesString([.. chapterNumbers]); Assert.That(actualRangeString, Is.EqualTo(expectedRangeString)); } @@ -479,14 +476,14 @@ public TestEnvironment(bool addMatthew = false) { Id = "file1", Filename = "file1.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "project1", }; CorpusFile file2 = new() { Id = "file2", Filename = "file2.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "project1", }; Engines = new MemoryRepository([ @@ -523,24 +520,24 @@ public TestEnvironment(bool addMatthew = false) new() { Id = "parallel_corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "src_1", Language = "en", Files = [file1], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "trg_1", Language = "es", Files = [file2], }, - }, + ], }, ], }, @@ -670,13 +667,11 @@ public TestEnvironment(bool addMatthew = false) ]); IOptionsMonitor dataFileOptions = Substitute.For>(); dataFileOptions.CurrentValue.Returns(new DataFileOptions() { FilesDirectory = _tempDir.Path }); - var parallelCorpusService = new ParallelCorpusService(); Service = new PretranslationService( Pretranslations, Engines, Builds, - new CorpusMappingService(dataFileOptions, parallelCorpusService), - parallelCorpusService + new ContractMapper(dataFileOptions, new ParallelCorpusService()) ); } diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs index 1403ffb4e..8ff459f2b 100644 --- a/src/Serval/test/Serval.Translation.Tests/Usings.cs +++ b/src/Serval/test/Serval.Translation.Tests/Usings.cs @@ -1,15 +1,19 @@ +global using System.IO.Compression; global using System.Text.RegularExpressions; -global using Grpc.Core; -global using Grpc.Net.ClientFactory; -global using MassTransit; +global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; +global using Serval.DataFiles.Contracts; global using Serval.Shared.Configuration; -global using Serval.Shared.Models; +global using Serval.Shared.Contracts; +global using Serval.Shared.Dtos; +global using Serval.Shared.Services; global using Serval.Shared.Utils; global using Serval.Translation.Contracts; +global using Serval.Translation.Dtos; +global using Serval.Translation.Features.Engines; global using Serval.Translation.Models; global using SIL.DataAccess; -global using SIL.ServiceToolkit.Services; +global using SIL.Machine.Utils; diff --git a/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj b/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj index 7b149d34c..651e26997 100644 --- a/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj +++ b/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj @@ -9,6 +9,7 @@ true true $(NoWarn);CS1591;CS1573 + Serval.Webhooks diff --git a/src/Serval/test/Serval.Webhooks.Tests/Usings.cs b/src/Serval/test/Serval.Webhooks.Tests/Usings.cs index fb600d13f..bc661145e 100644 --- a/src/Serval/test/Serval.Webhooks.Tests/Usings.cs +++ b/src/Serval/test/Serval.Webhooks.Tests/Usings.cs @@ -5,6 +5,5 @@ global using NSubstitute; global using NUnit.Framework; global using RichardSzalay.MockHttp; -global using Serval.Webhooks.Contracts; global using Serval.Webhooks.Models; global using SIL.DataAccess; diff --git a/src/Serval/test/Serval.WordAlignment.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.WordAlignment.Tests/Services/EngineServiceTests.cs index 150b86cad..651863582 100644 --- a/src/Serval/test/Serval.WordAlignment.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.WordAlignment.Tests/Services/EngineServiceTests.cs @@ -1,8 +1,3 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.WordAlignment.Configuration; -using Serval.WordAlignment.V1; -using SIL.ServiceToolkit.Services; - namespace Serval.WordAlignment.Services; [TestFixture] @@ -24,13 +19,13 @@ public async Task GetWordAlignmentAsync_EngineExists() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - Models.WordAlignmentResult? result = await env.Service.GetWordAlignmentAsync( + WordAlignmentResult? result = await env.Service.GetWordAlignmentAsync( engineId, "esto es una prueba.", "this is a test." ); Assert.That(result, Is.Not.Null); - Assert.That(result!.Alignment, Is.EqualTo(CreateNAlignedWordPair(5))); + Assert.That(result.Alignment, Is.EqualTo(CreateNAlignedWordPair(5))); } [Test] @@ -77,70 +72,53 @@ public async Task StartBuildAsync_TrainOnNotSpecified() var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; await env.Service.StartBuildAsync(new Build { Id = BUILD1_ID, EngineRef = engineId }); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -159,84 +137,61 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { - new() { CorpusRef = "corpus1-source1", TextIds = [] }, - }, - TargetFilters = new List() - { - new() { CorpusRef = "corpus1-target1", TextIds = [] }, - }, + SourceFilters = [new() { CorpusRef = "corpus1-source1", TextIds = [] }], + TargetFilters = [new() { CorpusRef = "corpus1-target1", TextIds = [] }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + TrainOnTextIds = [], + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - TrainOnTextIds = { }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + TrainOnTextIds = [], + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - TrainOnTextIds = { }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -255,84 +210,61 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { - new() { CorpusRef = "corpus1-source1", TextIds = ["text1"] }, - }, - TargetFilters = new List() - { - new() { CorpusRef = "corpus1-target1", TextIds = ["text1"] }, - }, + SourceFilters = [new() { CorpusRef = "corpus1-source1", TextIds = ["text1"] }], + TargetFilters = [new() { CorpusRef = "corpus1-target1", TextIds = ["text1"] }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + TrainOnTextIds = ["text1"], + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - TrainOnTextIds = { "text1" }, - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + TrainOnTextIds = ["text1"], + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - TrainOnTextIds = { "text1" }, - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -351,76 +283,59 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() { new() { CorpusRef = "corpus1-source1" } }, - TargetFilters = new List() { new() { CorpusRef = "corpus1-target1" } }, + SourceFilters = [new() { CorpusRef = "corpus1-source1" }], + TargetFilters = [new() { CorpusRef = "corpus1-target1" }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "text1", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "text1", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -438,70 +353,53 @@ await env.Service.StartBuildAsync( WordAlignOn = [new WordAlignmentCorpus { ParallelCorpusRef = "corpus1" }], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - WordAlignOnAll = true, - TrainOnAll = true, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -519,118 +417,95 @@ await env.Service.StartBuildAsync( WordAlignOn = [new WordAlignmentCorpus { ParallelCorpusRef = "corpus2" }], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + InferenceTextIds = [], + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - WordAlignOnAll = false, - TrainOnAll = true, + Location = "file1.txt", + Format = FileFormat.Text, + TextId = "MAT", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + InferenceTextIds = [], + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.txt", - Format = V1.FileFormat.Text, - TextId = "MAT", - }, - }, - WordAlignOnAll = false, - TrainOnAll = true, + Location = "file2.txt", + Format = FileFormat.Text, + TextId = "MAT", }, - }, + ], }, - }, - new V1.ParallelCorpus - { - Id = "corpus2", - SourceCorpora = + ], + }, + new() + { + Id = "corpus2", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus2-source1", + Language = "es", + TrainOnTextIds = [], + Files = + [ new() { - Id = "corpus2-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file3.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file3.txt", + Format = FileFormat.Text, + TextId = "MRK", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus2-target1", + Language = "en", + TrainOnTextIds = [], + Files = + [ new() { - Id = "corpus2-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file4.txt", - Format = V1.FileFormat.Text, - TextId = "MRK", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file4.txt", + Format = FileFormat.Text, + TextId = "MRK", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -650,24 +525,24 @@ public async Task StartBuildAsync_TextFilesScriptureRangeSpecified() new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { + SourceFilters = + [ new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT", TextIds = [], }, - }, - TargetFilters = new List() - { + ], + TargetFilters = + [ new() { CorpusRef = "corpus1-target1", ScriptureRange = "MAT", TextIds = [], }, - }, + ], }, ], } @@ -690,104 +565,61 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { - new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT 1;MRK" }, - }, - TargetFilters = new List() - { - new() { CorpusRef = "corpus1-target1", ScriptureRange = "MAT;MRK 1" }, - }, + SourceFilters = [new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT 1;MRK" }], + TargetFilters = [new() { CorpusRef = "corpus1-target1", ScriptureRange = "MAT;MRK 1" }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + TrainOnChapters = new Dictionary> { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { } } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + TrainOnChapters = new Dictionary> { ["MAT"] = [], ["MRK"] = [1] }, + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1 } } - }, - }, - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -806,82 +638,61 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { - new() { CorpusRef = "corpus1-source1", ScriptureRange = "" }, - }, - TargetFilters = new List() - { - new() { CorpusRef = "corpus1-target1", ScriptureRange = "" }, - }, + SourceFilters = [new() { CorpusRef = "corpus1-source1", ScriptureRange = "" }], + TargetFilters = [new() { CorpusRef = "corpus1-target1", ScriptureRange = "" }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new List - { + Id = "corpus1-source1", + Language = "es", + TrainOnChapters = [], + Files = + [ new() { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", }, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new List - { + Id = "corpus1-target1", + Language = "en", + TrainOnChapters = [], + Files = + [ new() { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", }, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -900,154 +711,111 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { + SourceFilters = + [ new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT 1-2;MRK 1-2" }, new() { CorpusRef = "corpus1-source2", ScriptureRange = "MAT 3;MRK 1" }, - }, - TargetFilters = new List() - { + ], + TargetFilters = + [ new() { CorpusRef = "corpus1-target1", ScriptureRange = "MAT 2-3;MRK 2" }, new() { CorpusRef = "corpus1-target2", ScriptureRange = "MAT 1;MRK 1-2" }, - }, + ], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new V1.MonolingualCorpus() + Id = "corpus1-source1", + Language = "es", + TrainOnChapters = new Dictionary> { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 1, 2 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1, 2 } } - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + ["MAT"] = [1, 2], + ["MRK"] = [1, 2], }, - new V1.MonolingualCorpus() - { - Id = "corpus1-source2", - Language = "es", - Files = + Files = + [ + new() { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", }, - TrainOnChapters = + ], + }, + new() + { + Id = "corpus1-source2", + Language = "es", + TrainOnChapters = new Dictionary> { ["MAT"] = [3], ["MRK"] = [1] }, + Files = + [ + new() { - { - "MAT", - new ScriptureChapters { Chapters = { 3 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1 } } - }, + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", }, - WordAlignOnAll = true, - TrainOnAll = false, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new V1.MonolingualCorpus() + Id = "corpus1-target1", + Language = "en", + TrainOnChapters = new Dictionary> { - Id = "corpus1-target1", - Language = "en", - Files = - { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, - }, - TrainOnChapters = - { - { - "MAT", - new ScriptureChapters { Chapters = { 2, 3 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 2 } } - }, - }, - WordAlignOnAll = true, - TrainOnAll = false, + ["MAT"] = [2, 3], + ["MRK"] = [2], }, - new V1.MonolingualCorpus() - { - Id = "corpus1-target2", - Language = "en", - Files = + Files = + [ + new() { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", }, - TrainOnChapters = + ], + }, + new() + { + Id = "corpus1-target2", + Language = "en", + TrainOnChapters = new Dictionary> + { + ["MAT"] = [1], + ["MRK"] = [1, 2], + }, + Files = + [ + new() { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { Chapters = { 1, 2 } } - }, + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", }, - WordAlignOnAll = true, - TrainOnAll = false, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -1066,115 +834,88 @@ await env.Service.StartBuildAsync( new TrainingCorpus { ParallelCorpusRef = "corpus1", - SourceFilters = new List() - { - new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT 1;MRK" }, - }, + SourceFilters = [new() { CorpusRef = "corpus1-source1", ScriptureRange = "MAT 1;MRK" }], }, ], } ); - _ = env - .OutboxService.Received() - .EnqueueMessageAsync( - EngineOutboxConstants.OutboxId, - EngineOutboxConstants.StartBuild, + await env + .WordAlignmentEngineService.Received() + .StartBuildAsync( engineId, - new StartBuildRequest - { - BuildId = BUILD1_ID, - EngineId = engineId, - EngineType = "Statistical", - Corpora = + BUILD1_ID, + ArgEx.IsEquivalentTo>([ + new() { - new V1.ParallelCorpus - { - Id = "corpus1", - SourceCorpora = + Id = "corpus1", + SourceCorpora = + [ + new() { - new V1.MonolingualCorpus() - { - Id = "corpus1-source1", - Language = "es", - Files = - { - new V1.CorpusFile - { - Location = "file1.zip", - Format = V1.FileFormat.Paratext, - TextId = "file1.zip", - }, - }, - TrainOnChapters = + Id = "corpus1-source1", + Language = "es", + TrainOnChapters = new Dictionary> { ["MAT"] = [1], ["MRK"] = [] }, + Files = + [ + new() { - { - "MAT", - new ScriptureChapters { Chapters = { 1 } } - }, - { - "MRK", - new ScriptureChapters { } - }, + Location = "file1.zip", + Format = FileFormat.Paratext, + TextId = "file1.zip", }, - WordAlignOnAll = true, - TrainOnAll = false, - }, - new V1.MonolingualCorpus() - { - Id = "corpus1-source2", - Language = "es", - Files = + ], + }, + new() + { + Id = "corpus1-source2", + Language = "es", + TrainOnTextIds = [], + Files = + [ + new() { - new V1.CorpusFile - { - Location = "file3.zip", - Format = V1.FileFormat.Paratext, - TextId = "file3.zip", - }, + Location = "file3.zip", + Format = FileFormat.Paratext, + TextId = "file3.zip", }, - WordAlignOnAll = true, - TrainOnAll = false, - }, + ], }, - TargetCorpora = + ], + TargetCorpora = + [ + new() { - new V1.MonolingualCorpus() - { - Id = "corpus1-target1", - Language = "en", - Files = + Id = "corpus1-target1", + Language = "en", + Files = + [ + new() { - new V1.CorpusFile - { - Location = "file2.zip", - Format = V1.FileFormat.Paratext, - TextId = "file2.zip", - }, + Location = "file2.zip", + Format = FileFormat.Paratext, + TextId = "file2.zip", }, - WordAlignOnAll = true, - TrainOnAll = true, - }, - new V1.MonolingualCorpus() - { - Id = "corpus1-target2", - Language = "en", - Files = + ], + }, + new() + { + Id = "corpus1-target2", + Language = "en", + Files = + [ + new() { - new V1.CorpusFile - { - Location = "file4.zip", - Format = V1.FileFormat.Paratext, - TextId = "file4.zip", - }, + Location = "file4.zip", + Format = FileFormat.Paratext, + TextId = "file4.zip", }, - WordAlignOnAll = true, - TrainOnAll = true, - }, + ], }, - }, + ], }, - }, - cancellationToken: Arg.Any() + ]), + null, + Arg.Any() ); } @@ -1193,11 +934,11 @@ public async Task UpdateCorpusAsync() Engine engine = await env.CreateEngineWithTextFilesAsync(); string corpusId = engine.ParallelCorpora[0].Id; - Shared.Models.ParallelCorpus? corpus = await env.Service.UpdateParallelCorpusAsync( + ParallelCorpus? corpus = await env.Service.UpdateParallelCorpusAsync( engine.Id, corpusId, - sourceCorpora: new List - { + sourceCorpora: + [ new() { Id = "corpus1-source1", @@ -1209,7 +950,7 @@ public async Task UpdateCorpusAsync() { Id = "file1", Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "text1", }, ], @@ -1225,12 +966,12 @@ public async Task UpdateCorpusAsync() { Id = "file3", Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "text2", }, ], }, - }, + ], null ); @@ -1317,7 +1058,7 @@ await env.Service.UpdateCorpusFilesAsync( Id = "file1", Filename = "newfilename", TextId = "text1", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, }, ] ); @@ -1329,66 +1070,55 @@ private class TestEnvironment public TestEnvironment() { Engines = new MemoryRepository(); - WordAlignmentServiceClient = Substitute.For(); - var wordAlignmentResult = new V1.WordAlignmentResult - { - SourceTokens = { "esto es una prueba .".Split() }, - TargetTokens = { "this is a test .".Split() }, - Alignment = + WordAlignments = new MemoryRepository(); + + WordAlignmentEngineService = Substitute.For(); + WordAlignmentEngineService + .AlignAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns( + new WordAlignmentResultContract + { + SourceTokens = "esto es una prueba .".Split(), + TargetTokens = "this is a test .".Split(), + Alignment = + [ + new AlignedWordPairContract { SourceIndex = 0, TargetIndex = 0 }, + new AlignedWordPairContract { SourceIndex = 1, TargetIndex = 1 }, + new AlignedWordPairContract { SourceIndex = 2, TargetIndex = 2 }, + new AlignedWordPairContract { SourceIndex = 3, TargetIndex = 3 }, + new AlignedWordPairContract { SourceIndex = 4, TargetIndex = 4 }, + ], + } + ); + + EngineServiceFactory = Substitute.For(); + EngineServiceFactory + .TryGetEngineService("Statistical", out Arg.Any()) + .Returns(x => { - new V1.AlignedWordPair { SourceIndex = 0, TargetIndex = 0 }, - new V1.AlignedWordPair { SourceIndex = 1, TargetIndex = 1 }, - new V1.AlignedWordPair { SourceIndex = 2, TargetIndex = 2 }, - new V1.AlignedWordPair { SourceIndex = 3, TargetIndex = 3 }, - new V1.AlignedWordPair { SourceIndex = 4, TargetIndex = 4 }, - }, - }; - var wordAlignmentResponse = new GetWordAlignmentResponse { Result = wordAlignmentResult }; - WordAlignmentServiceClient - .GetWordAlignmentAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(wordAlignmentResponse)); - WordAlignmentServiceClient - .CancelBuildAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(new CancelBuildResponse())); - WordAlignmentServiceClient.CreateAsync(Arg.Any()).Returns(CreateAsyncUnaryCall(new Empty())); - WordAlignmentServiceClient.DeleteAsync(Arg.Any()).Returns(CreateAsyncUnaryCall(new Empty())); - WordAlignmentServiceClient - .StartBuildAsync(Arg.Any()) - .Returns(CreateAsyncUnaryCall(new Empty())); - GrpcClientFactory grpcClientFactory = Substitute.For(); - grpcClientFactory - .CreateClient("Statistical") - .Returns(WordAlignmentServiceClient); + x[1] = WordAlignmentEngineService; + return true; + }); + IOptionsMonitor dataFileOptions = Substitute.For>(); dataFileOptions.CurrentValue.Returns(new DataFileOptions()); - WordAlignments = new MemoryRepository(); - OutboxService = Substitute.For(); - IOptionsMonitor wordAlignmentOptions = Substitute.For< - IOptionsMonitor - >(); - wordAlignmentOptions.CurrentValue.Returns( - new WordAlignmentOptions { Engines = [new EngineInfo { Type = "Statistical" }] } - ); - Service = new TestEngineService( Engines, new MemoryRepository(), WordAlignments, - grpcClientFactory, + EngineServiceFactory, dataFileOptions, new MemoryDataAccessContext(), - new LoggerFactory(), - OutboxService, - wordAlignmentOptions + new LoggerFactory() ); } public EngineService Service { get; } public IRepository Engines { get; } public IRepository WordAlignments { get; } - public WordAlignmentEngineApi.WordAlignmentEngineApiClient WordAlignmentServiceClient { get; } - public IOutboxService OutboxService { get; } + public IWordAlignmentEngineService WordAlignmentEngineService { get; } + public IEngineServiceFactory EngineServiceFactory { get; } public async Task CreateEngineWithTextFilesAsync() { @@ -1404,8 +1134,8 @@ public async Task CreateEngineWithTextFilesAsync() new() { Id = "corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus1-source1", @@ -1417,14 +1147,14 @@ public async Task CreateEngineWithTextFilesAsync() { Id = "file1", Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "text1", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus1-target1", @@ -1436,12 +1166,12 @@ public async Task CreateEngineWithTextFilesAsync() { Id = "file2", Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "text1", }, ], }, - }, + ], }, ], ModelRevision = 1, @@ -1464,8 +1194,8 @@ public async Task CreateEngineWithMultipleTextFilesAsync() new() { Id = "corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus1-source1", @@ -1477,7 +1207,7 @@ public async Task CreateEngineWithMultipleTextFilesAsync() { Id = "file1", Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MAT", }, ], @@ -1493,14 +1223,14 @@ public async Task CreateEngineWithMultipleTextFilesAsync() { Id = "file3", Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MRK", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus1-target1", @@ -1512,7 +1242,7 @@ public async Task CreateEngineWithMultipleTextFilesAsync() { Id = "file2", Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MAT", }, ], @@ -1528,12 +1258,12 @@ public async Task CreateEngineWithMultipleTextFilesAsync() { Id = "file4", Filename = "file4.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MRK", }, ], }, - }, + ], }, ], }; @@ -1555,8 +1285,8 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() new() { Id = "corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus1-source1", @@ -1568,14 +1298,14 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() { Id = "file1", Filename = "file1.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MAT", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus1-target1", @@ -1587,18 +1317,18 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() { Id = "file2", Filename = "file2.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MAT", }, ], }, - }, + ], }, new() { Id = "corpus2", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus2-source1", @@ -1610,14 +1340,14 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() { Id = "file3", Filename = "file3.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MRK", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus2-target1", @@ -1629,12 +1359,12 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() { Id = "file4", Filename = "file4.txt", - Format = Shared.Contracts.FileFormat.Text, + Format = FileFormat.Text, TextId = "MRK", }, ], }, - }, + ], }, ], }; @@ -1656,8 +1386,8 @@ public async Task CreateEngineWithParatextProjectAsync() new() { Id = "corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus1-source1", @@ -1669,14 +1399,14 @@ public async Task CreateEngineWithParatextProjectAsync() { Id = "file1", Filename = "file1.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file1.zip", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus1-target1", @@ -1688,12 +1418,12 @@ public async Task CreateEngineWithParatextProjectAsync() { Id = "file2", Filename = "file2.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file2.zip", }, ], }, - }, + ], }, ], }; @@ -1715,8 +1445,8 @@ public async Task CreateEngineWithMultipleParatextProjectAsync() new() { Id = "corpus1", - SourceCorpora = new List() - { + SourceCorpora = + [ new() { Id = "corpus1-source1", @@ -1728,7 +1458,7 @@ public async Task CreateEngineWithMultipleParatextProjectAsync() { Id = "file1", Filename = "file1.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file1.zip", }, ], @@ -1744,14 +1474,14 @@ public async Task CreateEngineWithMultipleParatextProjectAsync() { Id = "file3", Filename = "file3.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file3.zip", }, ], }, - }, - TargetCorpora = new List() - { + ], + TargetCorpora = + [ new() { Id = "corpus1-target1", @@ -1763,7 +1493,7 @@ public async Task CreateEngineWithMultipleParatextProjectAsync() { Id = "file2", Filename = "file2.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file2.zip", }, ], @@ -1779,37 +1509,26 @@ public async Task CreateEngineWithMultipleParatextProjectAsync() { Id = "file4", Filename = "file4.zip", - Format = Shared.Contracts.FileFormat.Paratext, + Format = FileFormat.Paratext, TextId = "file4.zip", }, ], }, - }, + ], }, ], }; await Engines.InsertAsync(engine); return engine; } - - private static AsyncUnaryCall CreateAsyncUnaryCall(TResponse response) - { - return new AsyncUnaryCall( - Task.FromResult(response), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - } } - private static IReadOnlyList CreateNAlignedWordPair(int numberOfAlignedWords) + private static IReadOnlyList CreateNAlignedWordPair(int numberOfAlignedWords) { - var alignedWordPairs = new List(); + var alignedWordPairs = new List(); for (int i = 0; i < numberOfAlignedWords; i++) { - alignedWordPairs.Add(new Models.AlignedWordPair { SourceIndex = i, TargetIndex = i }); + alignedWordPairs.Add(new AlignedWordPair { SourceIndex = i, TargetIndex = i }); } return alignedWordPairs; } @@ -1818,23 +1537,19 @@ private class TestEngineService( IRepository engines, IRepository builds, IRepository wordAlignments, - GrpcClientFactory grpcClientFactory, + IEngineServiceFactory engineServiceFactory, IOptionsMonitor dataFileOptions, IDataAccessContext dataAccessContext, - ILoggerFactory loggerFactory, - IOutboxService outboxService, - IOptionsMonitor wordAlignmentOptions + ILoggerFactory loggerFactory ) : EngineService( engines, builds, wordAlignments, - grpcClientFactory, + engineServiceFactory, dataFileOptions, dataAccessContext, - loggerFactory, - outboxService, - wordAlignmentOptions + loggerFactory ) { protected override Dictionary> GetChapters(string fileLocation, string scriptureRange) diff --git a/src/Serval/test/Serval.WordAlignment.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.WordAlignment.Tests/Services/PlatformServiceTests.cs index da8614674..c79e6f510 100644 --- a/src/Serval/test/Serval.WordAlignment.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.WordAlignment.Tests/Services/PlatformServiceTests.cs @@ -1,6 +1,3 @@ -using Serval.WordAlignment.V1; -using ExecutionData = Serval.WordAlignment.Models.ExecutionData; - namespace Serval.WordAlignment.Services; [TestFixture] @@ -22,41 +19,35 @@ await env.Engines.InsertAsync( } ); await env.Builds.InsertAsync(new Build() { Id = "b0", EngineRef = "e0" }); - await env.PlatformService.BuildStarted(new BuildStartedRequest() { BuildId = "b0" }, env.ServerCallContext); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Active)); + await env.PlatformService.BuildStartedAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Active)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.True); - await env.PlatformService.BuildCanceled(new BuildCanceledRequest() { BuildId = "b0" }, env.ServerCallContext); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Canceled)); + await env.PlatformService.BuildCanceledAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Canceled)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); - await env.PlatformService.BuildRestarting( - new BuildRestartingRequest() { BuildId = "b0" }, - env.ServerCallContext - ); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Pending)); + await env.PlatformService.BuildRestartingAsync("b0"); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Pending)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); Assert.That(env.WordAlignments.Count, Is.EqualTo(0)); - await env.PlatformService.InsertWordAlignments(new MockAsyncStreamReader("e0"), env.ServerCallContext); + await env.PlatformService.InsertWordAlignmentsAsync("e0", GetTestWordAlignments()); Assert.That(env.WordAlignments.Count, Is.EqualTo(1)); - await env.PlatformService.BuildFaulted(new BuildFaultedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildFaultedAsync("b0", "Faulted"); Assert.That(env.WordAlignments.Count, Is.EqualTo(0)); - Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Faulted)); + Assert.That(env.Builds.Get("b0").State, Is.EqualTo(JobState.Faulted)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); - await env.PlatformService.BuildRestarting( - new BuildRestartingRequest() { BuildId = "b0" }, - env.ServerCallContext - ); - await env.PlatformService.InsertWordAlignments(new MockAsyncStreamReader("e0"), env.ServerCallContext); + await env.PlatformService.BuildRestartingAsync("b0"); + await env.PlatformService.InsertWordAlignmentsAsync("e0", GetTestWordAlignments()); Assert.That(env.WordAlignments.Count, Is.EqualTo(1)); - await env.PlatformService.BuildCompleted(new BuildCompletedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.WordAlignments.Count, Is.EqualTo(1)); - await env.PlatformService.BuildStarted(new BuildStartedRequest() { BuildId = "b0" }, env.ServerCallContext); - await env.PlatformService.InsertWordAlignments(new MockAsyncStreamReader("e0"), env.ServerCallContext); - await env.PlatformService.BuildCompleted(new BuildCompletedRequest() { BuildId = "b0" }, env.ServerCallContext); + await env.PlatformService.BuildStartedAsync("b0"); + await env.PlatformService.InsertWordAlignmentsAsync("e0", GetTestWordAlignments()); + await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.WordAlignments.Count, Is.EqualTo(1)); } @@ -78,24 +69,25 @@ await env.Engines.InsertAsync( await env.Builds.InsertAsync(new Build() { Id = "b0", EngineRef = "e0" }); Assert.That(env.Builds.Get("b0").QueueDepth, Is.Null); Assert.That(env.Builds.Get("b0").Progress, Is.Null); - var request = new UpdateBuildStatusRequest - { - BuildId = "b0", - QueueDepth = 1, - Progress = 0.5, - }; - request.Phases.Add( - new Phase - { - Stage = PhaseStage.Train, - Step = 2, - StepCount = 3, - } + + await env.PlatformService.BuildStartedAsync("b0"); + await env.PlatformService.UpdateBuildStatusAsync( + "b0", + new() { PercentCompleted = 0.5 }, + queueDepth: 1, + phases: + [ + new() + { + Stage = PhaseStage.Train, + Step = 2, + StepCount = 3, + }, + ] ); - await env.PlatformService.UpdateBuildStatus(request, env.ServerCallContext); Assert.That(env.Builds.Get("b0").QueueDepth, Is.EqualTo(1)); Assert.That(env.Builds.Get("b0").Progress, Is.EqualTo(0.5)); - Assert.That(env.Builds.Get("b0").Phases![0].Stage, Is.EqualTo(BuildPhaseStage.Train)); + Assert.That(env.Builds.Get("b0").Phases![0].Stage, Is.EqualTo(PhaseStage.Train)); Assert.That(env.Builds.Get("b0").Phases![0].Step, Is.EqualTo(2)); Assert.That(env.Builds.Get("b0").Phases![0].StepCount, Is.EqualTo(3)); } @@ -120,25 +112,22 @@ public async Task UpdateBuildExecutionData() { Id = "123", EngineRef = "e0", - ExecutionData = new ExecutionData { TrainCount = 0, WordAlignCount = 0 }, + ExecutionData = new() { TrainCount = 0, WordAlignCount = 0 }, }; await env.Builds.InsertAsync(build); Assert.That(build.ExecutionData, Is.Not.Null); - var executionData = build.ExecutionData; + ExecutionData? executionData = build.ExecutionData; Assert.That(executionData.TrainCount, Is.EqualTo(0)); Assert.That(executionData.WordAlignCount, Is.EqualTo(0)); - var updateRequest = new UpdateBuildExecutionDataRequest() - { - BuildId = "123", - EngineId = engine.Id, - ExecutionData = new V1.ExecutionData { TrainCount = 4, WordAlignCount = 5 }, - }; - - await env.PlatformService.UpdateBuildExecutionData(updateRequest, env.ServerCallContext); + await env.PlatformService.UpdateBuildExecutionDataAsync( + engine.Id, + build.Id, + new() { TrainCount = 4, WordAlignCount = 5 } + ); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -165,13 +154,25 @@ await env.Engines.InsertAsync( } ); Assert.That(env.Engines.Get("e0").CorpusSize, Is.EqualTo(0)); - await env.PlatformService.IncrementEngineCorpusSize( - new IncrementEngineCorpusSizeRequest() { EngineId = "e0", Count = 1 }, - env.ServerCallContext - ); + await env.PlatformService.IncrementEngineCorpusSizeAsync("e0", 1); Assert.That(env.Engines.Get("e0").CorpusSize, Is.EqualTo(1)); } + private static async IAsyncEnumerable GetTestWordAlignments() + { + yield return new() + { + CorpusId = "corpus1", + TextId = "text1", + SourceRefs = ["ref1"], + TargetRefs = ["ref1"], + SourceTokens = ["esto"], + TargetTokens = ["this"], + Alignment = [new() { SourceIndex = 0, TargetIndex = 0 }], + }; + await Task.CompletedTask; + } + private class TestEnvironment { public TestEnvironment() @@ -180,8 +181,7 @@ public TestEnvironment() Engines = new MemoryRepository(); WordAlignments = new MemoryRepository(); DataAccessContext = Substitute.For(); - PublishEndpoint = Substitute.For(); - ServerCallContext = Substitute.For(); + EventRouter = Substitute.For(); DataAccessContext .WithTransactionAsync(Arg.Any>(), Arg.Any()) @@ -196,36 +196,14 @@ public TestEnvironment() return ((Func)x[0])((CancellationToken)x[1]); }); - PlatformService = new WordAlignmentPlatformServiceV1( - Builds, - Engines, - WordAlignments, - DataAccessContext, - PublishEndpoint - ); + PlatformService = new PlatformService(Builds, Engines, WordAlignments, DataAccessContext, EventRouter); } public MemoryRepository Builds { get; } public MemoryRepository Engines { get; } public MemoryRepository WordAlignments { get; } public IDataAccessContext DataAccessContext { get; } - public IPublishEndpoint PublishEndpoint { get; } - public ServerCallContext ServerCallContext { get; } - public WordAlignmentPlatformServiceV1 PlatformService { get; } - } - - private class MockAsyncStreamReader(string engineId) : IAsyncStreamReader - { - private bool _endOfStream = false; - - public string EngineId { get; } = engineId; - public InsertWordAlignmentsRequest Current => new() { EngineId = EngineId }; - - public Task MoveNext(CancellationToken cancellationToken) - { - var ret = Task.FromResult(!_endOfStream); - _endOfStream = true; - return ret; - } + public IEventRouter EventRouter { get; } + public PlatformService PlatformService { get; } } } diff --git a/src/Serval/test/Serval.WordAlignment.Tests/Usings.cs b/src/Serval/test/Serval.WordAlignment.Tests/Usings.cs index e1c4deec7..361d1450e 100644 --- a/src/Serval/test/Serval.WordAlignment.Tests/Usings.cs +++ b/src/Serval/test/Serval.WordAlignment.Tests/Usings.cs @@ -1,12 +1,11 @@ -global using Grpc.Core; -global using Grpc.Net.ClientFactory; -global using MassTransit; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Models; global using Serval.Shared.Utils; +global using Serval.WordAlignment.Contracts; global using Serval.WordAlignment.Models; global using SIL.DataAccess; diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs deleted file mode 100644 index 7987a759f..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IHealthChecksBuilderExtensions -{ - public static IHealthChecksBuilder AddHangfire(this IHealthChecksBuilder builder, string name = "Hangfire") - { - builder.AddCheck(name); - return builder; - } - - public static IHealthChecksBuilder AddOutbox(this IHealthChecksBuilder builder, string name = "Outbox") - { - builder.AddCheck(name); - return builder; - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfiguratorExtensions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfiguratorExtensions.cs deleted file mode 100644 index b5786d90c..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IOutboxConfiguratorExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IOutboxConfiguratorExtensions -{ - public static IOutboxConfigurator AddConsumer(this IOutboxConfigurator configurator) - where T : class, IOutboxConsumer - { - configurator.Services.AddScoped(); - return configurator; - } - - public static IOutboxConfigurator UseMongo(this IOutboxConfigurator configurator, string connectionString) - { - configurator.Services.AddMongoDataAccess( - connectionString, - "SIL.ServiceToolkit.Models", - o => - { - o.AddRepository( - "outbox_messages", - mapSetup: m => m.MapProperty(m => m.OutboxRef).SetSerializer(new StringSerializer()) - ); - o.AddRepository( - "outboxes", - mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) - ); - } - ); - return configurator; - } - - public static IOutboxConfigurator UseDeliveryService(this IOutboxConfigurator configurator) - { - configurator.Services.AddHostedService(); - return configurator; - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IServiceCollectionsExtensions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IServiceCollectionsExtensions.cs deleted file mode 100644 index d4941ca26..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IServiceCollectionsExtensions.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServiceCollectionExtensions -{ - public static IServiceCollection AddParallelCorpusService(this IServiceCollection services) - { - services.TryAddSingleton(); - return services; - } - - public static IServiceCollection AddFileSystem(this IServiceCollection services) - { - services.TryAddTransient(); - return services; - } - - public static IServiceCollection AddOutbox(this IServiceCollection services, Action configure) - { - services.TryAddScoped(); - configure(new OutboxConfigurator(services)); - return services; - } - - public static IServiceCollection AddOutbox( - this IServiceCollection services, - IConfiguration configuration, - Action configure - ) - { - services.Configure(configuration.GetSection(OutboxOptions.Key)); - services.TryAddScoped(); - configure(new OutboxConfigurator(services)); - return services; - } - - public static IServiceCollection AddDiagnostics(this IServiceCollection services) => - services.AddHostedService(); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxConfigurator.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxConfigurator.cs deleted file mode 100644 index 0308c6c83..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxConfigurator.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace SIL.ServiceToolkit.Configuration; - -internal class OutboxConfigurator(IServiceCollection services) : IOutboxConfigurator -{ - public IServiceCollection Services { get; } = services; -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxOptions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxOptions.cs deleted file mode 100644 index 63fef7784..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/OutboxOptions.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace SIL.ServiceToolkit.Configuration; - -public class OutboxOptions -{ - public const string Key = "MessageOutbox"; - - public string OutboxDir { get; set; } = "outbox"; - public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); - public int HealthyMessageLimit { get; set; } = 25; -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Outbox.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Outbox.cs deleted file mode 100644 index 78f92814f..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/Outbox.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace SIL.ServiceToolkit.Models; - -public record Outbox : IEntity -{ - public string Id { get; set; } = ""; - - public int Revision { get; set; } - - public int CurrentIndex { get; init; } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/OutboxMessage.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/OutboxMessage.cs deleted file mode 100644 index 1a6757b45..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/OutboxMessage.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.ServiceToolkit.Models; - -public record OutboxMessage : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required int Index { get; init; } - public required string OutboxRef { get; init; } - public required string Method { get; init; } - public required string GroupId { get; init; } - public required string Content { get; init; } - public required bool HasContentStream { get; init; } - public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow; -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelCorpus.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelCorpus.cs deleted file mode 100644 index 833741629..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelCorpus.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.ServiceToolkit.Models; - -public record ParallelCorpus -{ - public required string Id { get; set; } - public IReadOnlyList SourceCorpora { get; set; } = new List(); - public IReadOnlyList TargetCorpora { get; set; } = new List(); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelRow.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelRow.cs deleted file mode 100644 index 13a40e319..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Models/ParallelRow.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace SIL.ServiceToolkit.Models; - -public record ParallelRow -{ - public required IReadOnlyList SourceRefs { get; init; } - public required IReadOnlyList TargetRefs { get; init; } - public required string TargetText { get; init; } - public required IReadOnlyList? SourceTokens { get; init; } - public required IReadOnlyList? TargetTokens { get; init; } - public IReadOnlyList? Alignment { get; init; } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Properties/AssemblyInfo.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Properties/AssemblyInfo.cs deleted file mode 100644 index 5ae15f09c..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Properties/AssemblyInfo.cs +++ /dev/null @@ -1 +0,0 @@ -[assembly: InternalsVisibleTo("SIL.ServiceToolkit.Tests")] diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj b/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj deleted file mode 100644 index 2f4434b1b..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs deleted file mode 100644 index c33f509b5..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public class CancellationInterceptor(ILogger logger) : Interceptor -{ - private readonly ILogger _logger = logger; - - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (Exception ex) - { - if (ex is OperationCanceledException) - { - _logger.LogInformation("An operation was canceled."); - throw new RpcException(new Status(StatusCode.Cancelled, "An operation was canceled.")); - } - else - { - throw; - } - } - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxConsumer.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxConsumer.cs deleted file mode 100644 index 445f46fd9..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public interface IOutboxConsumer -{ - public string OutboxId { get; } - public string Method { get; } - - public Type ContentType { get; } - - public Task HandleMessageAsync(object content, Stream? stream, CancellationToken cancellationToken = default); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxService.cs deleted file mode 100644 index e3d995a44..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IOutboxService.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public interface IOutboxService -{ - public Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - object content, - Stream? stream = null, - CancellationToken cancellationToken = default - ); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusService.cs deleted file mode 100644 index 3b26e2e16..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusService.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public interface IParallelCorpusService -{ - QuoteConventionAnalysis AnalyzeTargetQuoteConvention(IEnumerable parallelCorpora); - - IReadOnlyList<( - string ParallelCorpusId, - string MonolingualCorpusId, - IReadOnlyList Errors - )> AnalyzeUsfmVersification(IEnumerable parallelCorpora); - - IReadOnlyList<( - string ParallelCorpusId, - string MonolingualCorpusId, - MissingParentProjectError - )> FindMissingParentProjects(IEnumerable parallelCorpora); - - Task PreprocessAsync( - IEnumerable parallelCorpora, - Func train, - Func inference, - bool useKeyTerms = false, - HashSet? ignoreUsfmMarkers = null - ); - - string UpdateSourceUsfm( - IReadOnlyList parallelCorpora, - string corpusId, - string bookId, - IReadOnlyList rows, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior embedBehavior, - UpdateUsfmMarkerBehavior styleBehavior, - bool placeParagraphMarkers, - IEnumerable? remarks, - string? targetQuoteConvention - ); - - string UpdateTargetUsfm( - IReadOnlyList parallelCorpora, - string corpusId, - string bookId, - IReadOnlyList rows, - UpdateUsfmTextBehavior textBehavior, - UpdateUsfmMarkerBehavior paragraphBehavior, - UpdateUsfmMarkerBehavior embedBehavior, - UpdateUsfmMarkerBehavior styleBehavior, - IEnumerable? remarks, - string? targetQuoteConvention - ); - - Dictionary> GetChapters( - IReadOnlyList parallelCorpora, - string fileLocation, - string scriptureRange - ); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxConsumerBase.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxConsumerBase.cs deleted file mode 100644 index 8da32a2a8..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxConsumerBase.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public abstract class OutboxConsumerBase(string outboxId, string method) : IOutboxConsumer -{ - public string OutboxId { get; } = outboxId; - - public string Method { get; } = method; - - public Type ContentType => typeof(T); - - public Task HandleMessageAsync(object content, Stream? stream, CancellationToken cancellationToken = default) - { - return HandleMessageAsync((T)content, stream, cancellationToken); - } - - protected abstract Task HandleMessageAsync(T content, Stream? stream, CancellationToken cancellationToken); -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxDeliveryService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxDeliveryService.cs deleted file mode 100644 index 4157c5736..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxDeliveryService.cs +++ /dev/null @@ -1,222 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public class OutboxDeliveryService( - IServiceProvider services, - IFileSystem fileSystem, - IOptionsMonitor options, - ILogger logger -) : BackgroundService -{ - private readonly IServiceProvider _services = services; - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - private readonly ILogger _logger = logger; - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - Initialize(); - using IServiceScope scope = _services.CreateScope(); - var messages = scope.ServiceProvider.GetRequiredService>(); - Dictionary<(string, string), IOutboxConsumer> consumers = scope - .ServiceProvider.GetServices() - .ToDictionary(o => (o.OutboxId, o.Method)); - TimeSpan timeout = await ProcessMessagesAsync(consumers, messages, stoppingToken) - ? TimeSpan.Zero // Success - no timeout retry - : TimeSpan.FromSeconds(30); // Failed - retry after 30 seconds max - using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); - while (!stoppingToken.IsCancellationRequested) - { - try - { - // This token is used to retry messages according to an exponential backoff - // to ensure that messages that fail to send are resent in a timely manner, - // not just only when a new message arrives in the outbox. - using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); - if (timeout > TimeSpan.Zero) - cts.CancelAfter(timeout); - try - { - await subscription.WaitForChangeAsync(cancellationToken: cts.Token); - } - catch (OperationCanceledException) - when (!stoppingToken.IsCancellationRequested && cts.IsCancellationRequested) - { - // Timeout reached - } - - stoppingToken.ThrowIfCancellationRequested(); - if (await ProcessMessagesAsync(consumers, messages, stoppingToken)) - // Processed - No timeout - timeout = TimeSpan.Zero; - else if (timeout == TimeSpan.Zero) - // First failure - wait 30 seconds - timeout = TimeSpan.FromSeconds(30); - else if (timeout < TimeSpan.FromMinutes(15)) - // Exponential backoff for subsequent failures - timeout *= 2; - else - // Maximum timeout of 15 minutes - timeout = timeout = TimeSpan.FromMinutes(15); - } - catch (TimeoutException e) - { - _logger.LogWarning(e, "Change stream interrupted, trying again..."); - await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); - } - catch (OperationCanceledException e) - { - _logger.LogInformation(e, "Cancellation requested, service is stopping..."); - break; - } - } - } - - private void Initialize() - { - _fileSystem.CreateDirectory(_options.CurrentValue.OutboxDir); - } - - /// - /// Processes the messages. - /// - /// The consumers. - /// The messages. - /// The cancellation token. - /// - /// true if messages were successfully processed (not necessarily sent), otherwise false. - /// - internal async Task ProcessMessagesAsync( - Dictionary<(string, string), IOutboxConsumer> consumers, - IRepository messages, - CancellationToken cancellationToken = default - ) - { - bool anyMessages = await messages.ExistsAsync(m => true, cancellationToken); - if (!anyMessages) - return true; - - IReadOnlyList curMessages = await messages.GetAllAsync(cancellationToken); - - IEnumerable> messageGroups = curMessages - .OrderBy(m => m.Index) - .GroupBy(m => (m.OutboxRef, m.GroupId)); - - foreach (IGrouping<(string OutboxId, string GroupId), OutboxMessage> messageGroup in messageGroups) - { - bool abortMessageGroup = false; - foreach (OutboxMessage message in messageGroup) - { - try - { - await ProcessGroupMessagesAsync(consumers, messages, message, cancellationToken); - } - catch (RpcException e) - { - switch (e.StatusCode) - { - case StatusCode.Unavailable: - case StatusCode.Unauthenticated: - case StatusCode.PermissionDenied: - case StatusCode.Cancelled: - _logger.LogWarning(e, "Platform Message sending failure: {statusCode}", e.StatusCode); - return false; - case StatusCode.DeadlineExceeded: - case StatusCode.Internal: - case StatusCode.ResourceExhausted: - case StatusCode.Unknown: - abortMessageGroup = !await CheckIfFinalMessageAttemptAsync(messages, message, e); - break; - case StatusCode.Aborted: - case StatusCode.FailedPrecondition: - case StatusCode.InvalidArgument: - default: - // delete message and log error - await PermanentlyFailedMessageAsync(messages, message, e); - break; - } - } - catch (Exception e) - { - await PermanentlyFailedMessageAsync(messages, message, e); - break; - } - if (abortMessageGroup) - break; - } - } - - return true; - } - - private async Task ProcessGroupMessagesAsync( - Dictionary<(string, string), IOutboxConsumer> consumers, - IRepository messages, - OutboxMessage message, - CancellationToken cancellationToken = default - ) - { - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, message.Id); - IOutboxConsumer consumer = consumers[(message.OutboxRef, message.Method)]; - object content = OutboxService.DeserializeContent(message.Content, consumer.ContentType); - if (message.HasContentStream) - { - using Stream stream = _fileSystem.OpenRead(filePath); - await consumer.HandleMessageAsync(content, stream, cancellationToken); - } - else - { - await consumer.HandleMessageAsync(content, null, cancellationToken); - } - await messages.DeleteAsync(message.Id, CancellationToken.None); - _fileSystem.DeleteFile(filePath); - } - - private async Task CheckIfFinalMessageAttemptAsync( - IRepository messages, - OutboxMessage message, - Exception e - ) - { - if (message.Created < DateTimeOffset.UtcNow.Subtract(_options.CurrentValue.MessageExpirationTimeout)) - { - await PermanentlyFailedMessageAsync(messages, message, e); - return true; - } - else - { - LogFailedAttempt(message, e); - return false; - } - } - - private async Task PermanentlyFailedMessageAsync( - IRepository messages, - OutboxMessage message, - Exception e - ) - { - // log error - _logger.LogError( - e, - "Permanently failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Id, - message.Method, - message.Content, - e.Message - ); - await messages.DeleteAsync(message.Id); - } - - private void LogFailedAttempt(OutboxMessage message, Exception e) - { - // log error - _logger.LogError( - e, - "Failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Id, - message.Method, - message.Content, - e.Message - ); - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxHealthCheck.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxHealthCheck.cs deleted file mode 100644 index 437f4ca87..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxHealthCheck.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public class OutboxHealthCheck(IMemoryCache cache, IOptions options, IRepository messages) - : IHealthCheck -{ - private const string FailureCountKey = "OutboxHealthCheck.ConsecutiveFailures"; - private readonly AsyncLock _lock = new AsyncLock(); - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - int count = (await messages.GetAllAsync(cancellationToken)).Count; - if (count > options.Value.HealthyMessageLimit) - { - using (await _lock.LockAsync(cancellationToken)) - { - int numConsecutiveFailures = cache.Get(FailureCountKey); - cache.Set(FailureCountKey, ++numConsecutiveFailures); - return numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy("Outbox messages are accumulating.") - : HealthCheckResult.Degraded("Outbox messages are accumulating."); - } - } - using (await _lock.LockAsync(cancellationToken)) - cache.Set(FailureCountKey, 0); - return HealthCheckResult.Healthy("Outbox messages are being processed successfully"); - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxService.cs deleted file mode 100644 index e5ce94555..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/OutboxService.cs +++ /dev/null @@ -1,100 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -public class OutboxService( - IRepository outboxes, - IRepository messages, - IIdGenerator idGenerator, - IFileSystem fileSystem, - IOptionsMonitor options -) : IOutboxService -{ - private static readonly JsonSerializerOptions JsonSerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - }; - - static OutboxService() - { - JsonSerializerOptions.AddProtobufSupport(); - } - - public static string SerializeContent(object content) - { - return JsonSerializer.Serialize(content, JsonSerializerOptions); - } - - public static object DeserializeContent(string content, Type type) - { - object? result = JsonSerializer.Deserialize(content, type, JsonSerializerOptions); - if (result == null) - throw new InvalidOperationException("The JSON content cannot be null."); - return result; - } - - private readonly IRepository _outboxes = outboxes; - private readonly IRepository _messages = messages; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - internal int MaxDocumentSize { get; set; } = 1_000_000; - - public async Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - object content, - Stream? stream = null, - CancellationToken cancellationToken = default - ) - { - string serializedContent = SerializeContent(content); - if (serializedContent.Length > MaxDocumentSize) - { - throw new ArgumentException( - $"The content is too large for request {method} with group ID {groupId}. " - + $"It is {serializedContent.Length} bytes, but the maximum is {MaxDocumentSize} bytes." - ); - } - Outbox outbox = ( - await _outboxes.UpdateAsync( - outboxId, - u => u.Inc(o => o.CurrentIndex, 1), - upsert: true, - cancellationToken: cancellationToken - ) - )!; - OutboxMessage outboxMessage = new() - { - Id = _idGenerator.GenerateId(), - Index = outbox.CurrentIndex, - OutboxRef = outboxId, - Method = method, - GroupId = groupId, - Content = serializedContent, - HasContentStream = stream is not null, - }; - if (stream is null) - { - await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); - } - else - { - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, outboxMessage.Id); - try - { - await using (Stream fileStream = _fileSystem.OpenWrite(filePath)) - { - await stream.CopyToAsync(fileStream, cancellationToken); - } - await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); - return outboxMessage.Id; - } - catch - { - _fileSystem.DeleteFile(filePath); - throw; - } - } - return outboxMessage.Id; - } -} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs deleted file mode 100644 index b52b3f447..000000000 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs +++ /dev/null @@ -1,30 +0,0 @@ -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using System.Text.RegularExpressions; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Hangfire; -global using Microsoft.Extensions.Caching.Memory; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.DependencyInjection.Extensions; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Bson.Serialization.Serializers; -global using Nito.AsyncEx; -global using SIL.DataAccess; -global using SIL.Machine.Corpora; -global using SIL.Machine.PunctuationAnalysis; -global using SIL.ObjectModel; -global using SIL.ServiceToolkit.Configuration; -global using SIL.ServiceToolkit.Models; -global using SIL.ServiceToolkit.Services; -global using SIL.ServiceToolkit.Utils; -global using SIL.WritingSystems; diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj deleted file mode 100644 index 041e1c3b7..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj +++ /dev/null @@ -1,33 +0,0 @@ - - - - net10.0 - enable - enable - SIL.ServiceToolkit - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxDeliveryServiceTests.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxDeliveryServiceTests.cs deleted file mode 100644 index 9abe4a85d..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxDeliveryServiceTests.cs +++ /dev/null @@ -1,232 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -[TestFixture] -public class OutboxDeliveryServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method1 = "Method1"; - private const string Method2 = "Method2"; - - [Test] - public async Task ProcessMessagesAsync() - { - TestEnvironment env = new(); - env.AddStandardMessages(); - await env.ProcessMessagesAsync(); - Received.InOrder(() => - { - env.Consumer2.HandleMessageAsync("B", null, Arg.Any()); - env.Consumer1.HandleMessageAsync("A", null, Arg.Any()); - env.Consumer2.HandleMessageAsync("C", null, Arg.Any()); - }); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - } - - [Test] - public async Task ProcessMessagesAsync_Timeout() - { - TestEnvironment env = new(); - env.AddStandardMessages(); - - // Timeout is long enough where the message attempt will be incremented, but not deleted. - EnableConsumerFailure(env.Consumer2, StatusCode.Internal); - await env.ProcessMessagesAsync(); - - // with now shorter timeout, the messages will be deleted. - // 4 start build attempts, and only one build completed attempt - env.Options.CurrentValue.Returns(new OutboxOptions { MessageExpirationTimeout = TimeSpan.FromMilliseconds(1) }); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Consumer1.Received(1) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Consumer2.Received(4) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_UnavailableFailure() - { - TestEnvironment env = new(); - env.AddStandardMessages(); - - EnableConsumerFailure(env.Consumer2, StatusCode.Unavailable); - await env.ProcessMessagesAsync(); - // Only the first group should be attempted - _ = env - .Consumer1.DidNotReceive() - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Consumer2.Received(1) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - - env.Consumer2.ClearReceivedCalls(); - EnableConsumerFailure(env.Consumer2, StatusCode.Internal); - await env.ProcessMessagesAsync(); - _ = env - .Consumer1.DidNotReceive() - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Consumer2.Received(2) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - - env.Consumer2.ClearReceivedCalls(); - DisableConsumerFailure(env.Consumer2); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Consumer1.Received(1) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Consumer2.Received(2) - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_File() - { - TestEnvironment env = new(); - env.AddContentStreamMessages(); - - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Consumer1.Received(1) - .HandleMessageAsync("A", Arg.Is(s => s != null), Arg.Any()); - env.FileSystem.Received().DeleteFile(Path.Combine("outbox", "A")); - } - - private static void EnableConsumerFailure(IOutboxConsumer consumer, StatusCode code) - { - consumer - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - consumer - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - } - - private static void DisableConsumerFailure(IOutboxConsumer consumer) - { - consumer - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - consumer - .HandleMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - } - - private class TestEnvironment - { - private readonly Dictionary<(string, string), IOutboxConsumer> _consumers; - - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - - Consumer1 = CreateConsumer(OutboxId, Method1); - Consumer2 = CreateConsumer(OutboxId, Method2); - _consumers = new Dictionary<(string, string), IOutboxConsumer> - { - { (OutboxId, Method1), Consumer1 }, - { (OutboxId, Method2), Consumer2 }, - }; - - FileSystem = Substitute.For(); - Options = Substitute.For>(); - Options.CurrentValue.Returns(new OutboxOptions()); - - Service = new OutboxDeliveryService( - Substitute.For(), - FileSystem, - Options, - Substitute.For>() - ); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public OutboxDeliveryService Service { get; } - public IOutboxConsumer Consumer1 { get; } - public IOutboxConsumer Consumer2 { get; } - public IOptionsMonitor Options { get; } - public IFileSystem FileSystem { get; } - - public Task ProcessMessagesAsync() - { - return Service.ProcessMessagesAsync(_consumers, Messages); - } - - public void AddStandardMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "\"A\"", - HasContentStream = false, - } - ); - Messages.Add( - new OutboxMessage - { - Id = "B", - Index = 1, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "A", - Content = "\"B\"", - HasContentStream = false, - } - ); - Messages.Add( - new OutboxMessage - { - Id = "C", - Index = 3, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "B", - Content = "\"C\"", - HasContentStream = false, - } - ); - } - - public void AddContentStreamMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "\"A\"", - HasContentStream = true, - } - ); - FileSystem - .OpenRead(Path.Combine("outbox", "A")) - .Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes("Content"))); - } - - private static IOutboxConsumer CreateConsumer(string outboxId, string method) - { - var consumer = Substitute.For(); - consumer.OutboxId.Returns(outboxId); - consumer.Method.Returns(method); - consumer.ContentType.Returns(typeof(string)); - return consumer; - } - } -} diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxServiceTests.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxServiceTests.cs deleted file mode 100644 index 6eb4e46da..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/OutboxServiceTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -[TestFixture] -public class OutboxServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method = "TestMethod"; - - [Test] - public async Task EnqueueMessageAsync_NoContentStream() - { - TestEnvironment env = new(); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(1)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("\"content\"")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_ExistingOutbox() - { - TestEnvironment env = new(); - env.Outboxes.Add(new Outbox { Id = OutboxId, CurrentIndex = 1 }); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(2)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(2)); - Assert.That(message.Content, Is.EqualTo("\"content\"")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_HasContentStream() - { - TestEnvironment env = new(); - await using MemoryStream fileStream = new(); - env.FileSystem.OpenWrite(Path.Combine("outbox", "1")).Returns(fileStream); - - await using MemoryStream stream = new(Encoding.UTF8.GetBytes("content")); - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content", stream); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("\"content\"")); - Assert.That(message.HasContentStream, Is.True); - Assert.That(fileStream.ToArray(), Is.EqualTo(stream.ToArray())); - } - - [Test] - public void EnqueueMessageAsync_ContentTooLarge() - { - TestEnvironment env = new(); - env.Service.MaxDocumentSize = 5; - - Assert.ThrowsAsync(() => env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content")); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - var idGenerator = Substitute.For(); - idGenerator.GenerateId().Returns("1"); - FileSystem = Substitute.For(); - var options = Substitute.For>(); - options.CurrentValue.Returns(new OutboxOptions()); - Service = new OutboxService(Outboxes, Messages, idGenerator, FileSystem, options); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public IFileSystem FileSystem { get; } - public OutboxService Service { get; } - } -} diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusServiceTests.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusServiceTests.cs deleted file mode 100644 index a0a018502..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusServiceTests.cs +++ /dev/null @@ -1,356 +0,0 @@ -namespace SIL.ServiceToolkit.Services; - -[TestFixture] -public class ParallelCorpusServiceTests -{ - [Test] - public void AnalyzeTargetQuoteConvention_FileFormatParatext() - { - using var env = new TestEnvironment(); - ParallelCorpus parallelCorpus = env.GetCorpora(paratextProject: true).First(); - const string ExpectedTargetName = "typewriter_english"; - - QuoteConventionAnalysis? targetQuotationConvention = env.Processor.AnalyzeTargetQuoteConvention([ - parallelCorpus, - ]); - - Assert.Multiple(() => - { - Assert.That(targetQuotationConvention, Is.Not.Null); - Assert.That(targetQuotationConvention!.BestQuoteConvention.Name, Is.EqualTo(ExpectedTargetName)); - }); - } - - [Test] - public void AnalyzeTargetQuoteConvention_FileFormatText() - { - using var env = new TestEnvironment(); - ParallelCorpus parallelCorpus = env.GetCorpora(paratextProject: false).First(); - - QuoteConventionAnalysis? targetQuotationConvention = env.Processor.AnalyzeTargetQuoteConvention([ - parallelCorpus, - ]); - - Assert.Multiple(() => - { - Assert.That(targetQuotationConvention, Is.Not.Null); - Assert.That(targetQuotationConvention!.BestQuoteConvention, Is.Null); - }); - } - - [Test] - public async Task Preprocess_FileFormatText() - { - using var env = new TestEnvironment(); - IReadOnlyList corpora = env.GetCorpora(paratextProject: false); - int trainCount = 0; - int inferenceCount = 0; - await env.Processor.PreprocessAsync( - corpora, - (row, _) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - return Task.CompletedTask; - }, - (row, isInTrainingData, _) => - { - if (row.SourceSegment.Length > 0 && !isInTrainingData) - { - inferenceCount++; - } - - return Task.CompletedTask; - }, - false - ); - - Assert.Multiple(() => - { - Assert.That(trainCount, Is.EqualTo(2)); - Assert.That(inferenceCount, Is.EqualTo(3)); - }); - } - - [Test] - public async Task TestPreprocess_FileFormatParatext() - { - using var env = new TestEnvironment(); - IReadOnlyList corpora = env.GetCorpora(paratextProject: true); - int trainCount = 0; - int inferenceCount = 0; - var trainRefs = new List(); - var inferenceRefs = new List(); - await env.Processor.PreprocessAsync( - corpora, - (row, _) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - { - trainCount++; - trainRefs.Add(row.TargetRefs[0].ToString() ?? ""); - } - return Task.CompletedTask; - }, - (row, isInTrainingData, _) => - { - if (row.SourceSegment.Length > 0 && !isInTrainingData) - { - inferenceCount++; - inferenceRefs.Add(row.TargetRefs[0].ToString() ?? ""); - } - - return Task.CompletedTask; - }, - false, - ["mt"] - ); - - Assert.Multiple(() => - { - Assert.That(trainCount, Is.EqualTo(5)); - Assert.That(inferenceCount, Is.EqualTo(16)); - }); - } - - [Test] - public void FindMissingParentProjects() - { - using var env = new TestEnvironment(); - ParallelCorpus parallelCorpus = env.GetCorpora(paratextProject: true).First(); - - IReadOnlyList<(string ParallelCorpusId, string MonolingualCorpusId, MissingParentProjectError error)> errors = - env.Processor.FindMissingParentProjects([parallelCorpus]); - - Assert.That(errors, Has.Count.EqualTo(0)); - } - - [Test] - public void FindMissingParentProjects_MissingParent() - { - using var env = new TestEnvironment(); - ParallelCorpus parallelCorpus = env.GetCorpora(paratextProject: true).Last(); - - IReadOnlyList<(string ParallelCorpusId, string MonolingualCorpusId, MissingParentProjectError Error)> errors = - env.Processor.FindMissingParentProjects([parallelCorpus]); - - Assert.That(errors, Has.Count.EqualTo(1)); - Assert.That(errors[0].Error.ProjectName, Is.EqualTo("Te2")); - Assert.That(errors[0].Error.ParentProjectName, Is.EqualTo("Te1")); - } - - [Test] - public void AnalyzeUsfmVersification() - { - using var env = new TestEnvironment(); - ParallelCorpus parallelCorpus = env.GetCorpora(paratextProject: true).First(); - - IReadOnlyList<( - string ParallelCorpusId, - string MonolingualCorpusId, - IReadOnlyList UsfmErrors - )> errors = env.Processor.AnalyzeUsfmVersification([parallelCorpus]); - - Assert.That(errors, Has.Count.EqualTo(1)); - Assert.That(errors[0].UsfmErrors, Has.Count.EqualTo(3)); - Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(UsfmVersificationErrorType.MissingVerse)); - Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(UsfmVersificationErrorType.MissingVerse)); - Assert.That(errors[0].UsfmErrors[0].Type, Is.EqualTo(UsfmVersificationErrorType.MissingVerse)); - } - - private class TestEnvironment : DisposableBase - { - private static readonly string TestDataPath = Path.Combine( - AppContext.BaseDirectory, - "..", - "..", - "..", - "Services", - "data" - ); - private readonly TempDirectory _tempDir = new TempDirectory(name: "ParallelCorpusServiceTests"); - - public IParallelCorpusService Processor { get; } = new ParallelCorpusService(); - - public ParallelCorpus[] GetCorpora(bool paratextProject) - { - if (paratextProject) - { - return - [ - new ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - [ - new MonolingualCorpus - { - Id = "pt-source1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Paratext, - Location = ZipParatextProject("pt-source1"), - }, - ], - InferenceTextIds = [], - }, - ], - TargetCorpora = - [ - new MonolingualCorpus - { - Id = "pt-target1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Paratext, - Location = ZipParatextProject("pt-target1"), - }, - ], - }, - ], - }, - new ParallelCorpus - { - Id = "corpus2", - SourceCorpora = - [ - new MonolingualCorpus - { - Id = "pt-source1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Paratext, - Location = ZipParatextProject("pt-source1"), - }, - ], - TrainOnTextIds = [], - }, - ], - TargetCorpora = - [ - new MonolingualCorpus - { - Id = "pt-target1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Paratext, - Location = ZipParatextProject("pt-target1"), - }, - ], - TrainOnTextIds = [], - }, - ], - }, - new ParallelCorpus - { - Id = "corpus3", - SourceCorpora = [], - TargetCorpora = - [ - new MonolingualCorpus - { - Id = "pt-target2", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Paratext, - Location = ZipParatextProject("pt-target1"), - }, - ], - TrainOnTextIds = [], - }, - ], - }, - ]; - } - - return - [ - new ParallelCorpus - { - Id = "corpus1", - SourceCorpora = - [ - new MonolingualCorpus - { - Id = "source-corpus1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "source1.txt"), - }, - ], - }, - new MonolingualCorpus - { - Id = "source-corpus2", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "source2.txt"), - }, - ], - }, - ], - TargetCorpora = - [ - new MonolingualCorpus - { - Id = "target-corpus1", - Language = "en", - Files = - [ - new CorpusFile - { - TextId = "textId1", - Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "target1.txt"), - }, - ], - }, - ], - }, - ]; - } - - protected override void DisposeManagedResources() - { - _tempDir.Dispose(); - } - - private string ZipParatextProject(string name) - { - string fileName = Path.Combine(_tempDir.Path, $"{name}.zip"); - if (!File.Exists(fileName)) - ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), fileName); - return fileName; - } - } -} diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM deleted file mode 100644 index b86652904..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM +++ /dev/null @@ -1,8 +0,0 @@ -\id LEV - Test -\h Leviticus -\mt Leviticus -\c 14 -\p -\v 55 Source one, chapter fourteen, verse fifty-five. -\v 55b Segment b. -\v 56 Source one, chapter fourteen, verse fifty-six. diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM deleted file mode 100644 index 4eb8b5fd2..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM +++ /dev/null @@ -1,12 +0,0 @@ -\id 1CH - Test -\h 1 Chronicles -\mt 1 Chronicles -\c 12 -\p -\v 1 Source one, chapter twelve, verse one. -\v 2 Source one, chapter twelve, verse two. -\v 3-7 Source one, chapter twelve, verses three through seven. -\v 8 Source one, chapter twelve, verse eight. -\c 13 -\p -\v 1 Source one, chapter thirteen, verse one. diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM deleted file mode 100644 index b5ba96515..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM +++ /dev/null @@ -1,19 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Source one, chapter one, verse one. -\v 2-3 Source one, chapter one, verse two and three. -\v 4 Source one, chapter one, verse four. -\v 5 Source one, chapter one, verse five. -\v 6 Source one, chapter one, verse six. -\v 7-9 Source one, chapter one, verse seven, eight, and nine. -\v 10 Source one, chapter one, verse ten. -\c 2 -\p -\v 1 Source one, chapter two, verse one. -\v 2 Source one, chapter two, verse two. “a quotation” -\v 3 ... -\v 4 ... diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM deleted file mode 100644 index ff8aaf6e2..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml deleted file mode 100644 index c80caedfa..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml +++ /dev/null @@ -1,34 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test1 - 65001 - T - - NFC - Te1 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Tes.SFM - Major::BiblicalTerms.xml - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml deleted file mode 100644 index b5c2bb971..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - Abraham - - - - - - - Zedekiah - - - - - - diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs deleted file mode 100644 index 9c1cd3873..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM deleted file mode 100644 index 7efc55394..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM +++ /dev/null @@ -1,18 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Target one, chapter one, verse one. -\v 2 Target one, chapter one, verse two. -\v 3 Target one, chapter one, verse three. -\v 4 -\v 5-6 Target one, chapter one, verse five and six. -\v 7-8 Target one, chapter one, verse seven and eight. -\v 9-10 Target one, chapter one, verse nine and ten. -\c 2 -\p -\v 1 Target one, chapter two, verse one. -\v 2 ... -\v 3 Target one, chapter two, verse three. "a quotation" diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM deleted file mode 100644 index 460009633..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark \ No newline at end of file diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml deleted file mode 100644 index b5c2bb971..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - Abraham - - - - - - - Zedekiah - - - - - - diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs deleted file mode 100644 index 9c1cd3873..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source1.txt b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source1.txt deleted file mode 100644 index 2aeb971ce..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source one, Line 1 -Source one, Line 2 - -Source one, Line 4 - -Source one, Line 6 - diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source2.txt b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source2.txt deleted file mode 100644 index 7f4a06693..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/source2.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source two, Line 1 -Source two, Line 2 - -Source two, Line 4 -Source two, Line 5 -Source two, Line 6 - diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/target1.txt b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/target1.txt deleted file mode 100644 index 816e94355..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/target1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Target one, Line 1 - - -Target one, Line 4 - - -Target one, Line 7 diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs deleted file mode 100644 index 19d74f19e..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs +++ /dev/null @@ -1,16 +0,0 @@ -global using System.IO.Compression; -global using System.Text; -global using Grpc.Core; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using NSubstitute; -global using NSubstitute.ExceptionExtensions; -global using NUnit.Framework; -global using NUnit.Framework.Constraints; -global using SIL.DataAccess; -global using SIL.Machine.Corpora; -global using SIL.Machine.PunctuationAnalysis; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.ServiceToolkit.Configuration; -global using SIL.ServiceToolkit.Models; diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Utils/NUnitExtensions.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Utils/NUnitExtensions.cs deleted file mode 100644 index e52803012..000000000 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Utils/NUnitExtensions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace SIL.ServiceToolkit.Utils; - -public static class NUnitExtensions -{ - public static EqualUsingConstraint IgnoreLineEndings(this EqualStringConstraint constraint) - { - return constraint.Using(new IgnoreLineEndingsStringComparer()); - } -}