diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 22edb610..644e5a8e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,7 +4,7 @@ on: push: branches: [main,development] pull_request: - branches: [main,development] + branches: [main, development, 'development-*'] jobs: lint: @@ -23,7 +23,7 @@ jobs: uses: astral-sh/setup-uv@v6 - name: Install linter - run: uv pip install --system ruff + run: uv pip install --system ruff==0.16.1 - name: Run linter run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 281b6b16..af033e0f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ on: - 'alembic.ini' - '.github/workflows/tests.yml' pull_request: - branches: [main,development] + branches: [main, development, 'development-*'] paths: - '**.py' - 'requirements.txt' diff --git a/Makefile b/Makefile index fd8a6947..6f45c119 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ help: @echo "make docs - Serve interactive API docs from the OpenAPI contract" @echo "make migrate - Run pending Alembic migrations" @echo "make migration - Generate new migration (msg='description')" + @echo "make generate-contract-models - Regenerate Pydantic models from the incident contract" @echo "make clean - Stop containers (preserves volumes)" @echo "make super-clean - [CAUTION] Stop containers, delete volumes, prune Docker" @@ -114,7 +115,11 @@ migrate: @$(COMPOSE) exec -T app alembic upgrade head migration: - @$(COMPOSE) exec -T app alembic revision --autogenerate -m "$(msg)" + @test -n "$(rev)" || { echo "rev is required, e.g. make migration rev=003 msg=\"...\""; exit 1; } + @$(COMPOSE) exec -T app alembic revision --autogenerate --rev-id "$(rev)" -m "$(msg)" + +generate-contract-models: + @$(COMPOSE) exec -T app sh -c "uv pip install --system -r requirements-dev.txt && python3 scripts/generate_contract_models.py" clean: @$(COMPOSE) down diff --git a/alembic/versions/003_move_incident_contract_onto_incident_row.py b/alembic/versions/003_move_incident_contract_onto_incident_row.py new file mode 100644 index 00000000..f2793670 --- /dev/null +++ b/alembic/versions/003_move_incident_contract_onto_incident_row.py @@ -0,0 +1,73 @@ +"""move incident contract onto incident row + +Revision ID: 003 +Revises: 002 +Create Date: 2026-07-30 09:26:53.879663 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '003' +down_revision: Union[str, None] = '002' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('extractions', sa.Column('partial_result', sa.JSON(), nullable=True)) + op.drop_column('extractions', 'incident_contract') + op.add_column('incidents', sa.Column('incident_contract', sa.JSON(), nullable=True)) + op.add_column('incidents', sa.Column('incident_category', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + op.add_column('incidents', sa.Column('incident_datetime', sa.DateTime(), nullable=True)) + op.add_column('incidents', sa.Column('city', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + op.add_column('incidents', sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + op.add_column('incidents', sa.Column('country', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + op.add_column('incidents', sa.Column('civilian_injuries', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('civilian_fatalities', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('responder_injuries', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('responder_fatalities', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('people_rescued', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('people_evacuated', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('structures_destroyed', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('area_burned_ha', sa.Float(), nullable=True)) + op.add_column('incidents', sa.Column('total_loss_amount', sa.Float(), nullable=True)) + op.add_column('incidents', sa.Column('total_loss_currency', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + op.add_column('incidents', sa.Column('call_to_arrival_seconds', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('turnout_seconds_first_unit', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('travel_seconds_first_unit', sa.Integer(), nullable=True)) + op.add_column('incidents', sa.Column('on_scene_duration_seconds', sa.Integer(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('incidents', 'on_scene_duration_seconds') + op.drop_column('incidents', 'travel_seconds_first_unit') + op.drop_column('incidents', 'turnout_seconds_first_unit') + op.drop_column('incidents', 'call_to_arrival_seconds') + op.drop_column('incidents', 'total_loss_currency') + op.drop_column('incidents', 'total_loss_amount') + op.drop_column('incidents', 'area_burned_ha') + op.drop_column('incidents', 'structures_destroyed') + op.drop_column('incidents', 'people_evacuated') + op.drop_column('incidents', 'people_rescued') + op.drop_column('incidents', 'responder_fatalities') + op.drop_column('incidents', 'responder_injuries') + op.drop_column('incidents', 'civilian_fatalities') + op.drop_column('incidents', 'civilian_injuries') + op.drop_column('incidents', 'country') + op.drop_column('incidents', 'state') + op.drop_column('incidents', 'city') + op.drop_column('incidents', 'incident_datetime') + op.drop_column('incidents', 'incident_category') + op.drop_column('incidents', 'incident_contract') + op.add_column('extractions', sa.Column('incident_contract', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.drop_column('extractions', 'partial_result') + # ### end Alembic commands ### diff --git a/alembic/versions/004_form_templates.py b/alembic/versions/004_form_templates.py new file mode 100644 index 00000000..ef23be52 --- /dev/null +++ b/alembic/versions/004_form_templates.py @@ -0,0 +1,78 @@ +"""form templates registry and pdf upload drafts + +Revision ID: 004 +Revises: 003 +Create Date: 2026-08-10 + +Two tables, both new in the contract Layer 6 template work. + +`form_templates` is the registry, keyed by `form_type` and distinct from the +legacy `template` table (int PK + uploaded PDF). Its `fields` JSON column holds +the TemplateField list, each with a nested `layout`. `form_type` stays a plain +VARCHAR rather than a Postgres enum, because jurisdictions register their own +form types and those are not part of the built-in FormType enum. + +`template_uploads` holds the drafts behind the PDF authoring flow: the stored +blank PDF, its page geometry, and the fields commonforms detected. Rows here +are working state, not templates. Registering a template copies the edited +fields across and keeps only the `pdf_template_ref` pointing back. + +Both JSON columns use sa.JSON for consistency with migrations 001-003 and for +the SQLite test harness. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '004' +down_revision: Union[str, None] = '003' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('form_templates', + sa.Column('template_id', sa.Uuid(), nullable=False), + sa.Column('form_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('display_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('jurisdiction', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('agency_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('fields', sa.JSON(), nullable=False), + sa.Column('source_standard', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('version', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('template_id') + ) + op.create_index(op.f('ix_form_templates_form_type'), 'form_templates', ['form_type'], unique=True) + op.create_table('template_uploads', + sa.Column('upload_id', sa.Uuid(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('pdf_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('original_filename', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('page_count', sa.Integer(), nullable=False), + sa.Column('pages', sa.JSON(), nullable=False), + sa.Column('detected_fields', sa.JSON(), nullable=True), + sa.Column('detection_error', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('job_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('upload_id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('template_uploads') + op.drop_index(op.f('ix_form_templates_form_type'), table_name='form_templates') + op.drop_table('form_templates') + # ### end Alembic commands ### diff --git a/alembic/versions/005_form_incident_template_batch.py b/alembic/versions/005_form_incident_template_batch.py new file mode 100644 index 00000000..24603214 --- /dev/null +++ b/alembic/versions/005_form_incident_template_batch.py @@ -0,0 +1,59 @@ +"""forms: key by template, require incident, group by batch, drop extract_id + +Revision ID: 005 +Revises: 004 +Create Date: 2026-08-17 + +Form generation (contract Layer 3, #552) keys a form by which template filled +it and reaches the extraction through the incident rather than a direct link +(incidents already FK the extraction, so extract_id was a redundant hop). +Batching is a grouping key only — batch_id has no Batch table, batch status is +derived on the fly from the Form rows that share an id. + +No route or repository writes a v1 Form row yet (form generation itself is +still unimplemented), so the forms table is empty in every environment and +the NOT NULL adds below are safe — there is no existing data to backfill. + +FK constraints are named explicitly, matching Postgres' own default +`__fkey` pattern, so downgrade can drop them by name. The same +naming_convention is handed to batch_alter_table so SQLite's reflect-and- +recreate path (needed for the incident_id nullable flip and the extract_id +drop, neither of which SQLite can ALTER in place) assigns identical names. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '005' +down_revision: Union[str, None] = '004' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +NAMING_CONVENTION = {"fk": "%(table_name)s_%(column_0_name)s_fkey"} + + +def upgrade() -> None: + with op.batch_alter_table('forms', naming_convention=NAMING_CONVENTION) as batch_op: + batch_op.add_column(sa.Column('template_id', sa.Uuid(), nullable=False)) + batch_op.create_foreign_key( + 'forms_template_id_fkey', 'form_templates', ['template_id'], ['template_id'] + ) + batch_op.add_column(sa.Column('batch_id', sa.Uuid(), nullable=True)) + batch_op.drop_constraint('forms_extract_id_fkey', type_='foreignkey') + batch_op.drop_column('extract_id') + batch_op.alter_column('incident_id', existing_type=sa.Uuid(), nullable=False) + + +def downgrade() -> None: + with op.batch_alter_table('forms', naming_convention=NAMING_CONVENTION) as batch_op: + batch_op.alter_column('incident_id', existing_type=sa.Uuid(), nullable=True) + batch_op.add_column(sa.Column('extract_id', sa.Uuid(), nullable=False)) + batch_op.create_foreign_key( + 'forms_extract_id_fkey', 'extractions', ['extract_id'], ['extract_id'] + ) + batch_op.drop_constraint('forms_template_id_fkey', type_='foreignkey') + batch_op.drop_column('template_id') + batch_op.drop_column('batch_id') diff --git a/alembic/versions/006_incidents_crud_indexes.py b/alembic/versions/006_incidents_crud_indexes.py new file mode 100644 index 00000000..53b58fe6 --- /dev/null +++ b/alembic/versions/006_incidents_crud_indexes.py @@ -0,0 +1,35 @@ +"""incidents crud indexes + +Revision ID: 006 +Revises: 005 +Create Date: 2026-08-22 06:49:34.422095 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '006' +down_revision: Union[str, None] = '005' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_index('ix_incidents_live_datetime', 'incidents', ['deleted_at', 'incident_datetime'], unique=False) + op.create_index('ix_incidents_number_live', 'incidents', ['incident_number'], unique=True, postgresql_where=sa.text('incident_number IS NOT NULL AND deleted_at IS NULL'), sqlite_where=sa.text('incident_number IS NOT NULL AND deleted_at IS NULL')) + op.drop_column('incidents', 'incident_date') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('incidents', sa.Column('incident_date', sa.DATE(), autoincrement=False, nullable=True)) + op.drop_index('ix_incidents_number_live', table_name='incidents', postgresql_where=sa.text('incident_number IS NOT NULL AND deleted_at IS NULL'), sqlite_where=sa.text('incident_number IS NOT NULL AND deleted_at IS NULL')) + op.drop_index('ix_incidents_live_datetime', table_name='incidents') + # ### end Alembic commands ### diff --git a/app/api/router.py b/app/api/router.py index cc59847a..d19767f9 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,13 +1,33 @@ from fastapi import APIRouter -from app.api.routes import forms, input, jobs, system, templates, weather, zipcode +from app.api.routes import ( + extraction, + form_generation, + forms, + form_templates, + incidents, + input, + jobs, + system, + weather, + zipcode, +) from app.core.config import API_PREFIX api_router = APIRouter() -api_router.include_router(templates.router, prefix=API_PREFIX) +api_router.include_router(form_templates.router, prefix=API_PREFIX) api_router.include_router(forms.router, prefix=API_PREFIX) +# v1 form generation — same "/forms" prefix as the legacy router above, kept +# in a separate file/router rather than added to forms.py. Included AFTER +# `forms` on purpose: the legacy router's literal GET paths (/forms/models, +# /forms/submissions, ...) must be matched before this router's catch-all +# GET /forms/{form_id}, same reasoning form_templates.py uses for /pdf vs +# /{template_id} — otherwise "models"/"submissions" would be read as a form_id. +api_router.include_router(form_generation.router, prefix=API_PREFIX) api_router.include_router(system.router, prefix=API_PREFIX) api_router.include_router(jobs.router, prefix=API_PREFIX) api_router.include_router(weather.router, prefix=API_PREFIX) api_router.include_router(zipcode.router, prefix=API_PREFIX) -api_router.include_router(input.router, prefix=API_PREFIX) \ No newline at end of file +api_router.include_router(input.router, prefix=API_PREFIX) +api_router.include_router(extraction.router, prefix=API_PREFIX) +api_router.include_router(incidents.router, prefix=API_PREFIX) \ No newline at end of file diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py index 2264aee0..17c68a8e 100644 --- a/app/api/routes/__init__.py +++ b/app/api/routes/__init__.py @@ -1,3 +1,3 @@ -from . import templates, forms +from . import form_templates, forms -__all__ = ["templates", "forms"] +__all__ = ["form_templates", "forms"] diff --git a/app/api/routes/extraction.py b/app/api/routes/extraction.py new file mode 100644 index 00000000..28f26eab --- /dev/null +++ b/app/api/routes/extraction.py @@ -0,0 +1,198 @@ +from uuid import UUID + +from fastapi import APIRouter, Body, Depends +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.enums import ExtractionStatus, InputStatus +from app.api.schemas.extraction import ( + ExtractionCompleted, + ExtractionJobResponse, + ExtractionProcessing, + ExtractionRequest, + ReadinessMatrix, + ValidationRequest, + ValidationResult, +) +from app.api.schemas.incident_contract import IncidentContract +from app.core.config import ( + ESTIMATED_EXTRACTION_SECONDS, + EXTRACTION_ALLOW_RERUN, + EXTRACTION_POLL_INTERVAL_SECONDS, +) +from app.core.errors.base import AppError +from app.db.repositories import ( + get_extraction, + get_extraction_by_input, + get_incident_by_extract, + get_input, + list_form_templates, +) +from app.models import Extraction, Incident +from app.services.extraction.service import ExtractionService +from app.services.extraction_readiness import readiness_matrix, validate_template +from app.services.extraction_review import ExtractionReviewService, load_for_review +from app.services.form_templates import require_template +from app.services import llm + +router = APIRouter(prefix="/extract", tags=["extraction"]) + +MERGE_PATCH_MEDIA_TYPE = "application/merge-patch+json" + + +def _completed_response(extraction: Extraction, incident: Incident | None) -> ExtractionCompleted: + """The completed shape, with the contract read from the incident row.""" + contract = IncidentContract.model_validate( + (incident.incident_contract if incident else None) or {} + ) + return ExtractionCompleted( + extract_id=extraction.extract_id, + input_id=extraction.input_id, + incident_id=incident.incident_id if incident else None, + status="completed", + incident_contract=contract, + completed_at=extraction.completed_at, + model_used=extraction.model_used, + processing_time_seconds=extraction.processing_time_seconds, + corrections=extraction.corrections, + ) + + +def _load_extraction(db: Session, extract_id: UUID) -> Extraction: + extraction = get_extraction(db, extract_id) + if extraction is None: + raise AppError( + f"Extraction with ID {extract_id} not found", + status_code=404, + error_code="EXTRACT_NOT_FOUND", + ) + return extraction + + +@router.post("/{input_id}", response_model=ExtractionJobResponse, status_code=202) +def create_extraction( + input_id: UUID, + body: ExtractionRequest | None = None, + db: Session = Depends(get_db), +): + record = get_input(db, input_id) + if record is None: + raise AppError( + f"Input with ID {input_id} not found", + status_code=404, + error_code="INPUT_NOT_FOUND", + ) + + if record.status != InputStatus.ready: + raise AppError( + f"Input is in '{record.status}' state. Wait until status is 'ready'.", + status_code=409, + error_code="INPUT_NOT_READY", + detail={"current_status": record.status}, + ) + + existing = get_extraction_by_input(db, input_id) + if existing is not None and not EXTRACTION_ALLOW_RERUN: + raise AppError( + "An extraction already exists for this input", + status_code=409, + error_code="EXTRACTION_EXISTS", + detail={"existing_extract_id": str(existing.extract_id)}, + ) + + provider = llm.health() + if provider.status == "unhealthy": + raise AppError( + f"The {provider.label} LLM service is not available", + status_code=503, + error_code="LLM_UNAVAILABLE", + detail={"provider": provider.provider, "reason": provider.detail}, + ) + + svc = ExtractionService() + extraction, job = svc.start_extraction( + db, + input_id, + model_override=body.model_override if body else None, + defaults=body.defaults if body else None, + hints=body.extraction_hints if body else None, + ) + + return ExtractionJobResponse( + extract_id=extraction.extract_id, + input_id=input_id, + job_id=job.job_id, + status=extraction.status, + queued_at=extraction.created_at, + estimated_seconds=ESTIMATED_EXTRACTION_SECONDS, + poll_url=f"/api/v1/extract/{extraction.extract_id}", + ) + + +@router.get("/{extract_id}", response_model=ExtractionCompleted | ExtractionProcessing) +def get_extraction_result(extract_id: UUID, db: Session = Depends(get_db)): + extraction = _load_extraction(db, extract_id) + + if extraction.status == ExtractionStatus.completed: + return _completed_response(extraction, get_incident_by_extract(db, extract_id)) + + retry_after = ( + EXTRACTION_POLL_INTERVAL_SECONDS + if extraction.status == ExtractionStatus.processing + else None + ) + partial = ( + IncidentContract.model_validate(extraction.partial_result) + if extraction.partial_result + else None + ) + return ExtractionProcessing( + extract_id=extraction.extract_id, + input_id=extraction.input_id, + status=extraction.status, + started_at=extraction.started_at, + retry_after_seconds=retry_after, + error_type=extraction.error_type, + error_detail=extraction.error_detail, + partial_result=partial, + ) + + +@router.patch("/{extract_id}", response_model=ExtractionCompleted) +def update_extraction( + extract_id: UUID, + patch: dict = Body( + ..., + media_type=MERGE_PATCH_MEDIA_TYPE, + description="JSON Merge Patch (RFC 7396) shaped like the incident contract. " + "Only send the fields that changed; a null deletes a field.", + ), + db: Session = Depends(get_db), +): + extraction = _load_extraction(db, extract_id) + incident = load_for_review(extraction, get_incident_by_extract(db, extract_id)) + + extraction, incident = ExtractionReviewService().apply_patch( + db, extraction, incident, patch + ) + return _completed_response(extraction, incident) + + +@router.get("/{extract_id}/readiness", response_model=ReadinessMatrix) +def get_readiness(extract_id: UUID, db: Session = Depends(get_db)): + extraction = _load_extraction(db, extract_id) + incident = load_for_review(extraction, get_incident_by_extract(db, extract_id)) + + return readiness_matrix(extraction, incident, list_form_templates(db)) + + +@router.post("/{extract_id}/validate", response_model=ValidationResult) +def validate_extraction( + extract_id: UUID, + body: ValidationRequest, + db: Session = Depends(get_db), +): + extraction = _load_extraction(db, extract_id) + incident = load_for_review(extraction, get_incident_by_extract(db, extract_id)) + + return validate_template(extraction, incident, require_template(db, body.template_id)) diff --git a/app/api/routes/form_generation.py b/app/api/routes/form_generation.py new file mode 100644 index 00000000..fcfe1a91 --- /dev/null +++ b/app/api/routes/form_generation.py @@ -0,0 +1,226 @@ +"""Contract Layer 3 form generation endpoints (contracts/path/forms.yaml). + +Serves POST /forms/generate and the retrieval endpoints at /api/v1/forms, +backed by the v1 Form model. Handlers are thin; business logic lives in +app/services/form_generation.py (write path) and app/services/form_fill_worker.py +(the Celery-dispatched fill). Distinct from the legacy prototype routes in +app/api/routes/forms.py (int template_id, no incident/batch concept), which +stay mounted at the same "/forms" prefix unchanged. + +/batch/{batch_id} is declared before /{form_id} for the same reason +form_templates.py declares /pdf before /{template_id}: FastAPI matches paths +in declaration order, so the literal segment has to come first or "batch" +gets read as a form_id. +""" + +from uuid import UUID + +from fastapi import APIRouter, Depends, Response +from fastapi.responses import FileResponse, JSONResponse +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.enums import FormStatus +from app.api.schemas.form_generation import ( + BatchFormEntry, + BatchGenerateResponse, + BatchStatus, + FormMappedJson, + FormRecord, + GenerateFormsRequest, + QueuedForm, + SkippedForm, +) +from app.core.config import ( + ESTIMATED_FORM_GENERATION_SECONDS, + FORM_GENERATION_POLL_INTERVAL_SECONDS, +) +from app.core.errors.base import AppError +from app.db.repositories import get_form, list_forms_by_batch +from app.services.form_generation import ( + FormGenerationService, + batch_state, + batch_zip, + download_filename, + form_version, + resolve_form_pdf, +) + +router = APIRouter(prefix="/forms", tags=["forms"]) + + +@router.post("/generate", response_model=BatchGenerateResponse, status_code=202) +def generate_forms(body: GenerateFormsRequest, db: Session = Depends(get_db)): + result = FormGenerationService().start_generation(db, body) + return BatchGenerateResponse( + batch_id=result.batch_id, + incident_id=result.incident_id, + forms_queued=[ + QueuedForm(form_id=f.form_id, template_id=f.template_id, form_type=f.form_type) + for f in result.queued + ], + forms_skipped=[ + SkippedForm(template_id=s.template_id, form_type=s.form_type, reason=s.reason) + for s in result.skipped + ], + estimated_seconds=ESTIMATED_FORM_GENERATION_SECONDS, + poll_url=f"/api/v1/forms/batch/{result.batch_id}", + ) + + +@router.get("/batch/{batch_id}", response_model=BatchStatus) +def get_batch_status(batch_id: UUID, db: Session = Depends(get_db)): + forms = list_forms_by_batch(db, batch_id) + if not forms: + raise AppError(f"Batch {batch_id} not found", status_code=404, error_code="BATCH_NOT_FOUND") + + completed = sum(1 for f in forms if f.status == FormStatus.completed) + failed = sum(1 for f in forms if f.status == FormStatus.failed) + total = len(forms) + status = batch_state(forms) + + return BatchStatus( + batch_id=batch_id, + status=status, + total=total, + completed=completed, + failed=failed, + forms=[ + BatchFormEntry( + form_id=f.form_id, + template_id=f.template_id, + form_type=f.form_type, + status=f.status, + ) + for f in forms + ], + # Only offered once there is something to bundle: a batch where every + # form failed has no PDFs behind the link. + download_url=( + f"/api/v1/forms/batch/{batch_id}/download" if status == "completed" else None + ), + ) + + +@router.get("/batch/{batch_id}/download") +def download_batch_zip(batch_id: UUID, db: Session = Depends(get_db)): + forms = list_forms_by_batch(db, batch_id) + if not forms: + raise AppError(f"Batch {batch_id} not found", status_code=404, error_code="BATCH_NOT_FOUND") + + status = batch_state(forms) + if status == "processing": + return JSONResponse( + status_code=202, + content={ + "message": "Batch is still generating", + "status": status, + "retry_after_seconds": FORM_GENERATION_POLL_INTERVAL_SECONDS, + }, + ) + + if status == "failed": + raise AppError( + f"Every form in batch {batch_id} failed to generate", + status_code=500, + error_code="FORM_GENERATION_FAILED", + detail={"reason": "No PDFs were produced for this batch"}, + ) + + archive, filename = batch_zip(db, batch_id, forms) + return Response( + content=archive, + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@router.get("/{form_id}", response_model=FormRecord) +def get_form_record(form_id: UUID, db: Session = Depends(get_db)): + form = get_form(db, form_id) + if not form: + raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND") + + return FormRecord( + form_id=form.form_id, + template_id=form.template_id, + form_type=form.form_type, + status=form.status, + incident_id=form.incident_id, + batch_id=form.batch_id, + created_at=form.created_at, + completed_at=form.completed_at, + pdf_ready=form.pdf_ready, + json_ready=form.json_ready, + field_mapping_summary=form.field_mapping_summary, + ) + + +@router.get("/{form_id}/pdf", response_class=FileResponse) +def download_form_pdf(form_id: UUID, db: Session = Depends(get_db)): + form = get_form(db, form_id) + if not form: + raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND") + + if form.status == FormStatus.failed: + raise AppError( + f"Form {form_id} failed to generate", + status_code=500, + error_code="PDF_GENERATION_FAILED", + detail={"reason": "Form generation failed"}, + ) + + if not form.pdf_ready or not form.pdf_path: + return JSONResponse( + status_code=202, + content={ + "message": "Form generation is still in progress", + "status": form.status, + "retry_after_seconds": FORM_GENERATION_POLL_INTERVAL_SECONDS, + }, + ) + + path = resolve_form_pdf(form) + if path is None: + raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND") + + return FileResponse( + path, media_type="application/pdf", filename=download_filename(db, form) + ) + + +@router.get("/{form_id}/json", response_model=FormMappedJson) +def get_form_json(form_id: UUID, db: Session = Depends(get_db)): + form = get_form(db, form_id) + if not form: + raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND") + + # Same three answers as /pdf, so a client polling both after one generate + # call reads them the same way: 500 once the fill failed, 202 while it is + # still running, the payload once it is there. + if form.status == FormStatus.failed: + raise AppError( + f"Form {form_id} failed to generate", + status_code=500, + error_code="FORM_GENERATION_FAILED", + detail={"reason": "Form generation failed"}, + ) + + if not form.json_ready or form.json_data is None: + return JSONResponse( + status_code=202, + content={ + "message": "Form generation is still in progress", + "status": form.status, + "retry_after_seconds": FORM_GENERATION_POLL_INTERVAL_SECONDS, + }, + ) + + return FormMappedJson( + form_type=form.form_type, + form_version=form_version(db, form), + form_id=form.form_id, + template_id=form.template_id, + incident_id=form.incident_id, + agency_fields=form.json_data, + ) diff --git a/app/api/routes/form_templates.py b/app/api/routes/form_templates.py new file mode 100644 index 00000000..eec93076 --- /dev/null +++ b/app/api/routes/form_templates.py @@ -0,0 +1,127 @@ +"""Contract Layer 6 template registry endpoints (contracts/path/templates.yaml). + +Serves the form-template registry at /api/v1/templates, backed by the +UUID-keyed `FormTemplate` model, plus the PDF-authoring flow that feeds it: +upload a blank PDF, poll the detection draft, register the edited fields. +Handlers are thin, business logic lives in app/services/form_templates.py. The +legacy prototype template routes (upload / create / make-fillable / preview / +delete) were removed in the contract migration; the legacy int-PK `Template` +model survives only as the lookup target of the fill pipeline (forms.py / +jobs.py / tasks/fill.py). +""" + +from pathlib import Path +from uuid import UUID + +from fastapi import APIRouter, Depends, File, Form, Query, UploadFile +from fastapi.responses import FileResponse +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.templates import ( + CreateTemplateRequest, + TemplateDetail, + TemplateDraft, + TemplateDraftAccepted, + TemplateFieldsResponse, + TemplateSummary, +) +from app.core.config import MAX_TEMPLATE_PDF_BYTES +from app.core.errors.base import AppError +from app.services import form_templates as service + +router = APIRouter(prefix="/templates", tags=["templates"]) + +_PDF_MAGIC = b"%PDF-" + + +def _reject_if_too_large(size: int | None) -> None: + """Guard the 50MB cap. Checked once on the declared size before the body is + read into memory, and again on what actually arrived.""" + if size is None or size <= MAX_TEMPLATE_PDF_BYTES: + return + raise AppError( + "PDF exceeds maximum size of 50MB", + status_code=413, + error_code="FILE_TOO_LARGE", + detail={ + "max_size_bytes": MAX_TEMPLATE_PDF_BYTES, + "received_size_bytes": size, + }, + ) + + +@router.get("", response_model=list[TemplateSummary]) +def list_templates(db: Session = Depends(get_db)): + return service.list_templates(db) + + +@router.post("", response_model=TemplateDetail, status_code=201) +def create_template(body: CreateTemplateRequest, db: Session = Depends(get_db)): + return service.create_template(db, body) + + +# The two /pdf routes are declared before /{template_id} on purpose. FastAPI +# matches in declaration order, so the literal path has to come first or "pdf" +# gets read as a template id. +@router.post("/pdf", response_model=TemplateDraftAccepted, status_code=202) +def upload_template_pdf( + pdf_file: UploadFile = File(...), + detect_fields: bool = Form(default=True), + db: Session = Depends(get_db), +): + filename = pdf_file.filename or "" + + _reject_if_too_large(pdf_file.size) + + content = pdf_file.file.read() + if not content: + raise AppError( + "No PDF file was uploaded", + status_code=400, + error_code="MISSING_FILE", + ) + _reject_if_too_large(len(content)) + # Trust the bytes, not the extension or the declared content type. + if not content.startswith(_PDF_MAGIC): + raise AppError( + "Uploaded file is not a PDF", + status_code=415, + error_code="UNSUPPORTED_FORMAT", + detail={"accepted_formats": ["pdf"]}, + ) + + upload, job = service.store_upload(db, content, filename or None, detect_fields) + return service.draft_response(upload, job) + + +@router.get("/pdf/{upload_id}", response_model=TemplateDraft) +def get_template_draft(upload_id: UUID, db: Session = Depends(get_db)): + return service.get_draft(db, upload_id) + + +@router.get("/{template_id}", response_model=TemplateDetail) +def get_template(template_id: UUID, db: Session = Depends(get_db)): + return service.get_template(db, template_id) + + +@router.put("/{template_id}", response_model=TemplateDetail) +def replace_template( + template_id: UUID, body: CreateTemplateRequest, db: Session = Depends(get_db) +): + return service.replace_template(db, template_id, body) + + +@router.get("/{template_id}/fields", response_model=TemplateFieldsResponse) +def get_template_fields( + template_id: UUID, + required_only: bool = Query(False, description="Return only required fields"), + db: Session = Depends(get_db), +): + return service.get_template_fields(db, template_id, required_only) + + +@router.get("/{template_id}/pdf", response_class=FileResponse) +def download_template_pdf(template_id: UUID, db: Session = Depends(get_db)): + path: Path = service.resolve_template_pdf(db, template_id) + return FileResponse(path, media_type="application/pdf", filename=path.name) diff --git a/app/api/routes/forms.py b/app/api/routes/forms.py index be160291..ef6e8910 100644 --- a/app/api/routes/forms.py +++ b/app/api/routes/forms.py @@ -1,6 +1,5 @@ from datetime import datetime, timedelta, timezone from pathlib import Path -import requests from fastapi import APIRouter, Depends, File, UploadFile, Query from sqlmodel import Session, select @@ -11,11 +10,12 @@ ModelsResponse, TranscriptionResponse, ) -from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, BASE_DIR, RETENTION_PERIOD_DAYS +from app.core.config import BASE_DIR, RETENTION_PERIOD_DAYS from app.services.whisper import call_whisper_asr from app.core.errors.base import AppError from app.db.repositories import create_form, get_template, get_form_submission, delete_form_submission from app.models import FormSubmission, Template +from app.services import llm from app.services.controller import Controller PROJECT_ROOT = BASE_DIR @@ -66,24 +66,14 @@ def fill_form(form: FormFill, db: Session = Depends(get_db)): @router.get("/models", response_model=ModelsResponse) def list_models(): - """List the Whisper-independent extraction models available in the local - Ollama instance, plus the configured default. Used by the Fill Form UI's - model picker. Falls back to just the default if Ollama is unreachable.""" - default_model = OLLAMA_MODEL + """Models the configured provider will serve, for the Fill Form model picker. - models: list[str] = [] - try: - response = requests.get(f"{OLLAMA_HOST}/api/tags", timeout=10) - response.raise_for_status() - models = [m["name"] for m in response.json().get("models", []) if m.get("name")] - except requests.exceptions.RequestException: - models = [] - - # Always surface the configured default, even if Ollama hasn't pulled it yet. - if default_model not in models: - models.insert(0, default_model) - - return ModelsResponse(models=models, default=default_model) + Falls back to the configured default alone when the provider will not list + them. + """ + available = llm.list_models() + default = next((m.name for m in available if m.default), llm.get_settings().model) + return ModelsResponse(models=[m.name for m in available], default=default) @router.post("/transcribe", response_model=TranscriptionResponse) diff --git a/app/api/routes/incidents.py b/app/api/routes/incidents.py new file mode 100644 index 00000000..15403080 --- /dev/null +++ b/app/api/routes/incidents.py @@ -0,0 +1,165 @@ +"""Contract Layer 4 incident endpoints (contracts/path/incidents.yaml). + +Handlers are thin; the logic lives in app/services/incident_crud.py. The one +job kept here is assembling the response shapes, since the DB stores the +promoted analytics as flat columns while the contract nests them under +`analytics`. +""" + +from datetime import date +from math import ceil +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.common import Pagination +from app.api.schemas.enums import IncidentCategory, ReportStatus +from app.api.schemas.form_generation import FormRecord +from app.api.schemas.incidents import ( + CreateIncidentRequest, + DeleteIncidentResponse, + GeneratedForm, + IncidentAnalytics, + IncidentListItem, + IncidentListResponse, + IncidentRecord, + IncidentRecordFull, + SubmissionLogEntry, + UpdateIncidentRequest, +) +from app.api.schemas.incident_contract import IncidentContract +from app.models import Form, Incident +from app.services.incident_crud import IncidentService + +router = APIRouter(prefix="/incidents", tags=["incidents"]) + + +def _form_summary(form: Form) -> GeneratedForm: + return GeneratedForm(form_id=form.form_id, form_type=form.form_type, status=form.status) + + +def _record(incident: Incident, forms: list[Form]) -> IncidentRecord: + """The incident row as the contract's IncidentRecord.""" + return IncidentRecord( + incident_id=incident.incident_id, + extract_id=incident.extract_id, + incident_number=incident.incident_number, + status=incident.status, + incident_name=incident.incident_name, + incident_type=incident.incident_type, + incident_category=incident.incident_category, + incident_datetime=incident.incident_datetime, + analytics=IncidentAnalytics.model_validate(incident), + forms_generated=[_form_summary(f) for f in forms], + tags=incident.tags or [], + notes=incident.notes, + created_at=incident.created_at, + updated_at=incident.updated_at, + deleted_at=incident.deleted_at, + ) + + +def _submission_log(contract: dict | None) -> list[SubmissionLogEntry]: + """Submissions read out of the contract document. + + Empty until the submission layer exists; nothing writes it today. + """ + entries = (contract or {}).get("submission_log") + if not isinstance(entries, list): + return [] + return [SubmissionLogEntry.model_validate(e) for e in entries if isinstance(e, dict)] + + +@router.post("", response_model=IncidentRecord, status_code=201) +def create_incident(body: CreateIncidentRequest, db: Session = Depends(get_db)): + service = IncidentService() + incident = service.finalize(db, body) + return _record(incident, service.forms(db, incident.incident_id)) + + +@router.get("", response_model=IncidentListResponse) +def list_incidents( + db: Session = Depends(get_db), + date_from: date | None = Query(default=None), + date_to: date | None = Query(default=None), + incident_category: IncidentCategory | None = Query(default=None), + status: ReportStatus | None = Query(default=None), + page: int = Query(default=1, ge=1), + per_page: int = Query(default=20, ge=1, le=100), + sort: str = Query(default="date_desc", pattern="^(date_asc|date_desc)$"), +): + rows, counts, total = IncidentService().list_page( + db, + date_from=date_from, + date_to=date_to, + incident_category=incident_category, + status=status, + page=page, + per_page=per_page, + sort=sort, + ) + total_pages = ceil(total / per_page) if total else 0 + return IncidentListResponse( + data=[ + IncidentListItem( + incident_id=row.incident_id, + incident_number=row.incident_number, + status=row.status, + incident_name=row.incident_name, + incident_type=row.incident_type, + incident_category=row.incident_category, + incident_datetime=row.incident_datetime, + city=row.city, + country=row.country, + forms_count=counts.get(row.incident_id, 0), + created_at=row.created_at, + ) + for row in rows + ], + pagination=Pagination( + total=total, + page=page, + per_page=per_page, + total_pages=total_pages, + has_next=page < total_pages, + has_prev=page > 1, + ), + ) + + +@router.get("/{incident_id}", response_model=IncidentRecordFull) +def get_incident(incident_id: UUID, db: Session = Depends(get_db)): + service = IncidentService() + incident = service.get(db, incident_id) + forms = service.forms(db, incident_id) + base = _record(incident, forms) + return IncidentRecordFull( + **base.model_dump(), + incident_contract=( + IncidentContract.model_validate(incident.incident_contract) + if incident.incident_contract + else None + ), + forms=[FormRecord.model_validate(f, from_attributes=True) for f in forms], + submission_log=_submission_log(incident.incident_contract), + ) + + +@router.patch("/{incident_id}", response_model=IncidentRecord) +def update_incident( + incident_id: UUID, body: UpdateIncidentRequest, db: Session = Depends(get_db) +): + service = IncidentService() + incident = service.update(db, incident_id, body) + return _record(incident, service.forms(db, incident_id)) + + +@router.delete("/{incident_id}", response_model=DeleteIncidentResponse) +def delete_incident(incident_id: UUID, db: Session = Depends(get_db)): + incident = IncidentService().soft_delete(db, incident_id) + return DeleteIncidentResponse( + incident_id=incident.incident_id, + deleted_at=incident.deleted_at, + ) diff --git a/app/api/routes/system.py b/app/api/routes/system.py index eadd1d9f..174f5f7b 100644 --- a/app/api/routes/system.py +++ b/app/api/routes/system.py @@ -7,7 +7,7 @@ import time import requests -from fastapi import APIRouter +from fastapi import APIRouter, Query from fastapi.responses import JSONResponse from sqlalchemy import text @@ -15,9 +15,11 @@ ComponentHealth, HealthComponents, HealthStatus, - ModelInfo, + SchemaFieldEntry, + SchemaFieldSearchResponse, ) -from app.core.config import APP_VERSION, DATA_DIR, OLLAMA_HOST, WHISPER_HOST +from app.services import field_catalog, llm +from app.core.config import APP_VERSION, DATA_DIR, WHISPER_HOST from app.db.database import engine router = APIRouter(tags=["system"]) @@ -41,59 +43,32 @@ def _check_database() -> ComponentHealth: return ComponentHealth(status="unhealthy", detail=str(exc)) -def _check_ollama() -> ComponentHealth: - t0 = time.monotonic() - try: - tags_resp = requests.get(f"{OLLAMA_HOST}/api/tags", timeout=_PROBE_TIMEOUT) - tags_resp.raise_for_status() - elapsed = int((time.monotonic() - t0) * 1000) - - tags_data = tags_resp.json() - - ollama_version: str | None = None - try: - ver_resp = requests.get(f"{OLLAMA_HOST}/api/version", timeout=_PROBE_TIMEOUT) - ver_resp.raise_for_status() - ollama_version = ver_resp.json().get("version") - except Exception: - pass - - running_names: set[str] = set() - model_loaded: str | None = None - try: - ps_resp = requests.get(f"{OLLAMA_HOST}/api/ps", timeout=_PROBE_TIMEOUT) - ps_resp.raise_for_status() - running = ps_resp.json().get("models", []) - running_names = {m.get("name", "") for m in running} - if running: - model_loaded = running[0].get("name") - except Exception: - pass - - raw_models = tags_data.get("models", []) - models_available: list[ModelInfo] = [] - for m in raw_models: - name = m.get("name", "") - size_gb = round(m.get("size", 0) / (1024 ** 3), 2) - quantization = m.get("details", {}).get("quantization_level") - models_available.append( - ModelInfo(name=name, size_gb=size_gb, quantization=quantization, loaded=name in running_names) - ) - - status = "degraded" if elapsed > _SLOW_MS else "healthy" - - return ComponentHealth( - status=status, - response_time_ms=elapsed, - model_loaded=model_loaded, - ollama_version=ollama_version, - models_available=models_available if models_available else None, - # current_load is always None — Ollama exposes no queue-depth API; - # populating it would require fabricating data. - current_load=None, - ) - except requests.exceptions.RequestException as exc: - return ComponentHealth(status="unhealthy", detail=str(exc)) +def _check_llm() -> ComponentHealth: + """Health for whichever provider this deployment is configured for. + + A local provider is probed. A hosted one is not, so the model list is left + out there too: both cost quota to answer a question the configuration + already answers. + """ + report = llm.health() + status = report.status + if status == "healthy" and report.response_time_ms and report.response_time_ms > _SLOW_MS: + status = "degraded" + + models: list[str] | None = None + if report.probed and status != "unhealthy": + models = [model.name for model in llm.list_models()] + + return ComponentHealth( + status=status, + response_time_ms=report.response_time_ms, + detail=report.detail, + provider=report.provider, + model=report.model, + external=report.external, + probed=report.probed, + models_available=models, + ) def _check_whisper() -> ComponentHealth: @@ -128,15 +103,15 @@ def _check_storage() -> ComponentHealth: ) def get_health(): database = _check_database() - ollama = _check_ollama() + provider = _check_llm() whisper = _check_whisper() storage = _check_storage() components = HealthComponents( - database=database, ollama=ollama, whisper=whisper, storage=storage + database=database, llm=provider, whisper=whisper, storage=storage ) - statuses = {database.status, ollama.status, whisper.status, storage.status} + statuses = {database.status, provider.status, whisper.status, storage.status} if database.status == "unhealthy": overall = "unhealthy" @@ -179,3 +154,35 @@ def get_schema_versions(): "message": "Schema version history not yet available — see issue #555", }, ) + + +@router.get( + "/schema/fields", + response_model=SchemaFieldSearchResponse, + summary="Search or list the incident-contract field catalog", +) +def search_schema_fields( + q: str | None = Query(None, description="Search text, matched against names, aliases and descriptions"), + section: str | None = Query(None, description="Restrict to one top-level contract section"), + limit: int = Query(20, ge=1, le=100, description="Caps search results, ignored when q is omitted"), +): + hits = field_catalog.search(q, section, limit) + return SchemaFieldSearchResponse( + query=q, + total=len(hits), + schema_version=field_catalog.schema_version(), + fields=[ + SchemaFieldEntry( + path=entry.path, + label=entry.label, + field_type=entry.field_type, + section=entry.section, + description=entry.description, + enum_values=list(entry.enum_values) if entry.enum_values else None, + pii=entry.pii, + aliases=list(entry.aliases), + score=score, + ) + for entry, score in hits + ], + ) diff --git a/app/api/routes/templates.py b/app/api/routes/templates.py deleted file mode 100644 index ee7189aa..00000000 --- a/app/api/routes/templates.py +++ /dev/null @@ -1,240 +0,0 @@ -import re -from datetime import datetime, timezone -from pathlib import Path - -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile -from fastapi.responses import FileResponse -from sqlmodel import Session - -from app.api.deps import get_db, verify_api_key -from app.api.schemas.templates import ( - TemplateCreate, - TemplateResponse, - TemplateUploadResponse, - MakeFillableRequest, - MakeFillableResponse, -) -from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR -from app.db.repositories import create_template, list_templates, get_template, delete_template -from app.models import Template, FormSubmission, Job -from app.services.controller import Controller -from sqlmodel import select - -router = APIRouter(prefix="/templates", tags=["templates"]) -PROJECT_ROOT = BASE_DIR - - -def _resolve_target_directory(directory: str) -> Path: - dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() - if not dir_value: - raise HTTPException(status_code=400, detail="Directory is required.") - - candidate = Path(dir_value) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Directory must be inside the project.") - - return candidate - - -def _resolve_project_file(file_path: str) -> Path: - raw_path = (file_path or "").strip() - if not raw_path: - raise HTTPException(status_code=400, detail="Path is required.") - - candidate = Path(raw_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Path must be inside the project.") - - return candidate - - -@router.post("/upload", response_model=TemplateUploadResponse) -async def upload_template_pdf( - file: UploadFile = File(...), - directory: str = Form(DEFAULT_TEMPLATE_DIR), -): - filename = Path(file.filename or "").name - if not filename: - raise HTTPException(status_code=400, detail="A PDF filename is required.") - - if not filename.lower().endswith(".pdf"): - raise HTTPException(status_code=400, detail="Only PDF files are supported.") - - target_dir = _resolve_target_directory(directory) - target_dir.mkdir(parents=True, exist_ok=True) - - target_path = target_dir / filename - if target_path.exists(): - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" - - content = await file.read() - with target_path.open("wb") as output_file: - output_file.write(content) - - relative_path = target_path.relative_to(PROJECT_ROOT).as_posix() - extracted = _extract_pdf_fields(relative_path) - return TemplateUploadResponse( - filename=target_path.name, - pdf_path=relative_path, - field_count=None if extracted is None else len(extracted), - fields=extracted or [], - ) - - -# PDF field-type codes -> the type values the frontend field builder uses. -_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} - - -def _pdf_text(value) -> str: - """Decode a pdfrw string (field name / tooltip) to plain text.""" - if value is None: - return "" - if hasattr(value, "to_unicode"): - return value.to_unicode().strip() - return str(value).strip() - - -def _humanize(name: str) -> str: - """Turn a raw field name into a readable description (JobTitle -> Job Title).""" - text = re.sub(r"_+", " ", name) - text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) - return re.sub(r"\s+", " ", text).strip() - - -def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: - """Fillable widgets in the same order Filler.fill_form writes them - (top-to-bottom, left-to-right per page), so seeded rows line up with the - fill order. Returns None if the PDF can't be read.""" - try: - from pdfrw import PdfReader - candidate = Path(pdf_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - pdf = PdfReader(str(candidate)) - fields: list[dict] = [] - for page in pdf.pages: - widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] - widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) - for annot in widgets: - name = _pdf_text(annot.T) - fields.append({ - "name": name, - "description": _pdf_text(annot.TU) or _humanize(name), - "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), - }) - return fields - except Exception: - return None - - -def _count_pdf_widgets(pdf_path: str) -> int | None: - """Number of fillable widgets in a PDF, or None if unreadable.""" - fields = _extract_pdf_fields(pdf_path) - return None if fields is None else len(fields) - - -@router.get("", response_model=list[TemplateResponse]) -def get_templates(db: Session = Depends(get_db)): - return list_templates(db) - - -@router.get("/preview") -def preview_template_pdf(path: str = Query(..., description="Project-relative PDF path")): - resolved_path = _resolve_project_file(path) - - if not resolved_path.exists() or not resolved_path.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - if resolved_path.suffix.lower() != ".pdf": - raise HTTPException(status_code=400, detail="Only PDF files can be previewed.") - - return FileResponse( - resolved_path, - media_type="application/pdf", - filename=resolved_path.name, - content_disposition_type="inline", - ) - - -@router.post("/create", response_model=TemplateResponse) -def create(template: TemplateCreate, db: Session = Depends(get_db)): - tpl = Template(**template.model_dump()) - created = create_template(db, tpl) - return TemplateResponse( - id=created.id, - name=created.name, - pdf_path=created.pdf_path, - fields=created.fields, - field_count=_count_pdf_widgets(created.pdf_path), - ) - - -@router.post("/make-fillable", response_model=MakeFillableResponse) -def make_fillable(req: MakeFillableRequest): - # Validate the path stays inside the project root. - resolved = _resolve_project_file(req.pdf_path) - if not resolved.exists() or not resolved.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - controller = Controller() - new_absolute = controller.prepare_fillable(str(resolved)) - new_path = Path(new_absolute) - if not new_path.is_absolute(): - new_path = (PROJECT_ROOT / new_path).resolve() - relative_path = new_path.relative_to(PROJECT_ROOT).as_posix() - - return MakeFillableResponse( - pdf_path=relative_path, - field_count=_count_pdf_widgets(relative_path), - ) - - -@router.delete("/{template_id}", dependencies=[Depends(verify_api_key)]) -def delete_template_endpoint(template_id: int, db: Session = Depends(get_db)): - template = get_template(db, template_id) - if not template: - raise HTTPException(status_code=404, detail="Template not found") - - # 1. Clean up associated submissions and their generated PDFs - sub_stmt = select(FormSubmission).where(FormSubmission.template_id == template_id) - submissions = list(db.exec(sub_stmt)) - for sub in submissions: - if sub.output_pdf_path: - try: - resolved_out = _resolve_project_file(sub.output_pdf_path) - if resolved_out.exists() and resolved_out.is_file(): - resolved_out.unlink() - except Exception: - pass - db.delete(sub) - - # 2. Clean up associated jobs - job_stmt = select(Job).where(Job.template_id == template_id) - jobs = list(db.exec(job_stmt)) - for job in jobs: - db.delete(job) - - # 3. Delete template PDF file - if template.pdf_path: - try: - resolved_pdf = _resolve_project_file(template.pdf_path) - if resolved_pdf.exists() and resolved_pdf.is_file(): - resolved_pdf.unlink() - except Exception: - pass - - # 4. Delete the template itself - delete_template(db, template) - return {"status": "success", "message": "Template and all associated data deleted"} - diff --git a/app/api/schemas/enums.py b/app/api/schemas/enums.py index b6335389..4a2a3438 100644 --- a/app/api/schemas/enums.py +++ b/app/api/schemas/enums.py @@ -47,6 +47,7 @@ class JobType(str, Enum): form_generation = "form_generation" batch_form_generation = "batch_form_generation" report_generation = "report_generation" + template_field_detection = "template_field_detection" class FormType(str, Enum): @@ -72,15 +73,52 @@ class FormType(str, Enum): state_new_york = "state_new_york" +class DetectionStatus(str, Enum): + """Field-detection state of an uploaded template PDF. The PDF itself is + stored before any of this runs, so a failed detection is recoverable.""" + + processing = "processing" + completed = "completed" + failed = "failed" + + +class TemplateStatus(str, Enum): + active = "active" + legacy = "legacy" + draft = "draft" + + +class TextAlign(str, Enum): + left = "left" + center = "center" + right = "right" + + +class TemplateFieldType(str, Enum): + string = "string" + integer = "integer" + number = "number" + boolean = "boolean" + date = "date" + datetime = "datetime" + time = "time" + enum = "enum" + text = "text" + array = "array" + + class IncidentCategory(str, Enum): fire = "fire" + overpressure_explosion = "overpressure_explosion" ems = "ems" rescue = "rescue" hazardous_conditions = "hazardous_conditions" service_call = "service_call" good_intent = "good_intent" false_alarm = "false_alarm" + natural_disaster = "natural_disaster" law_enforcement = "law_enforcement" + special_incident = "special_incident" class CauseCertainty(str, Enum): @@ -94,7 +132,9 @@ class InjurySeverity(str, Enum): minor = "minor" moderate = "moderate" severe = "severe" + life_threatening = "life_threatening" fatal = "fatal" + undetermined = "undetermined" class RateOfSpread(str, Enum): @@ -114,3 +154,10 @@ class OutputFormat(str, Enum): pdf = "pdf" json = "json" both = "both" + + +class FieldSource(str, Enum): + schema = "schema" + static = "static" + manual = "manual" + open = "open" diff --git a/app/api/schemas/extraction.py b/app/api/schemas/extraction.py new file mode 100644 index 00000000..80a4bb22 --- /dev/null +++ b/app/api/schemas/extraction.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from app.api.schemas.enums import FieldSource +from app.api.schemas.incident_contract import IncidentContract + + +# --------------------------------------------------------------------------- +# Request +# --------------------------------------------------------------------------- + +class ExtractionHints(BaseModel): + """Optional hints to improve extraction accuracy. + + Extra keys are allowed; the contract marks this object + additionalProperties: true. + """ + + model_config = ConfigDict(extra="allow") + + incident_type: str | None = None + state: str | None = None + agency_type: str | None = None + + +class ExtractionDefaults(BaseModel): + """Deployment context the extractor falls back on when the narrative is + silent: timezone for relative dates, country, and currency for Money.""" + + country: str | None = None + timezone: str | None = None + currency: str | None = None + + +class ExtractionRequest(BaseModel): + model_override: str | None = None + extraction_hints: ExtractionHints | None = None + defaults: ExtractionDefaults | None = None + + +# --------------------------------------------------------------------------- +# Responses +# --------------------------------------------------------------------------- + +class ExtractionJobResponse(BaseModel): + """202 body for POST /extract/{input_id}. Carries the ids the client needs + to poll the extraction plus the underlying async job.""" + + extract_id: UUID + input_id: UUID + job_id: str + job_type: str = "extraction" + status: str + queued_at: datetime | None = None + estimated_seconds: int | None = None + poll_url: str + + +class Correction(BaseModel): + """One manual correction applied to the contract via PATCH.""" + + field_path: str | None = None + original_value: Any = None + corrected_value: Any = None + corrected_at: datetime | None = None + corrected_by: str | None = None + + +class ExtractionCompleted(BaseModel): + """A completed extraction. The contract document is embedded here, read + from the linked incident row; the extraction itself keeps only job + metadata and the corrections audit trail.""" + + extract_id: UUID + input_id: UUID + incident_id: UUID + status: Literal["completed"] + incident_contract: IncidentContract + completed_at: datetime | None = None + model_used: str | None = None + processing_time_seconds: float | None = None + corrections: list[Correction] | None = None + + +class ExtractionProcessing(BaseModel): + extract_id: UUID + input_id: UUID + status: Literal["processing", "failed"] + started_at: datetime | None = None + retry_after_seconds: int | None = None + error_type: str | None = None + error_detail: str | None = None + partial_result: IncidentContract | None = None + + +# --------------------------------------------------------------------------- +# Validation and readiness +# --------------------------------------------------------------------------- + +class FieldGap(BaseModel): + """One template field with no value yet, with enough context for the UI to + explain it and offer the right fix.""" + + field_name: str + source: FieldSource + incident_mapping: str | None = None + description: str | None = None + + +class ValidationRequest(BaseModel): + """Which registered template to check the extraction against.""" + + template_id: UUID + + +class ValidationResult(BaseModel): + valid: bool + template_id: UUID + extract_id: UUID + # form_type is an open string, not a closed enum: users register their own + # templates and each carries its own label. + form_type: str | None = None + missing_required: list[FieldGap] | None = None + missing_recommended: list[FieldGap] | None = None + warnings: list[str] | None = None + field_coverage_percent: float | None = None + + +class TemplateReadiness(BaseModel): + template_id: UUID + form_type: str + display_name: str + ready: bool + missing_required: list[FieldGap] | None = None + missing_recommended: list[FieldGap] | None = None + field_coverage_percent: float | None = None + + +class ReadinessMatrix(BaseModel): + """Per-template fill readiness for one extraction, computed by comparing + the contract against every registered template's field list.""" + + extract_id: UUID + templates: list[TemplateReadiness] + computed_at: datetime | None = None diff --git a/app/api/schemas/form_generation.py b/app/api/schemas/form_generation.py new file mode 100644 index 00000000..b82cc748 --- /dev/null +++ b/app/api/schemas/form_generation.py @@ -0,0 +1,141 @@ +"""Contract Layer 3 form generation schemas (contracts/schemas/form-record.yaml). + +Separate from app/api/schemas/forms.py, which holds the legacy prototype +fill-pipeline shapes (int template_id, no incident/batch concept) still served +by the old routes in app/api/routes/forms.py. This file is the v1 contract +shape only — mirrors the extraction.py / templates.py split, one file per +contract domain. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + +from app.api.schemas.enums import FormStatus, OutputFormat + + +# --------------------------------------------------------------------------- +# Request +# --------------------------------------------------------------------------- + +class GenerateFormsOptions(BaseModel): + output_format: OutputFormat | None = None + force_partial: bool = False + + +class GenerateFormsRequest(BaseModel): + """POST /forms/generate body. + + Omitting template_ids generates every template the readiness matrix + reports as ready. An empty list is a different thing from an absent one + and is rejected: it reads as a selection screen that sent nothing. + """ + + incident_id: UUID + template_ids: list[UUID] | None = Field(default=None, min_length=1) + options: GenerateFormsOptions | None = None + + +# --------------------------------------------------------------------------- +# Responses +# --------------------------------------------------------------------------- + +class QueuedForm(BaseModel): + form_id: UUID + template_id: UUID + form_type: str + + +class SkippedForm(BaseModel): + template_id: UUID + form_type: str + reason: str + + +class BatchGenerateResponse(BaseModel): + """202 body for POST /forms/generate.""" + + batch_id: UUID + status: Literal["processing"] = "processing" + incident_id: UUID + forms_queued: list[QueuedForm] = Field(default_factory=list) + forms_skipped: list[SkippedForm] = Field(default_factory=list) + estimated_seconds: int | None = None + poll_url: str + + +class FieldMappingSummary(BaseModel): + total_form_fields: int + fields_filled: int + fields_blank: int + coverage_percent: float + + +class FormRecord(BaseModel): + """GET /forms/{form_id} response.""" + + form_id: UUID + template_id: UUID + # form_type is an open string on the wire: registries can add form types + # the closed FormType enum does not know about yet (see FormTemplate.form_type). + form_type: str + status: FormStatus + incident_id: UUID + batch_id: UUID | None = None + created_at: datetime + completed_at: datetime | None = None + pdf_ready: bool + json_ready: bool + field_mapping_summary: FieldMappingSummary | None = None + + +class FormMappedJson(BaseModel): + """GET /forms/{form_id}/json response.""" + + form_type: str + # The template's version, read at request time. Null when the template has + # since been deleted out of the registry. + form_version: str | None = None + form_id: UUID + template_id: UUID + incident_id: UUID + agency_fields: dict = Field(default_factory=dict) + + +class BatchFormEntry(BaseModel): + form_id: UUID + template_id: UUID + form_type: str + status: FormStatus + + +class BatchStatus(BaseModel): + """GET /forms/batch/{batch_id} response, derived on the fly from the + batch's Form rows — there is no Batch table.""" + + batch_id: UUID + status: Literal["processing", "completed", "failed"] + total: int + completed: int + failed: int + forms: list[BatchFormEntry] = Field(default_factory=list) + # Set once the batch has finished with at least one PDF to bundle. + download_url: str | None = None + + +__all__ = [ + "GenerateFormsOptions", + "GenerateFormsRequest", + "QueuedForm", + "SkippedForm", + "BatchGenerateResponse", + "FieldMappingSummary", + "FormRecord", + "FormMappedJson", + "BatchFormEntry", + "BatchStatus", +] diff --git a/app/api/schemas/incident_contract.py b/app/api/schemas/incident_contract.py new file mode 100644 index 00000000..ce1094b4 --- /dev/null +++ b/app/api/schemas/incident_contract.py @@ -0,0 +1,2425 @@ +# This file is generated from contracts/schemas/incident-contract.yaml. +# DO NOT EDIT BY HAND. Run `make generate-contract-models` to regenerate. + +from __future__ import annotations + +from datetime import date as date_type, time as time_type +from enum import Enum +from typing import Annotated, Any, Optional +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, Field, RootModel +from app.api.schemas.enums import ( + CauseCertainty, + IncidentCategory, + InjurySeverity, + RateOfSpread, + ReportStatus, +) + + +class SchemaName(Enum): + """ + Schema name identifier + """ + + fireform_incident_contract = "fireform_incident_contract" + + +class Money(BaseModel): + """ + Monetary amount with ISO 4217 currency code + """ + + amount: Optional[float] = None + currency: Annotated[Optional[str], Field(None, examples=["USD"])] + """ + ISO 4217 code + """ + + +class Scheme(Enum): + """ + Coding scheme identifier + """ + + neris = "neris" + nfirs = "nfirs" + uk_irs = "uk_irs" + airs = "airs" + ontario_sir = "ontario_sir" + nemsis = "nemsis" + nibrs = "nibrs" + un_ssirs = "un_ssirs" + local = "local" + other = "other" + + +class CodeRef(BaseModel): + """ + A code from a named external coding scheme + """ + + scheme: Optional[Scheme] = None + """ + Coding scheme identifier + """ + code: Optional[str] = None + label: Optional[str] = None + + +class Quantity(BaseModel): + """ + Value with explicit reported unit, used where the unit itself is data (hazmat) + """ + + value: Optional[float] = None + unit: Optional[str] = None + """ + Unit as reported (e.g. l, kg, gal, lb, m3) + """ + + +class PresenceStatus(Enum): + present = "present" + absent = "absent" + undetermined = "undetermined" + + +class OperationStatus(Enum): + """ + Whether a protection system operated when exposed to the incident + """ + + operated = "operated" + failed_to_operate = "failed_to_operate" + fire_too_small_to_activate = "fire_too_small_to_activate" + not_reached_by_fire = "not_reached_by_fire" + undetermined = "undetermined" + + +class Coordinates(BaseModel): + latitude: Optional[float] = None + longitude: Optional[float] = None + accuracy_meters: Optional[float] = None + + +class InputType(Enum): + voice = "voice" + text = "text" + + +class Completeness(BaseModel): + """ + Extraction-quality summary, recalculated server-side after every + PATCH /extract. Per-template readiness is not stored here; it is + computed live by GET /extract/{extract_id}/readiness against the + registered templates. + + """ + + overall_percent: Annotated[Optional[int], Field(None, ge=0, le=100)] + missing_fields: Optional[list[str]] = None + """ + JSON paths of fields that have no value + """ + low_confidence_fields: Optional[list[str]] = None + """ + JSON paths of fields where LLM confidence is low + """ + inferred_fields: Optional[list[str]] = None + """ + JSON paths of fields that were inferred (not explicitly stated) + """ + + +class ExternalId(BaseModel): + scheme: Annotated[Optional[str], Field(None, examples=["irwin"])] + value: Optional[str] = None + + +class FormType(RootModel[Optional[str]]): + root: Annotated[Optional[str], Field(None, examples=["neris"])] = None + """ + Stable string identifier for a form type. An open string, not a closed + enum, because users register their own templates (any jurisdiction, any + agency) and each carries its own form_type label. Generation and + validation are keyed by template_id; form_type is a human-friendly label + and grouping key. + + Well-known built-in values: neris, nemsis_epcr, nibrs, nfirs_basic, + nfirs_fire, nfirs_structure, nfirs_wildland, nfirs_ems, nfirs_hazmat, + nfirs_apparatus, nfirs_personnel, nfirs_arson, nfirs_casualty_civilian, + nfirs_casualty_responder, cal_fire_ics209, osha_301, un_ssirs, + state_georgia, state_california, state_new_york. + + """ + + +class ReportingUnit(BaseModel): + station_name: Optional[str] = None + station_id: Optional[str] = None + agency_name: Optional[str] = None + agency_id: Optional[UUID] = None + agency_type: Optional[str] = None + + +class Personnel(BaseModel): + name: Optional[str] = None + badge_number: Optional[str] = None + rank: Optional[str] = None + role: Optional[str] = None + assignment: Optional[str] = None + contact_number: Optional[str] = None + signature_captured: Optional[bool] = None + + +class Reviewer(BaseModel): + name: Optional[str] = None + badge_number: Optional[str] = None + rank: Optional[str] = None + role: Optional[str] = None + reviewed_at: Optional[AwareDatetime] = None + approved: Optional[bool] = None + + +class Reason(Enum): + malicious = "malicious" + good_intent = "good_intent" + automatic_system_fault = "automatic_system_fault" + automatic_system_accidental = "automatic_system_accidental" + human_error = "human_error" + undetermined = "undetermined" + other = "other" + + +class FalseAlarm(BaseModel): + """ + Populated when the final type is a false alarm + """ + + reason: Optional[Reason] = None + reason_description: Optional[str] = None + + +class DelayIgnitionToDiscovery(Enum): + immediate = "immediate" + under_5_min = "under_5_min" + field_5_to_30_min = "5_to_30_min" + over_30_min = "over_30_min" + undetermined = "undetermined" + + +class DelayDiscoveryToCall(Enum): + immediate = "immediate" + under_5_min = "under_5_min" + field_5_to_30_min = "5_to_30_min" + over_30_min = "over_30_min" + undetermined = "undetermined" + + +class CallOrigin(Enum): + person_landline = "person_landline" + person_mobile = "person_mobile" + person_in_person = "person_in_person" + automatic_alarm_originator = "automatic_alarm_originator" + automatic_alarm_monitoring_center = "automatic_alarm_monitoring_center" + other_agency = "other_agency" + police = "police" + ambulance = "ambulance" + coastguard = "coastguard" + other_fire_service = "other_fire_service" + other = "other" + undetermined = "undetermined" + + +class DispatcherComment(BaseModel): + comment: Optional[str] = None + timestamp: Optional[AwareDatetime] = None + + +class Dispatch(BaseModel): + """ + Call handling data, usually pre-populated from CAD/PSAP when available + """ + + psap_id: Optional[str] = None + """ + Dispatch center / PSAP identifier + """ + dispatch_center: Optional[str] = None + """ + Dispatch center name + """ + cad_event_id: Optional[str] = None + call_received_datetime: Optional[AwareDatetime] = None + """ + Call arrived at PSAP or department dispatch center + """ + call_answered_datetime: Optional[AwareDatetime] = None + call_created_datetime: Optional[AwareDatetime] = None + """ + CAD event created + """ + first_unit_dispatched_datetime: Optional[AwareDatetime] = None + call_origin: Optional[CallOrigin] = None + automatic_alarm: Optional[bool] = None + """ + Call originated from an automatic alarm system + """ + incident_type_at_dispatch: Optional[str] = None + """ + Incident type as received by the control room; may differ from final type + """ + determinate_code: Optional[str] = None + """ + Output code from the dispatch protocol (e.g. EMD/ProQA) + """ + priority_at_call: Annotated[Optional[int], Field(None, ge=1, le=5)] + dispatcher_comments: Optional[list[DispatcherComment]] = None + + +class LocationType(Enum): + street_address = "street_address" + intersection = "intersection" + milepost_or_highway = "milepost_or_highway" + coordinates_only = "coordinates_only" + unaddressable_area = "unaddressable_area" + water_body = "water_body" + other = "other" + + +class Scheme1(Enum): + usng = "usng" + mgrs = "mgrs" + utm = "utm" + osgb = "osgb" + other = "other" + + +class GridReference(BaseModel): + """ + National grid reference where used instead of lat/long + """ + + scheme: Optional[Scheme1] = None + value: Optional[str] = None + + +class Jurisdiction(BaseModel): + federal: Optional[bool] = None + state: Optional[bool] = None + private: Optional[bool] = None + tribal: Optional[bool] = None + + +class OwnershipAtOrigin(Enum): + """ + Ownership of the property at the point of origin + """ + + private = "private" + city_or_local = "city_or_local" + county = "county" + state_or_province = "state_or_province" + federal = "federal" + tribal = "tribal" + military = "military" + foreign = "foreign" + other = "other" + undetermined = "undetermined" + + +class PopulationDensity(Enum): + urban = "urban" + suburban = "suburban" + rural = "rural" + wilderness = "wilderness" + + +class OverBorder(BaseModel): + """ + Incident on another service's ground (UK IRS 1.5-1.7) + """ + + is_over_border: Optional[bool] = None + other_service_name: Optional[str] = None + other_service_incident_number: Optional[str] = None + + +class Location(BaseModel): + location_type: Optional[LocationType] = None + address: Optional[str] = None + """ + Full street address as one line + """ + cross_streets: Optional[list[str]] = None + """ + Nearest cross street(s) + """ + nearest_landmark: Optional[str] = None + city: Optional[str] = None + """ + City, town or nearest settlement + """ + district_or_zone: Optional[str] = None + """ + Administrative district, borough or fire zone + """ + county: Optional[str] = None + state: Optional[str] = None + """ + State, province or region + """ + country: Annotated[Optional[str], Field(None, examples=["US"])] + """ + ISO 3166-1 alpha-2 code preferred + """ + postal_code: Optional[str] = None + census_area: Optional[str] = None + """ + Census tract or national statistical area code + """ + coordinates: Optional[Coordinates] = None + ignition_point_coordinates: Optional[Coordinates] = None + grid_reference: Optional[GridReference] = None + """ + National grid reference where used instead of lat/long + """ + elevation_m: Optional[float] = None + legal_description: Optional[str] = None + """ + Township / section / range or equivalent cadastral reference + """ + jurisdiction: Optional[Jurisdiction] = None + ownership_at_origin: Optional[OwnershipAtOrigin] = None + """ + Ownership of the property at the point of origin + """ + population_density: Optional[PopulationDensity] = None + property_type: Optional[str] = None + property_use: Optional[str] = None + """ + Use of the property at the time (residential, commercial, school...) + """ + property_use_codes: Optional[list[CodeRef]] = None + """ + Property use in external coding schemes (NFIRS 3-digit, NERIS location use) + """ + mixed_use: Optional[str] = None + """ + Mixed-use classification when the property has multiple uses + """ + over_border: Optional[OverBorder] = None + """ + Incident on another service's ground (UK IRS 1.5-1.7) + """ + + +class Category(Enum): + fire_suppression = "fire_suppression" + search = "search" + rescue = "rescue" + ems_care = "ems_care" + extrication = "extrication" + hazmat_mitigation = "hazmat_mitigation" + ventilation = "ventilation" + forcible_entry = "forcible_entry" + salvage_overhaul = "salvage_overhaul" + water_supply = "water_supply" + exposure_protection = "exposure_protection" + evacuation = "evacuation" + command_control = "command_control" + investigation = "investigation" + public_assist = "public_assist" + standby = "standby" + information_referral = "information_referral" + systems_restoration = "systems_restoration" + other = "other" + + +class Action(BaseModel): + category: Optional[Category] = None + description: Optional[str] = None + codes: Optional[list[CodeRef]] = None + + +class ActionsTaken(BaseModel): + """ + What responders did on scene. Every reporting standard requires this. + """ + + actions: Optional[list[Action]] = None + no_action_reason: Optional[str] = None + """ + Why no action was taken (canceled enroute, no hazard found...) + """ + + +class AidDirection(Enum): + given = "given" + received = "received" + both = "both" + none = "none" + + +class AidType(Enum): + automatic = "automatic" + mutual = "mutual" + other = "other" + none = "none" + + +class IncidentCommander(BaseModel): + name: Optional[str] = None + agency: Optional[str] = None + position: Optional[str] = None + + +class RespondingAgency(BaseModel): + agency_name: Optional[str] = None + agency_type: Optional[str] = None + """ + fire, ems, police, forestry, military, utility, ngo, other + """ + role: Optional[str] = None + personnel_count: Optional[int] = None + incident_number_at_agency: Optional[str] = None + """ + That agency's own incident number for cross-referencing + """ + + +class ApparatusType(Enum): + engine_pumper = "engine_pumper" + ladder_aerial = "ladder_aerial" + quint = "quint" + tanker_tender = "tanker_tender" + brush_wildland = "brush_wildland" + arff = "arff" + dozer_plow = "dozer_plow" + heavy_equipment = "heavy_equipment" + aircraft_fixed_wing = "aircraft_fixed_wing" + helicopter = "helicopter" + boat = "boat" + rescue_unit = "rescue_unit" + usar_unit = "usar_unit" + hazmat_unit = "hazmat_unit" + ambulance_bls = "ambulance_bls" + ambulance_als = "ambulance_als" + command_vehicle = "command_vehicle" + support_unit = "support_unit" + hand_crew = "hand_crew" + privately_owned = "privately_owned" + other = "other" + + +class Use(Enum): + suppression = "suppression" + ems = "ems" + rescue = "rescue" + hazmat = "hazmat" + command = "command" + support = "support" + other = "other" + + +class PersonnelItem(BaseModel): + personnel_id: Optional[str] = None + name: Optional[str] = None + rank: Optional[str] = None + role: Optional[str] = None + + +class ResponseMode(Enum): + emergency_lights_siren = "emergency_lights_siren" + non_emergency = "non_emergency" + undetermined = "undetermined" + + +class TransportMode(Enum): + emergency_lights_siren = "emergency_lights_siren" + non_emergency = "non_emergency" + undetermined = "undetermined" + + +class UnitResponse(BaseModel): + """ + One responding unit (apparatus or resource) and its timeline + """ + + unit_id: Optional[str] = None + """ + Callsign or unit identifier + """ + unit_name: Optional[str] = None + agency_name: Optional[str] = None + apparatus_type: Optional[ApparatusType] = None + use: Optional[Use] = None + personnel_count: Optional[int] = None + personnel: Optional[list[PersonnelItem]] = None + response_mode: Optional[ResponseMode] = None + canceled_enroute: Optional[bool] = None + dispatched_datetime: Optional[AwareDatetime] = None + enroute_datetime: Optional[AwareDatetime] = None + arrived_datetime: Optional[AwareDatetime] = None + staged_datetime: Optional[AwareDatetime] = None + at_patient_datetime: Optional[AwareDatetime] = None + enroute_hospital_datetime: Optional[AwareDatetime] = None + arrived_hospital_datetime: Optional[AwareDatetime] = None + transfer_of_care_datetime: Optional[AwareDatetime] = None + cleared_datetime: Optional[AwareDatetime] = None + in_service_datetime: Optional[AwareDatetime] = None + """ + Back available for calls + """ + turnout_seconds: Optional[int] = None + """ + Computed, dispatched to enroute + """ + travel_seconds: Optional[int] = None + """ + Computed, enroute to arrived + """ + transport_mode: Optional[TransportMode] = None + hospital_destination: Optional[str] = None + actions_taken: Optional[list[str]] = None + """ + Actions by this unit, same categories as incident actions + """ + + +class PersonnelBreakdown(BaseModel): + firefighters: Optional[int] = None + officers: Optional[int] = None + engineers_operators: Optional[int] = None + ems_personnel: Optional[int] = None + incident_command: Optional[int] = None + support_staff: Optional[int] = None + + +class ApparatusCounts(BaseModel): + """ + Unit counts by primary use (NFIRS Basic G1) + """ + + suppression: Optional[int] = None + ems: Optional[int] = None + other: Optional[int] = None + + +class ResourcesSummary(BaseModel): + """ + Aggregate counts; per-unit detail lives in units[] + """ + + total_personnel: Optional[int] = None + personnel_breakdown: Optional[PersonnelBreakdown] = None + apparatus_counts: Optional[ApparatusCounts] = None + """ + Unit counts by primary use (NFIRS Basic G1) + """ + crew_types: Optional[list[str]] = None + """ + Wildland crew types deployed (hand crew type 1/2, engine crew...) + """ + counts_include_aid_received: Optional[bool] = None + + +class CauseCategory(Enum): + intentional = "intentional" + unintentional = "unintentional" + equipment_failure = "equipment_failure" + act_of_nature = "act_of_nature" + cause_under_investigation = "cause_under_investigation" + undetermined = "undetermined" + other = "other" + + +class HumanFactor(Enum): + asleep = "asleep" + impaired_by_alcohol_or_drugs = "impaired_by_alcohol_or_drugs" + unattended_person = "unattended_person" + mentally_disabled = "mentally_disabled" + physically_disabled = "physically_disabled" + multiple_persons_involved = "multiple_persons_involved" + age_was_factor = "age_was_factor" + other = "other" + + +class Portability(Enum): + portable = "portable" + stationary = "stationary" + + +class EquipmentInvolved(BaseModel): + """ + Equipment involved in ignition, if any + """ + + involved: Optional[bool] = None + equipment_type: Optional[str] = None + brand: Optional[str] = None + model: Optional[str] = None + serial_number: Optional[str] = None + year: Optional[int] = None + power_source: Optional[str] = None + portability: Optional[Portability] = None + + +class WaterSupplyType(Enum): + pressurized_hydrant = "pressurized_hydrant" + rural_water_supply = "rural_water_supply" + tanker_shuttle = "tanker_shuttle" + drafting_static_source = "drafting_static_source" + onboard_water_only = "onboard_water_only" + none_needed = "none_needed" + other = "other" + undetermined = "undetermined" + + +class ExtinguishingAgent(Enum): + water = "water" + foam = "foam" + co2 = "co2" + dry_chemical = "dry_chemical" + wet_chemical = "wet_chemical" + halon_clean_agent = "halon_clean_agent" + sand_earth = "sand_earth" + blanket_smothering = "blanket_smothering" + other = "other" + + +class EquipmentUsedItem(BaseModel): + equipment_type: Optional[str] = None + count: Optional[int] = None + + +class FirefightingDelay(BaseModel): + occurred: Optional[bool] = None + reason: Optional[str] = None + + +class SuppressionOperations(BaseModel): + water_supply_type: Optional[WaterSupplyType] = None + water_used_l: Optional[float] = None + extinguishing_agents: Optional[list[ExtinguishingAgent]] = None + suppression_appliances: Optional[list[str]] = None + """ + Appliances used for suppression (jets, hose reels, monitors, extinguishers) + """ + equipment_used: Optional[list[EquipmentUsedItem]] = None + """ + Equipment used at the incident with counts (UK IRS 6.16-6.17) + """ + ba_wearers_count: Optional[int] = None + """ + Breathing apparatus wearers + """ + firefighting_delay: Optional[FirefightingDelay] = None + public_action_before_arrival: Optional[str] = None + """ + Main action taken by the public before responders arrived + """ + + +class Stage(Enum): + before_fire = "before_fire" + during_fire = "during_fire" + after_fire = "after_fire" + no_fire = "no_fire" + + +class Explosion(BaseModel): + """ + Explosion or overpressure event, with or without fire (UK IRS 8.10-8.13) + """ + + occurred: Optional[bool] = None + cause: Optional[str] = None + stage: Optional[Stage] = None + containers_involved: Optional[list[str]] = None + + +class AlarmType(Enum): + smoke = "smoke" + heat = "heat" + combination = "combination" + sprinkler_waterflow = "sprinkler_waterflow" + multiple_types = "multiple_types" + other = "other" + undetermined = "undetermined" + + +class PowerSupply(Enum): + battery_only = "battery_only" + hardwire_only = "hardwire_only" + hardwire_with_battery = "hardwire_with_battery" + plug_in = "plug_in" + plug_in_with_battery = "plug_in_with_battery" + mechanical = "mechanical" + multiple = "multiple" + other = "other" + undetermined = "undetermined" + + +class Effectiveness(Enum): + alerted_occupants_responded = "alerted_occupants_responded" + alerted_occupants_no_response = "alerted_occupants_no_response" + no_occupants = "no_occupants" + failed_to_alert = "failed_to_alert" + undetermined = "undetermined" + + +class FailureReason(Enum): + power_failure_or_disconnect = "power_failure_or_disconnect" + improper_installation = "improper_installation" + defective = "defective" + lack_of_maintenance = "lack_of_maintenance" + battery_missing = "battery_missing" + battery_dead = "battery_dead" + other = "other" + undetermined = "undetermined" + + +class SmokeAlarm(BaseModel): + presence: Optional[PresenceStatus] = None + alarm_type: Optional[AlarmType] = None + power_supply: Optional[PowerSupply] = None + working: Optional[bool] = None + operation: Optional[OperationStatus] = None + effectiveness: Optional[Effectiveness] = None + failure_reason: Optional[FailureReason] = None + occupant_response: Optional[str] = None + + +class FireAlarm(BaseModel): + """ + Building fire alarm system + """ + + presence: Optional[PresenceStatus] = None + alarm_type: Optional[str] = None + monitored: Optional[bool] = None + operation: Optional[OperationStatus] = None + failure_reason: Optional[str] = None + + +class OtherAlarm(BaseModel): + """ + CO, gas, security or other alarm + """ + + presence: Optional[PresenceStatus] = None + alarm_type: Optional[str] = None + + +class SystemType(Enum): + wet_pipe_sprinkler = "wet_pipe_sprinkler" + dry_pipe_sprinkler = "dry_pipe_sprinkler" + other_sprinkler = "other_sprinkler" + dry_chemical = "dry_chemical" + foam = "foam" + halon_clean_agent = "halon_clean_agent" + co2 = "co2" + water_mist = "water_mist" + other_special_hazard = "other_special_hazard" + other = "other" + undetermined = "undetermined" + + +class Coverage(Enum): + full = "full" + partial = "partial" + undetermined = "undetermined" + + +class FailureReason1(Enum): + system_shut_off = "system_shut_off" + not_enough_agent = "not_enough_agent" + agent_did_not_reach_fire = "agent_did_not_reach_fire" + wrong_system_type = "wrong_system_type" + fire_outside_protected_area = "fire_outside_protected_area" + components_damaged = "components_damaged" + lack_of_maintenance = "lack_of_maintenance" + manual_intervention = "manual_intervention" + other = "other" + undetermined = "undetermined" + + +class SuppressionSystem(BaseModel): + """ + Automatic extinguishing system + """ + + presence: Optional[PresenceStatus] = None + system_type: Optional[SystemType] = None + coverage: Optional[Coverage] = None + operation: Optional[OperationStatus] = None + sprinkler_heads_activated: Optional[int] = None + effective: Optional[bool] = None + failure_reason: Optional[FailureReason1] = None + + +class CookingSuppression(BaseModel): + presence: Optional[PresenceStatus] = None + system_type: Optional[str] = None + + +class FixedFirefightingFacility(BaseModel): + facility_type: Optional[str] = None + used: Optional[bool] = None + worked: Optional[bool] = None + failure_reason: Optional[str] = None + + +class RiskReduction(BaseModel): + """ + Alarms, detectors and suppression systems and how they performed + """ + + smoke_alarm: Optional[SmokeAlarm] = None + fire_alarm: Optional[FireAlarm] = None + """ + Building fire alarm system + """ + other_alarm: Optional[OtherAlarm] = None + """ + CO, gas, security or other alarm + """ + suppression_system: Optional[SuppressionSystem] = None + """ + Automatic extinguishing system + """ + cooking_suppression: Optional[CookingSuppression] = None + fixed_firefighting_facilities: Optional[list[FixedFirefightingFacility]] = None + """ + Built-in firefighting facilities (risers, hose reels, smoke control, fire lift) + """ + + +class BuildingStatus(Enum): + occupied_in_use = "occupied_in_use" + vacant_secured = "vacant_secured" + vacant_unsecured = "vacant_unsecured" + under_construction = "under_construction" + under_renovation = "under_renovation" + under_demolition = "under_demolition" + derelict = "derelict" + undetermined = "undetermined" + + +class FireSpreadExtent(Enum): + confined_to_object = "confined_to_object" + confined_to_room = "confined_to_room" + confined_to_floor = "confined_to_floor" + confined_to_building = "confined_to_building" + beyond_building = "beyond_building" + no_flame_damage = "no_flame_damage" + undetermined = "undetermined" + + +class ArrivalConditions(Enum): + no_visible_smoke_or_fire = "no_visible_smoke_or_fire" + smoke_showing = "smoke_showing" + fire_showing = "fire_showing" + fully_involved = "fully_involved" + collapsed = "collapsed" + undetermined = "undetermined" + + +class StoriesDamaged(BaseModel): + """ + Count of stories by flame damage band (NFIRS-3 J3) + """ + + minor: Optional[int] = None + """ + 1-24% flame damage + """ + significant: Optional[int] = None + """ + 25-49% flame damage + """ + heavy: Optional[int] = None + """ + 50-74% flame damage + """ + extreme: Optional[int] = None + """ + 75-100% flame damage + """ + + +class Structure(BaseModel): + is_structure_involved: Optional[bool] = None + structures_threatened: Optional[int] = None + structures_damaged: Optional[int] = None + structures_destroyed: Optional[int] = None + structures_protected: Optional[int] = None + buildings_involved: Optional[int] = None + """ + Number of buildings involved at the origin property + """ + building_status: Optional[BuildingStatus] = None + construction_type: Optional[str] = None + construction_codes: Optional[list[CodeRef]] = None + special_construction_method: Optional[str] = None + """ + Notable construction method involved (timber frame, sandwich panel, cladding system) + """ + stories_above_grade: Optional[int] = None + stories_below_grade: Optional[int] = None + total_floor_area_m2: Optional[float] = None + main_floor_area_m2: Optional[float] = None + residential_units: Optional[int] = None + """ + Residential living units in the building of origin + """ + occupancy_at_time: Optional[int] = None + """ + Estimated number of people in the building at the time + """ + fire_safety_regulations_apply: Optional[bool] = None + means_of_escape_condition: Optional[str] = None + compartmentation_effective: Optional[bool] = None + story_of_origin: Optional[int] = None + """ + Negative below grade, 1 is ground floor + """ + room_of_origin: Optional[str] = None + room_of_origin_area_m2: Optional[float] = None + floor_of_origin_area_m2: Optional[float] = None + fire_spread_extent: Optional[FireSpreadExtent] = None + item_contributing_most_to_spread: Optional[str] = None + material_contributing_most_to_spread: Optional[str] = None + arrival_conditions: Optional[ArrivalConditions] = None + progressed_beyond_arrival: Optional[bool] = None + """ + Fire extended beyond the conditions found on arrival + """ + smoke_damage_only: Optional[bool] = None + """ + Heat/smoke damage with no flame damage (UK IRS 8.19) + """ + stories_damaged: Optional[StoriesDamaged] = None + """ + Count of stories by flame damage band (NFIRS-3 J3) + """ + damage_area_on_arrival_m2: Optional[float] = None + damage_area_at_stop_m2: Optional[float] = None + """ + Horizontal area damaged by flame/heat when fire was stopped + """ + + +class AreaType(Enum): + urban = "urban" + suburban = "suburban" + rural = "rural" + wildland_urban_interface = "wildland_urban_interface" + remote_wilderness = "remote_wilderness" + + +class LandOwnershipBreakdown(BaseModel): + federal_ha: Optional[float] = None + state_ha: Optional[float] = None + private_ha: Optional[float] = None + tribal_ha: Optional[float] = None + other_ha: Optional[float] = None + + +class FireDangerRating(Enum): + """ + Fire danger rating in effect at the time + """ + + low = "low" + moderate = "moderate" + high = "high" + very_high = "very_high" + severe = "severe" + extreme = "extreme" + catastrophic = "catastrophic" + + +class ComplexityLevel(Enum): + """ + Incident complexity (type 5 lowest, type 1 highest) + """ + + type_5 = "type_5" + type_4 = "type_4" + type_3 = "type_3" + type_2 = "type_2" + type_1 = "type_1" + + +class Status(Enum): + identified = "identified" + unidentified = "unidentified" + fire_not_caused_by_person = "fire_not_caused_by_person" + + +class PersonResponsible(BaseModel): + """ + Person who caused the wildland fire, if any (NFIRS-8 L) + """ + + status: Optional[Status] = None + age: Optional[int] = None + sex: Optional[str] = None + activity: Optional[str] = None + + +class RightOfWay(BaseModel): + """ + Nearby road/rail/power right-of-way (NFIRS-8 M) + """ + + row_type: Optional[str] = None + distance_m: Optional[float] = None + + +class FireLines(BaseModel): + primary_line_km: Optional[float] = None + secondary_line_km: Optional[float] = None + dozer_line_km: Optional[float] = None + hand_line_km: Optional[float] = None + + +class AerialOperations(BaseModel): + water_dropped_l: Optional[float] = None + retardant_dropped_l: Optional[float] = None + total_flight_hours: Optional[float] = None + + +class Wildland(BaseModel): + is_wildland_incident: Optional[bool] = None + discovery_datetime: Optional[AwareDatetime] = None + area_type: Optional[AreaType] = None + area_burned_ha: Optional[float] = None + """ + Total area burned in hectares + """ + land_ownership_breakdown: Optional[LandOwnershipBreakdown] = None + percent_contained: Annotated[Optional[int], Field(None, ge=0, le=100)] + fire_danger_rating: Optional[FireDangerRating] = None + """ + Fire danger rating in effect at the time + """ + fuel_model: Optional[str] = None + """ + NFDRS or local fuel model at origin + """ + fuel_moisture_percent: Optional[float] = None + complexity_level: Optional[ComplexityLevel] = None + """ + Incident complexity (type 5 lowest, type 1 highest) + """ + slope_position: Optional[str] = None + """ + Relative position on slope at origin + """ + aspect: Optional[str] = None + person_responsible: Optional[PersonResponsible] = None + """ + Person who caused the wildland fire, if any (NFIRS-8 L) + """ + right_of_way: Optional[RightOfWay] = None + """ + Nearby road/rail/power right-of-way (NFIRS-8 M) + """ + crops_burned: Optional[list[str]] = None + fire_lines: Optional[FireLines] = None + aerial_operations: Optional[AerialOperations] = None + containment_strategies: Optional[list[str]] = None + + +class DamageRating(Enum): + none = "none" + minor = "minor" + significant = "significant" + heavy = "heavy" + destroyed = "destroyed" + undetermined = "undetermined" + + +class Exposure(BaseModel): + """ + A property beyond the origin damaged or threatened by spread + """ + + exposure_number: Optional[int] = None + """ + 0 is the origin; exposures count up from 1 (NFIRS convention) + """ + exposure_type: Optional[str] = None + item_damaged: Optional[str] = None + address: Optional[str] = None + coordinates: Optional[Coordinates] = None + property_use: Optional[str] = None + people_present: Optional[bool] = None + damage_rating: Optional[DamageRating] = None + people_displaced: Optional[int] = None + + +class Affiliation(Enum): + civilian = "civilian" + ems_non_fd = "ems_non_fd" + police = "police" + other_responder = "other_responder" + undetermined = "undetermined" + + +class HumanFactor1(Enum): + asleep = "asleep" + unconscious = "unconscious" + impaired_by_alcohol = "impaired_by_alcohol" + impaired_by_drugs = "impaired_by_drugs" + mentally_disabled = "mentally_disabled" + physically_disabled = "physically_disabled" + physically_restrained = "physically_restrained" + unattended_person = "unattended_person" + other = "other" + + +class ActivityWhenInjured(Enum): + escaping = "escaping" + rescue_attempt = "rescue_attempt" + fire_control = "fire_control" + returning_before_control = "returning_before_control" + returning_after_control = "returning_after_control" + sleeping = "sleeping" + unable_to_act = "unable_to_act" + irrational_act = "irrational_act" + other = "other" + undetermined = "undetermined" + + +class LocationAtIgnition(Enum): + in_area_of_origin = "in_area_of_origin" + in_building_not_in_area = "in_building_not_in_area" + outside_building = "outside_building" + not_on_property = "not_on_property" + undetermined = "undetermined" + + +class CareerOrVolunteer(Enum): + career = "career" + volunteer = "volunteer" + undetermined = "undetermined" + + +class PhysicalConditionPrior(Enum): + rested = "rested" + fatigued = "fatigued" + ill_or_injured = "ill_or_injured" + other = "other" + undetermined = "undetermined" + + +class WhereOccurred(Enum): + enroute_to_scene = "enroute_to_scene" + at_scene_inside = "at_scene_inside" + at_scene_outside = "at_scene_outside" + enroute_to_facility = "enroute_to_facility" + at_facility = "at_facility" + returning = "returning" + at_station = "at_station" + other = "other" + undetermined = "undetermined" + + +class ProtectiveEquipmentFailure(BaseModel): + failed: Optional[bool] = None + item: Optional[str] = None + problem: Optional[str] = None + + +class DutyStatus(Enum): + on_duty = "on_duty" + off_duty_responding = "off_duty_responding" + off_duty = "off_duty" + undetermined = "undetermined" + + +class TakenTo(Enum): + hospital = "hospital" + doctors_office = "doctors_office" + morgue = "morgue" + residence = "residence" + station = "station" + not_transported = "not_transported" + other = "other" + + +class ResponderCasualty(BaseModel): + personnel_id: Optional[str] = None + name: Optional[str] = None + age: Optional[int] = None + sex: Optional[str] = None + agency: Optional[str] = None + role: Optional[str] = None + rank: Optional[str] = None + career_or_volunteer: Optional[CareerOrVolunteer] = None + years_of_service: Optional[float] = None + usual_assignment: Optional[str] = None + physical_condition_prior: Optional[PhysicalConditionPrior] = None + prior_responses_24h: Optional[int] = None + injury_datetime: Optional[AwareDatetime] = None + injury_type: Optional[str] = None + primary_symptom: Optional[str] = None + primary_body_part: Optional[str] = None + severity: Optional[InjurySeverity] = None + cause: Optional[str] = None + contributing_factor: Optional[str] = None + object_involved: Optional[str] = None + activity_at_injury: Optional[str] = None + where_occurred: Optional[WhereOccurred] = None + story_where_injured: Optional[int] = None + protective_equipment_failure: Optional[ProtectiveEquipmentFailure] = None + duty_status: Optional[DutyStatus] = None + treatment: Optional[str] = None + taken_to: Optional[TakenTo] = None + transported: Optional[bool] = None + hospital: Optional[str] = None + hospitalized_overnight: Optional[bool] = None + return_to_duty_date: Optional[date_type] = None + osha_recordable: Optional[bool] = None + exposure_only: Optional[bool] = None + """ + Chemical/biological exposure without immediate symptoms + """ + + +class PersonType(Enum): + civilian = "civilian" + firefighter = "firefighter" + other_responder = "other_responder" + + +class RescueType(Enum): + rescue = "rescue" + assist = "assist" + self_evacuation = "self_evacuation" + body_recovery = "body_recovery" + no_rescue_needed = "no_rescue_needed" + + +class RelativeTimeToSuppression(Enum): + before_suppression = "before_suppression" + during_suppression = "during_suppression" + after_suppression = "after_suppression" + undetermined = "undetermined" + + +class Mayday(BaseModel): + """ + Firefighter emergencies only + """ + + called: Optional[bool] = None + relative_time: Optional[str] = None + rit_activated: Optional[bool] = None + + +class Rescue(BaseModel): + """ + One person rescued, assisted or self-evacuated (NERIS rescue modules) + """ + + person_type: Optional[PersonType] = None + rescue_type: Optional[RescueType] = None + presence_known_beforehand: Optional[bool] = None + age: Optional[int] = None + sex: Optional[str] = None + primary_mode: Optional[str] = None + """ + Primary rescue mode (interior search, ladder, water, rope, extrication) + """ + actions: Optional[list[str]] = None + impediments: Optional[list[str]] = None + room_type: Optional[str] = None + elevation: Optional[str] = None + """ + Elevation at which the person was found (below grade, ground, upper story, roof) + """ + removal_path: Optional[str] = None + """ + Route used to remove the person (internal stairs, window, aerial) + """ + relative_time_to_suppression: Optional[RelativeTimeToSuppression] = None + gas_isolation: Optional[bool] = None + """ + Space was isolated from heat/toxic gas flow + """ + mayday: Optional[Mayday] = None + """ + Firefighter emergencies only + """ + resulting_casualty: Optional[bool] = None + """ + True if this person also appears in casualties + """ + + +class EvacuationStatus(Enum): + none = "none" + planned = "planned" + in_progress = "in_progress" + completed = "completed" + repopulation_in_progress = "repopulation_in_progress" + shelter_in_place = "shelter_in_place" + + +class EvacuationDisplacement(BaseModel): + evacuation_occurred: Optional[bool] = None + evacuation_status: Optional[EvacuationStatus] = None + people_evacuated_without_assistance: Optional[int] = None + people_evacuated_with_assistance: Optional[int] = None + people_assisted_by_fd: Optional[int] = None + total_people_evacuated: Optional[int] = None + buildings_evacuated: Optional[int] = None + evacuation_delay_reason: Optional[str] = None + evacuation_completion_minutes: Optional[int] = None + people_sheltering_in_place: Optional[int] = None + people_in_temporary_shelters: Optional[int] = None + people_trapped: Optional[int] = None + people_missing: Optional[int] = None + people_displaced: Optional[int] = None + """ + People who cannot return to the property + """ + displacement_causes: Optional[list[str]] = None + + +class InjuryIntent(Enum): + accidental = "accidental" + self_inflicted = "self_inflicted" + inflicted_by_other = "inflicted_by_other" + undetermined = "undetermined" + + +class BodySite(BaseModel): + site: Optional[str] = None + injury_type: Optional[str] = None + + +class CardiacArrest(BaseModel): + occurred: Optional[bool] = None + pre_arrival: Optional[bool] = None + witnessed: Optional[bool] = None + bystander_cpr: Optional[bool] = None + initial_rhythm: Optional[str] = None + + +class HighestCareLevelOnScene(Enum): + first_responder = "first_responder" + emt_basic = "emt_basic" + emt_intermediate = "emt_intermediate" + paramedic = "paramedic" + physician = "physician" + other = "other" + + +class PatientStatus(Enum): + improved = "improved" + unchanged = "unchanged" + worsened = "worsened" + + +class Disposition(Enum): + treated_and_transported_by_fd = "treated_and_transported_by_fd" + transported_by_other_agency = "transported_by_other_agency" + treated_no_transport = "treated_no_transport" + refused_care = "refused_care" + dead_at_scene = "dead_at_scene" + transferred_care = "transferred_care" + other = "other" + + +class EMSPatient(BaseModel): + """ + Summary-level patient record; the ePCR is the clinical record + """ + + patient_ref_id: Optional[str] = None + nemsis_report_id: Optional[str] = None + age_approx: Optional[int] = None + date_of_birth: Optional[date_type] = None + sex: Optional[str] = None + chief_complaint: Optional[str] = None + provider_impression: Optional[str] = None + """ + Provider's primary impression/assessment + """ + injury_intent: Optional[InjuryIntent] = None + body_sites: Optional[list[BodySite]] = None + """ + Injured body sites with injury type per site + """ + procedures: Optional[list[str]] = None + """ + Procedures performed on scene (CPR, oxygen, splinting, defibrillation) + """ + cardiac_arrest: Optional[CardiacArrest] = None + safety_equipment_used: Optional[list[str]] = None + """ + Safety equipment used by the patient (seat belt, airbag, helmet) + """ + highest_care_level_on_scene: Optional[HighestCareLevelOnScene] = None + patient_status: Optional[PatientStatus] = None + at_patient_datetime: Optional[AwareDatetime] = None + transfer_of_care_datetime: Optional[AwareDatetime] = None + disposition: Optional[Disposition] = None + transported: Optional[bool] = None + hospital_destination: Optional[str] = None + + +class IgnitionOrReleaseFirst(Enum): + ignition_first = "ignition_first" + release_first = "release_first" + no_fire = "no_fire" + undetermined = "undetermined" + + +class ReleaseCause(Enum): + intentional = "intentional" + unintentional = "unintentional" + container_failure = "container_failure" + act_of_nature = "act_of_nature" + cause_under_investigation = "cause_under_investigation" + undetermined = "undetermined" + + +class EquipmentInvolvedInRelease(BaseModel): + involved: Optional[bool] = None + equipment_type: Optional[str] = None + brand: Optional[str] = None + model: Optional[str] = None + year: Optional[int] = None + + +class Disposition1(Enum): + """ + Who the cleanup/scene was released to. Evacuee counts live in evacuation_displacement. + """ + + completed_by_fire_service = "completed_by_fire_service" + completed_with_fire_service_present = "completed_with_fire_service_present" + released_to_local_agency = "released_to_local_agency" + released_to_state_agency = "released_to_state_agency" + released_to_federal_agency = "released_to_federal_agency" + released_to_private_contractor = "released_to_private_contractor" + released_to_owner = "released_to_owner" + undetermined = "undetermined" + + +class PhysicalState(Enum): + solid = "solid" + liquid = "liquid" + gas = "gas" + undetermined = "undetermined" + + +class ReleasedInto(Enum): + air = "air" + water = "water" + soil = "soil" + contained_on_site = "contained_on_site" + sewer_drain = "sewer_drain" + other = "other" + undetermined = "undetermined" + + +class HazmatMaterial(BaseModel): + name: Optional[str] = None + un_number: Optional[str] = None + dot_hazard_class: Optional[str] = None + """ + DOT/UN hazard class and division, e.g. "3" or "2.1" + """ + cas_number: Optional[str] = None + physical_state: Optional[PhysicalState] = None + container_type: Optional[str] = None + container_capacity: Optional[Quantity] = None + released: Optional[bool] = None + amount_released: Optional[Quantity] = None + released_into: Optional[ReleasedInto] = None + released_from_story: Optional[int] = None + released_inside_structure: Optional[bool] = None + + +class Category1(Enum): + battery_energy_storage = "battery_energy_storage" + electric_vehicle = "electric_vehicle" + micromobility_device = "micromobility_device" + consumer_electronics = "consumer_electronics" + photovoltaic_system = "photovoltaic_system" + power_generation = "power_generation" + csst_gas_tubing = "csst_gas_tubing" + other = "other" + + +class SourceOrTarget(Enum): + ignition_source = "ignition_source" + target_only = "target_only" + both = "both" + undetermined = "undetermined" + + +class EmergingHazard(BaseModel): + """ + Stored-energy and similar emerging hazards (NERIS emerging_hazard module) + """ + + category: Optional[Category1] = None + subtype: Optional[str] = None + source_or_target: Optional[SourceOrTarget] = None + suppression_approach: Optional[str] = None + reignition_occurred: Optional[bool] = None + ev_crash_involved: Optional[bool] = None + """ + Electric vehicle was involved in a crash + """ + lightning_suspected: Optional[bool] = None + """ + CSST cases, lightning as suspected cause + """ + notes: Optional[str] = None + + +class CaseStatus(Enum): + open = "open" + closed_with_arrest = "closed_with_arrest" + closed_exceptional = "closed_exceptional" + closed = "closed" + inactive = "inactive" + + +class AgencyReferredTo(BaseModel): + name: Optional[str] = None + case_number: Optional[str] = None + + +class IncendiaryDevice(BaseModel): + container: Optional[str] = None + ignition_delay_mechanism: Optional[str] = None + fuel: Optional[str] = None + + +class MaterialAvailability(Enum): + transported_to_scene = "transported_to_scene" + available_at_scene = "available_at_scene" + undetermined = "undetermined" + + +class Subject(BaseModel): + age: Optional[int] = None + sex: Optional[str] = None + family_type: Optional[str] = None + risk_factors: Optional[list[str]] = None + disposition: Optional[str] = None + + +class JuvenileFiresetter(BaseModel): + involved: Optional[bool] = None + subjects: Optional[list[Subject]] = None + + +class Arson(BaseModel): + suspected: Optional[bool] = None + confirmed: Optional[bool] = None + motivation_factors: Optional[list[str]] = None + """ + Suspected motivations (fraud, intimidation, concealment, thrill, protest) + """ + group_involvement: Optional[str] = None + entry_method: Optional[str] = None + extent_of_involvement_on_arrival: Optional[str] = None + incendiary_device: Optional[IncendiaryDevice] = None + material_availability: Optional[MaterialAvailability] = None + initial_observations: Optional[list[str]] = None + """ + Scene observations (forced entry, doors locked, security system state) + """ + other_indicators: Optional[list[str]] = None + """ + Contextual indicators (vacancy, for sale, insurance change, financial problems) + """ + juvenile_firesetter: Optional[JuvenileFiresetter] = None + + +class Role(Enum): + owner = "owner" + occupant = "occupant" + tenant = "tenant" + reporting_party = "reporting_party" + responsible_party = "responsible_party" + witness = "witness" + business_representative = "business_representative" + insurance_holder = "insurance_holder" + other = "other" + + +class Insurance(BaseModel): + insured: Optional[bool] = None + company: Optional[str] = None + policy_number: Optional[str] = None + + +class PersonInvolved(BaseModel): + """ + Owner, occupant or other party connected to the incident (not casualties) + """ + + role: Optional[Role] = None + name: Optional[str] = None + business_name: Optional[str] = None + address: Optional[str] = None + same_address_as_incident: Optional[bool] = None + phone: Optional[str] = None + email: Optional[str] = None + insurance: Optional[Insurance] = None + + +class Involvement(Enum): + ignition_source_and_burned = "ignition_source_and_burned" + ignition_source_not_burned = "ignition_source_not_burned" + burned_not_ignition_source = "burned_not_ignition_source" + collision = "collision" + hazmat_release = "hazmat_release" + rescued_from = "rescued_from" + threatened_only = "threatened_only" + other = "other" + + +class PropertyType(Enum): + passenger_car = "passenger_car" + motorcycle = "motorcycle" + bus = "bus" + heavy_goods_vehicle = "heavy_goods_vehicle" + agricultural_vehicle = "agricultural_vehicle" + construction_vehicle = "construction_vehicle" + recreational_vehicle = "recreational_vehicle" + train_rail = "train_rail" + boat_vessel = "boat_vessel" + aircraft = "aircraft" + trailer = "trailer" + mobile_home = "mobile_home" + other = "other" + + +class FuelType(Enum): + petrol_gasoline = "petrol_gasoline" + diesel = "diesel" + electric = "electric" + hybrid = "hybrid" + hydrogen = "hydrogen" + cng_lpg = "cng_lpg" + other = "other" + undetermined = "undetermined" + + +class Extrication(BaseModel): + """ + Extrication from this vehicle (UK IRS RTC block) + """ + + performed: Optional[bool] = None + method: Optional[str] = None + vehicle_position: Optional[str] = None + time_taken_minutes: Optional[int] = None + + +class MobileProperty(BaseModel): + involvement: Optional[Involvement] = None + property_type: Optional[PropertyType] = None + make: Optional[str] = None + model: Optional[str] = None + year: Optional[int] = None + fuel_type: Optional[FuelType] = None + license_plate: Optional[str] = None + license_region: Optional[str] = None + """ + Registering state/province/country + """ + vin: Optional[str] = None + dot_icc_number: Optional[str] = None + reported_stolen: Optional[bool] = None + appeared_abandoned: Optional[bool] = None + occupants: Optional[int] = None + extrication: Optional[Extrication] = None + """ + Extrication from this vehicle (UK IRS RTC block) + """ + + +class EstimateMethod(Enum): + rough_estimate = "rough_estimate" + owner_estimate = "owner_estimate" + insurance_assessment = "insurance_assessment" + investigator_assessment = "investigator_assessment" + official_valuation = "official_valuation" + other = "other" + + +class Losses(BaseModel): + """ + Monetary values. Currency follows the reporting agency. + """ + + no_loss: Optional[bool] = None + property_loss: Optional[Money] = None + contents_loss: Optional[Money] = None + pre_incident_property_value: Optional[Money] = None + pre_incident_contents_value: Optional[Money] = None + property_saved: Optional[Money] = None + other_costs: Optional[Money] = None + estimate_method: Optional[EstimateMethod] = None + + +class WeatherType(Enum): + clear = "clear" + cloudy = "cloudy" + rain = "rain" + snow_ice = "snow_ice" + fog = "fog" + high_winds = "high_winds" + thunderstorm_lightning = "thunderstorm_lightning" + extreme_heat = "extreme_heat" + extreme_cold = "extreme_cold" + other = "other" + + +class WeatherReading(BaseModel): + datetime: Optional[AwareDatetime] = None + temperature_c: Optional[float] = None + relative_humidity_percent: Optional[float] = None + wind_speed_kph: Optional[float] = None + wind_gusts_kph: Optional[float] = None + wind_direction: Optional[str] = None + haines_index: Optional[int] = None + + +class EnvironmentalImpact(BaseModel): + habitat_affected_ha: Optional[float] = None + watershed_impact: Optional[str] = None + soil_erosion_risk: Optional[str] = None + sensitive_species_affected: Optional[list[str]] = None + air_quality_impact: Optional[str] = None + water_body_affected: Optional[bool] = None + + +class InfrastructureItem(BaseModel): + infrastructure_type: Optional[str] = None + unit: Optional[str] = None + quantity: Optional[float] = None + severity: Optional[str] = None + + +class NearMissEvent(BaseModel): + description: Optional[str] = None + date: Optional[date_type] = None + contributing_factors: Optional[list[str]] = None + lessons_learned: Optional[str] = None + corrective_action: Optional[str] = None + + +class AttackType(Enum): + verbal_abuse = "verbal_abuse" + physical_no_weapon = "physical_no_weapon" + weapon = "weapon" + objects_thrown = "objects_thrown" + vehicle_used = "vehicle_used" + other = "other" + + +class AttacksOnPersonnel(BaseModel): + """ + Attacks on responders travelling to, at, or from the incident (UK IRS 3.10-3.13) + """ + + occurred: Optional[bool] = None + attack_type: Optional[AttackType] = None + serious_injuries: Optional[int] = None + slight_injuries: Optional[int] = None + + +class NearMissAndSafety(BaseModel): + near_miss_events: Optional[list[NearMissEvent]] = None + safety_breaches: Optional[int] = None + maydays_count: Optional[int] = None + attacks_on_personnel: Optional[AttacksOnPersonnel] = None + """ + Attacks on responders travelling to, at, or from the incident (UK IRS 3.10-3.13) + """ + weather_related_risks: Optional[list[str]] = None + + +class ReportVersion(Enum): + initial = "initial" + update = "update" + final = "final" + + +class ComplexityLevel1(Enum): + type_5 = "type_5" + type_4 = "type_4" + type_3 = "type_3" + type_2 = "type_2" + type_1 = "type_1" + + +class ThreatManagementEnum(Enum): + no_likely_threat = "no_likely_threat" + potential_future_threat = "potential_future_threat" + mass_notifications_in_progress = "mass_notifications_in_progress" + mass_notifications_completed = "mass_notifications_completed" + no_evacuations_imminent = "no_evacuations_imminent" + planning_for_evacuation = "planning_for_evacuation" + planning_for_shelter_in_place = "planning_for_shelter_in_place" + evacuations_in_progress = "evacuations_in_progress" + shelter_in_place_in_progress = "shelter_in_place_in_progress" + repopulation_in_progress = "repopulation_in_progress" + area_restriction_in_effect = "area_restriction_in_effect" + other = "other" + + +class ProjectedActivity(BaseModel): + """ + Projected incident activity by timeframe + """ + + next_12_hours: Optional[str] = None + next_24_hours: Optional[str] = None + next_48_hours: Optional[str] = None + next_72_hours: Optional[str] = None + beyond_72_hours: Optional[str] = None + + +class SituationStatus(BaseModel): + """ + Evolving large-incident status for sitreps (ICS-209 shape). Person counts + live in casualties and evacuation_displacement; structure counts live in + structure. Mappers compose the ICS-209 matrices from those sections. + + """ + + report_version: Optional[ReportVersion] = None + report_number: Optional[int] = None + period_from: Optional[AwareDatetime] = None + period_to: Optional[AwareDatetime] = None + complexity_level: Optional[ComplexityLevel1] = None + imt_type: Optional[str] = None + """ + Incident management organization (single resource, type 3 IMT, unified command) + """ + significant_events: Optional[str] = None + primary_hazards: Optional[str] = None + """ + Primary materials or hazards involved + """ + threat_management: Optional[list[ThreatManagementEnum]] = None + """ + Active protective actions + """ + projected_activity: Optional[ProjectedActivity] = None + """ + Projected incident activity by timeframe + """ + strategic_objectives: Optional[str] = None + threat_summary: Optional[str] = None + critical_resource_needs: Optional[list[str]] = None + planned_actions: Optional[str] = None + projected_final_size_ha: Optional[float] = None + anticipated_completion_date: Optional[date_type] = None + demobilization_start_date: Optional[date_type] = None + costs_to_date: Optional[Money] = None + projected_final_cost: Optional[Money] = None + + +class LessonsLearned(BaseModel): + successful_tactics: Optional[list[str]] = None + areas_for_improvement: Optional[list[str]] = None + recommendations: Optional[list[str]] = None + + +class MopUp(BaseModel): + percent_complete: Optional[int] = None + estimated_completion_date: Optional[date_type] = None + personnel_assigned: Optional[int] = None + + +class Rehabilitation(BaseModel): + erosion_control_ha: Optional[float] = None + reseeding_ha: Optional[float] = None + hazard_tree_removal_required: Optional[bool] = None + + +class FollowUp(BaseModel): + mop_up: Optional[MopUp] = None + rehabilitation: Optional[Rehabilitation] = None + next_inspection_date: Optional[date_type] = None + + +class PeriodicReporting(BaseModel): + contributes_to_monthly_report: Optional[bool] = None + contributes_to_quarterly_report: Optional[bool] = None + contributes_to_annual_report: Optional[bool] = None + neris_submitted: Optional[bool] = None + neris_submitted_at: Optional[AwareDatetime] = None + state_submitted: Optional[bool] = None + state_submitted_at: Optional[AwareDatetime] = None + + +class AttachmentRef(BaseModel): + ref_id: Optional[UUID] = None + filename: Optional[str] = None + content_type: Optional[str] = None + size_bytes: Optional[int] = None + + +class Attachments(BaseModel): + maps: Optional[bool] = None + photos_count: Optional[int] = None + weather_charts: Optional[bool] = None + resource_tracking_logs: Optional[bool] = None + incident_action_plans: Optional[bool] = None + attachment_refs: Optional[list[AttachmentRef]] = None + + +class ExtractionMetadata(BaseModel): + extract_id: Optional[UUID] = None + input_id: Optional[UUID] = None + input_type: Optional[InputType] = None + extracted_at: Optional[AwareDatetime] = None + llm_model: Annotated[Optional[str], Field(None, examples=["llama3:8b"])] + confidence_score: Annotated[Optional[float], Field(None, ge=0.0, le=1.0)] + """ + Overall confidence score from the LLM extraction (0.0-1.0) + """ + completeness: Optional[Completeness] = None + + +class SubmissionLogItem(BaseModel): + form_type: Optional[FormType] = None + submitted_at: Optional[AwareDatetime] = None + submitted_to: Optional[str] = None + + +class ReportMetadata(BaseModel): + report_id: Annotated[Optional[str], Field(None, examples=["FF-2024-CA-0157"])] + incident_number: Annotated[Optional[str], Field(None, examples=["CA-SQF-2024-0421"])] + """ + Department's own incident number + """ + external_ids: Optional[list[ExternalId]] = None + """ + Identifiers for this incident in external systems (CAD event, IRWIN, state registry, partner agency) + """ + report_date: Optional[date_type] = None + report_time: Optional[time_type] = None + report_status: Optional[ReportStatus] = None + reporting_unit: Optional[ReportingUnit] = None + prepared_by: Optional[list[Personnel]] = None + """ + Member(s) making the report + """ + officer_in_charge: Optional[Personnel] = None + reviewed_by: Optional[list[Reviewer]] = None + submission_log: Optional[list[SubmissionLogItem]] = None + + +class IncidentType(BaseModel): + primary: Optional[bool] = None + """ + Only one entry may be primary + """ + category: Optional[IncidentCategory] = None + subcategory: Optional[str] = None + """ + Free-form specific type within the category + """ + codes: Optional[list[CodeRef]] = None + """ + This type expressed in external coding schemes + """ + + +class RespondingAgencies(BaseModel): + primary_agency: Optional[str] = None + all_agencies: Optional[list[RespondingAgency]] = None + mutual_aid_activated: Optional[bool] = None + aid_direction: Optional[AidDirection] = None + aid_type: Optional[AidType] = None + non_fd_entities: Optional[list[str]] = None + """ + Non fire department entities that assisted (utility, red cross, public works) + """ + unified_command: Optional[bool] = None + incident_commander: Optional[IncidentCommander] = None + + +class Fire(BaseModel): + cause_category: Optional[CauseCategory] = None + cause_specific: Optional[str] = None + cause_codes: Optional[list[CodeRef]] = None + cause_certainty: Optional[CauseCertainty] = None + arson_suspected: Optional[bool] = None + area_of_origin: Optional[str] = None + """ + Room or area where the fire began + """ + heat_source: Optional[str] = None + """ + What provided the heat that started the fire + """ + ignition_power_source: Optional[str] = None + """ + What powered the ignition source (mains, battery, gas, open flame) + """ + item_first_ignited: Optional[str] = None + material_first_ignited: Optional[str] = None + multiple_seats_of_fire: Optional[bool] = None + """ + More than one independent point of origin (arson indicator) + """ + human_factors: Optional[list[HumanFactor]] = None + """ + Human factors contributing to ignition + """ + person_involved_age: Optional[int] = None + """ + Estimated age of person whose age was a factor + """ + person_involved_sex: Optional[str] = None + contributing_factors: Optional[list[str]] = None + """ + Non-human factors contributing to ignition + """ + equipment_involved: Optional[EquipmentInvolved] = None + """ + Equipment involved in ignition, if any + """ + on_site_materials: Optional[list[str]] = None + """ + Significant commercial/industrial/agricultural materials on the property + """ + fuel_types: Optional[list[str]] = None + fire_spread_directions: Optional[list[str]] = None + rate_of_spread: Optional[RateOfSpread] = None + rate_of_spread_m_per_min: Optional[float] = None + flame_length_m: Optional[float] = None + spotting_distance_km: Optional[float] = None + unusual_behaviors: Optional[list[str]] = None + rapid_growth_cause: Optional[str] = None + """ + Cause of any rapid fire growth (UK IRS 8.8) + """ + fire_suppression_factors: Optional[list[str]] = None + """ + Factors that helped or hindered suppression + """ + suppression_operations: Optional[SuppressionOperations] = None + + +class CivilianCasualty(BaseModel): + name: Optional[str] = None + age: Optional[int] = None + date_of_birth: Optional[date_type] = None + sex: Optional[str] = None + race_ethnicity: Optional[str] = None + """ + Only where the target jurisdiction collects it (US, UK) + """ + affiliation: Optional[Affiliation] = None + injury_datetime: Optional[AwareDatetime] = None + injury_type: Optional[str] = None + """ + Nature of the injury (burns, smoke inhalation, trauma) + """ + primary_symptom: Optional[str] = None + primary_body_part: Optional[str] = None + severity: Optional[InjurySeverity] = None + cause: Optional[str] = None + human_factors: Optional[list[HumanFactor1]] = None + contributing_factors: Optional[list[str]] = None + activity_when_injured: Optional[ActivityWhenInjured] = None + location_at_ignition: Optional[LocationAtIgnition] = None + story_at_start: Optional[int] = None + story_where_injured: Optional[int] = None + location_where_found: Optional[str] = None + cause_of_failure_to_escape: Optional[str] = None + """ + Why the person could not escape (Canada NFID casualty file) + """ + disposition: Optional[str] = None + transported: Optional[bool] = None + hospital: Optional[str] = None + fatal_circumstances: Optional[str] = None + death_certificate_reconciled: Optional[bool] = None + + +class EMS(BaseModel): + ems_response_required: Optional[bool] = None + patients: Optional[list[EMSPatient]] = None + total_patients: Optional[int] = None + ems_agency_responded: Optional[str] = None + nemsis_report_required: Optional[bool] = None + nemsis_report_ids: Optional[list[str]] = None + """ + Linked ePCR ids (NEMSIS eRecord.01); the full ePCR stays in the EMS system + """ + + +class Hazmat(BaseModel): + involved: Optional[bool] = None + materials: Optional[list[HazmatMaterial]] = None + ignition_or_release_first: Optional[IgnitionOrReleaseFirst] = None + release_cause: Optional[ReleaseCause] = None + release_factors: Optional[list[str]] = None + mitigation_factors: Optional[list[str]] = None + """ + Factors or impediments that affected mitigation + """ + actions_taken: Optional[list[str]] = None + """ + Hazmat-specific actions (identification, containment, decontamination, neutralization) + """ + equipment_involved_in_release: Optional[EquipmentInvolvedInRelease] = None + area_affected: Optional[Quantity] = None + area_evacuated: Optional[Quantity] = None + epa_reportable_quantity_exceeded: Optional[bool] = None + disposition: Optional[Disposition1] = None + """ + Who the cleanup/scene was released to. Evacuee counts live in evacuation_displacement. + """ + + +class Investigation(BaseModel): + investigation_needed: Optional[bool] = None + """ + Incident commander's assessment that formal investigation is required + """ + investigation_types: Optional[list[str]] = None + """ + Types of investigation completed (origin_and_cause, arson, insurance, forensic) + """ + investigation_ongoing: Optional[bool] = None + case_status: Optional[CaseStatus] = None + agency_referred_to: Optional[AgencyReferredTo] = None + law_enforcement_notified: Optional[bool] = None + evidence_collected: Optional[bool] = None + laboratory_used: Optional[str] = None + nibrs_report_required: Optional[bool] = None + arson: Optional[Arson] = None + notes: Optional[str] = None + + +class Weather(BaseModel): + on_arrival: Optional[WeatherReading] = None + worst_conditions: Optional[WeatherReading] = None + weather_type: Optional[WeatherType] = None + factors_influencing_fire: Optional[list[str]] = None + + +class InfrastructureImpact(BaseModel): + items: Optional[list[InfrastructureItem]] = None + + +class Incident(BaseModel): + name: Optional[str] = None + """ + Human-readable incident name + """ + types: Optional[list[IncidentType]] = None + special_modifiers: Annotated[ + Optional[list[str]], Field(None, examples=[["mass_casualty", "major_incident"]]) + ] + """ + Magnitude or class tags qualifying the incident (NERIS special modifiers) + """ + false_alarm: Optional[FalseAlarm] = None + """ + Populated when the final type is a false alarm + """ + special_service_type: Optional[str] = None + """ + Non-fire service call subtype (lift release, lock-in, flooding, animal assist, co-response) + """ + chimney_fire: Optional[bool] = None + """ + Flame confined to a chimney (UK IRS 3.9) + """ + timezone: Annotated[Optional[str], Field(None, examples=["America/Los_Angeles"])] + """ + IANA timezone of the incident, for local wall-clock rendering + """ + discovered_datetime: Optional[AwareDatetime] = None + how_discovered: Optional[str] = None + """ + How the fire or emergency was first discovered + """ + delay_ignition_to_discovery: Optional[DelayIgnitionToDiscovery] = None + delay_discovery_to_call: Optional[DelayDiscoveryToCall] = None + start_datetime: Optional[AwareDatetime] = None + """ + Estimated ignition or emergency start + """ + alarm_datetime: Optional[AwareDatetime] = None + """ + Department alerted / alarm time + """ + first_arrival_datetime: Optional[AwareDatetime] = None + """ + First unit on scene + """ + command_established_datetime: Optional[AwareDatetime] = None + sizeup_complete_datetime: Optional[AwareDatetime] = None + water_on_fire_datetime: Optional[AwareDatetime] = None + primary_search_begin_datetime: Optional[AwareDatetime] = None + primary_search_complete_datetime: Optional[AwareDatetime] = None + knocked_down_datetime: Optional[AwareDatetime] = None + containment_datetime: Optional[AwareDatetime] = None + controlled_datetime: Optional[AwareDatetime] = None + extrication_complete_datetime: Optional[AwareDatetime] = None + suppression_complete_datetime: Optional[AwareDatetime] = None + loss_stopped_datetime: Optional[AwareDatetime] = None + stop_message_datetime: Optional[AwareDatetime] = None + """ + Stop / situation-under-control message to control room (UK IRS 2.5) + """ + cleared_datetime: Optional[AwareDatetime] = None + """ + Last unit cleared the scene + """ + closed_datetime: Optional[AwareDatetime] = None + """ + Incident administratively closed + """ + total_duration_hours: Optional[float] = None + alarm_level: Optional[int] = None + """ + Number of alarms / escalation level + """ + shift_or_platoon: Optional[str] = None + district: Optional[str] = None + """ + Response district or box area + """ + people_present: Optional[bool] = None + """ + Whether people were present at the location at the time + """ + animals_rescued: Optional[int] = None + animals_deceased: Optional[int] = None + narrative: Optional[str] = None + """ + Free text summary of the incident + """ + narrative_impediment: Optional[str] = None + """ + Obstacles that impacted the response (NERIS) + """ + narrative_outcome: Optional[str] = None + """ + Final disposition of the incident (NERIS) + """ + raw_transcript: Optional[str] = None + """ + Original voice or text input verbatim + """ + + +class Casualties(BaseModel): + """ + Injuries and deaths. Uninjured rescues live in rescues[]. + """ + + civilian: Optional[list[CivilianCasualty]] = None + responder: Optional[list[ResponderCasualty]] = None + total_civilian_injuries: Optional[int] = None + total_civilian_fatalities: Optional[int] = None + total_responder_injuries: Optional[int] = None + total_responder_fatalities: Optional[int] = None + + +class IncidentContract(BaseModel): + """ + The FireForm incident contract. This is the superset schema + containing every field any downstream form could need. Form-specific + mappers select only the relevant fields for each agency template. + Stored as a single JSONB document; queryable stats are promoted to + IncidentRecord columns server-side. + + """ + + schema_version: Annotated[Optional[str], Field(None, examples=["1.1.0"])] + """ + Schema version identifier + """ + schema_name: Optional[SchemaName] = None + """ + Schema name identifier + """ + extraction_metadata: Optional[ExtractionMetadata] = None + report_metadata: Optional[ReportMetadata] = None + incident: Optional[Incident] = None + dispatch: Optional[Dispatch] = None + location: Optional[Location] = None + actions_taken: Optional[ActionsTaken] = None + responding_agencies: Optional[RespondingAgencies] = None + units: Optional[list[UnitResponse]] = None + """ + Per-unit (apparatus/resource) response records with timestamps + """ + resources_summary: Optional[ResourcesSummary] = None + fire: Optional[Fire] = None + explosion: Optional[Explosion] = None + risk_reduction: Optional[RiskReduction] = None + structure: Optional[Structure] = None + wildland: Optional[Wildland] = None + exposures: Optional[list[Exposure]] = None + """ + Properties beyond the origin affected by spread of the incident + """ + casualties: Optional[Casualties] = None + rescues: Optional[list[Rescue]] = None + """ + Rescues and assisted evacuations, with or without injury + """ + evacuation_displacement: Optional[EvacuationDisplacement] = None + ems: Optional[EMS] = None + hazmat: Optional[Hazmat] = None + emerging_hazards: Optional[list[EmergingHazard]] = None + """ + Battery, EV, solar and other stored-energy hazards (NERIS) + """ + investigation: Optional[Investigation] = None + persons_involved: Optional[list[PersonInvolved]] = None + """ + Owners, occupants and other parties connected to the incident + """ + mobile_property: Optional[list[MobileProperty]] = None + """ + Vehicles, vessels and other mobile property involved + """ + losses: Optional[Losses] = None + weather: Optional[Weather] = None + environmental_impact: Optional[EnvironmentalImpact] = None + infrastructure_impact: Optional[InfrastructureImpact] = None + near_miss_and_safety: Optional[NearMissAndSafety] = None + situation_status: Optional[SituationStatus] = None + lessons_learned: Optional[LessonsLearned] = None + follow_up: Optional[FollowUp] = None + periodic_reporting: Optional[PeriodicReporting] = None + attachments: Optional[Attachments] = None + custom_fields: Optional[dict[str, Any]] = None + """ + Agency-local fields with no contract home (NFIRS special studies, + IRS local options and similar). Keys are agency-defined strings, + values are scalars. Passed through to mappers untouched. + + """ diff --git a/app/api/schemas/incidents.py b/app/api/schemas/incidents.py new file mode 100644 index 00000000..4319f867 --- /dev/null +++ b/app/api/schemas/incidents.py @@ -0,0 +1,166 @@ +"""Contract Layer 4 incident schemas (contracts/schemas/incident-record.yaml). + +The DB stores the promoted analytics as flat columns on the incident row, but +the contract nests them under an `analytics` object. `IncidentAnalytics` owns +that reshaping so routes and services never assemble the block by hand. +""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from app.api.schemas.common import Pagination +from app.api.schemas.enums import FormStatus, IncidentCategory, ReportStatus +from app.api.schemas.form_generation import FormRecord +from app.api.schemas.incident_contract import IncidentContract + + +# --------------------------------------------------------------------------- +# Requests +# --------------------------------------------------------------------------- + +class CreateIncidentRequest(BaseModel): + """POST /incidents body. + + Finalizes the draft incident that was created when the extraction + completed. `extract_id` resolves to that existing row; a second row is + never created for the same extraction. + """ + + extract_id: UUID + incident_number: str | None = None + tags: list[str] | None = None + + +class UpdateIncidentRequest(BaseModel): + """PATCH /incidents/{id} body. + + A partial update: only the fields actually present in the request body are + applied, so omitting a field leaves it untouched rather than nulling it. + Callers read that distinction off `model_fields_set`, which is why no + field here carries a meaningful default. + """ + + status: ReportStatus | None = None + tags: list[str] | None = None + incident_number: str | None = None + notes: str | None = None + + +# --------------------------------------------------------------------------- +# Responses +# --------------------------------------------------------------------------- + +class IncidentAnalytics(BaseModel): + """Read-only stats promoted out of the incident contract. + + Recomputed server-side by `app.services.incidents.promote` on every change + to the document. Clients never write these. + """ + + model_config = ConfigDict(from_attributes=True) + + city: str | None = None + state: str | None = None + country: str | None = None + civilian_injuries: int | None = None + civilian_fatalities: int | None = None + responder_injuries: int | None = None + responder_fatalities: int | None = None + people_rescued: int | None = None + people_evacuated: int | None = None + structures_destroyed: int | None = None + area_burned_ha: float | None = None + total_loss_amount: float | None = None + total_loss_currency: str | None = None + call_to_arrival_seconds: int | None = None + turnout_seconds_first_unit: int | None = None + travel_seconds_first_unit: int | None = None + on_scene_duration_seconds: int | None = None + + +class GeneratedForm(BaseModel): + """One entry in `forms_generated`, the at-a-glance form summary.""" + + form_id: UUID + # Open string rather than the FormType enum, matching FormRecord: a + # registry can hold form types the closed enum does not know about yet. + form_type: str + status: FormStatus + + +class IncidentRecord(BaseModel): + """The incident record without its contract document or full form rows.""" + + incident_id: UUID + extract_id: UUID + incident_number: str | None = None + status: ReportStatus + incident_name: str | None = None + incident_type: str | None = None + incident_category: IncidentCategory | None = None + incident_datetime: datetime | None = None + analytics: IncidentAnalytics | None = None + forms_generated: list[GeneratedForm] = Field(default_factory=list) + tags: list[str] = Field(default_factory=list) + notes: str | None = None + created_at: datetime + updated_at: datetime + deleted_at: datetime | None = None + + +class SubmissionLogEntry(BaseModel): + """One agency submission, read out of the contract document. + + Stays empty until the submission layer exists; nothing writes it today. + """ + + form_type: str | None = None + submitted_at: datetime | None = None + submitted_to: str | None = None + status: str | None = None + + +class IncidentRecordFull(IncidentRecord): + """GET /incidents/{id}: the record plus everything hanging off it.""" + + incident_contract: IncidentContract | None = None + forms: list[FormRecord] = Field(default_factory=list) + submission_log: list[SubmissionLogEntry] = Field(default_factory=list) + + +class IncidentListItem(BaseModel): + """One row of GET /incidents. + + Deliberately flatter than IncidentRecord: a list view needs the location + columns but not the whole analytics block, and a form count rather than + the forms themselves. + """ + + incident_id: UUID + incident_number: str | None = None + status: ReportStatus + incident_name: str | None = None + incident_type: str | None = None + incident_category: IncidentCategory | None = None + incident_datetime: datetime | None = None + city: str | None = None + country: str | None = None + forms_count: int = 0 + created_at: datetime + + +class IncidentListResponse(BaseModel): + data: list[IncidentListItem] = Field(default_factory=list) + pagination: Pagination + + +class DeleteIncidentResponse(BaseModel): + """200 body for DELETE /incidents/{id}. The row is never removed.""" + + incident_id: UUID + deleted_at: datetime + recoverable: bool = True diff --git a/app/api/schemas/system.py b/app/api/schemas/system.py index bafa5cfc..7c0db95d 100644 --- a/app/api/schemas/system.py +++ b/app/api/schemas/system.py @@ -1,32 +1,21 @@ from pydantic import BaseModel -class ModelInfo(BaseModel): - name: str - size_gb: float - quantization: str | None = None - loaded: bool - - -class CurrentLoad(BaseModel): - active_requests: int - queued_requests: int - - class ComponentHealth(BaseModel): status: str response_time_ms: int | None = None detail: str | None = None - model_loaded: str | None = None - ollama_version: str | None = None - models_available: list[ModelInfo] | None = None - current_load: CurrentLoad | None = None disk_free_gb: float | None = None + provider: str | None = None + model: str | None = None + external: bool | None = None + probed: bool | None = None + models_available: list[str] | None = None class HealthComponents(BaseModel): database: ComponentHealth - ollama: ComponentHealth + llm: ComponentHealth whisper: ComponentHealth storage: ComponentHealth @@ -43,3 +32,32 @@ class SchemaVersion(BaseModel): released_at: str changelog: str | None = None breaking_changes: bool | None = None + + +class SchemaFieldEntry(BaseModel): + """One leaf field of the incident contract, as the catalog exposes it + (contracts/schemas/template-record.yaml#/SchemaFieldEntry). + + Array hops show as `[]`, so a person's address reads + `persons_involved[].address`. + """ + + path: str + label: str | None = None + field_type: str + section: str + description: str | None = None + enum_values: list[str] | None = None + pii: bool = False + aliases: list[str] = [] + # Only set on search results, absent when the whole catalog is listed. + score: float | None = None + + +class SchemaFieldSearchResponse(BaseModel): + """GET /schema/fields response (path/system.yaml#/schema_fields).""" + + query: str | None = None + total: int + schema_version: str | None = None + fields: list[SchemaFieldEntry] diff --git a/app/api/schemas/templates.py b/app/api/schemas/templates.py index df39832b..a192aaf4 100644 --- a/app/api/schemas/templates.py +++ b/app/api/schemas/templates.py @@ -1,38 +1,266 @@ -from pydantic import BaseModel +import re +from datetime import date, datetime +from uuid import UUID -class TemplateCreate(BaseModel): - name: str - pdf_path: str - fields: dict +from pydantic import BaseModel, Field, field_validator, model_validator +from app.api.schemas.enums import ( + DetectionStatus, + FieldSource, + TemplateFieldType, + TemplateStatus, + TextAlign, +) -class MakeFillableRequest(BaseModel): - pdf_path: str +_HEX_COLOR = re.compile(r"^#[0-9A-Fa-f]{6}$") +_FORM_TYPE = re.compile(r"^[a-z0-9_-]+$") -class MakeFillableResponse(BaseModel): - pdf_path: str - field_count: int | None = None +# --------------------------------------------------------------------------- +# Contract Layer 6 schemas (contracts/schemas/template-record.yaml) +# --------------------------------------------------------------------------- +class TemplateFieldLayout(BaseModel): + """Visual placement of a field on the PDF (schemas/template-record.yaml#/TemplateFieldLayout). -class TemplateResponse(BaseModel): - id: int - name: str - pdf_path: str - fields: dict - field_count: int | None = None + Coordinates are PDF points with the origin at the bottom-left of the page: + box = start (x, y) .. end (x + width, y + height). + """ - class Config: - from_attributes = True + page: int = Field(ge=0, description="Zero-based page index") + x: float = Field(ge=0, description="Lower-left X in PDF points") + y: float = Field(ge=0, description="Lower-left Y in PDF points (origin bottom-left)") + width: float = Field(gt=0) + height: float = Field(gt=0) + font: str = "Helvetica" + font_size: float = Field(default=10, gt=0) + color: str = "#000000" + align: TextAlign = TextAlign.left + @field_validator("font") + @classmethod + def _font_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("font must not be blank") + return v -class ExtractedField(BaseModel): - name: str - description: str - type: str + @field_validator("color") + @classmethod + def _color_is_hex(cls, v: str) -> str: + if not _HEX_COLOR.match(v): + raise ValueError('color must be a hex string like "#000000"') + return v -class TemplateUploadResponse(BaseModel): - filename: str - pdf_path: str - field_count: int | None = None - fields: list[ExtractedField] = [] +class TemplateField(BaseModel): + """One field definition within a template (schemas/template-record.yaml#/TemplateField). + + `source` decides where the value comes from, and each source brings its own + requirement: schema needs an `incident_mapping`, static needs `static_text`, + open needs a `description` (it is the instruction the extractor is given), + and manual needs neither. `layout` places the box on the PDF. + """ + + field_name: str + field_type: TemplateFieldType + source: FieldSource + required: bool + description: str | None = None + max_length: int | None = Field(default=None, gt=0) + # No bound on min/max themselves: contract values legitimately go negative + # (temperatures, elevations). Only their order is checked below. + min_value: float | None = None + max_value: float | None = None + allowed_values: list[str] | None = None + incident_mapping: str | None = None + static_text: str | None = None + default_value: object | None = None + unit: str | None = None + layout: TemplateFieldLayout | None = None + + @field_validator("field_name") + @classmethod + def _name_not_blank(cls, v: str) -> str: + v = v.strip() + if not v: + raise ValueError("field_name must not be empty") + return v + + @model_validator(mode="after") + def _check_field(self) -> "TemplateField": + has_mapping = bool(self.incident_mapping and self.incident_mapping.strip()) + has_static = self.static_text is not None + has_description = bool(self.description and self.description.strip()) + + if self.source is FieldSource.schema and not has_mapping: + raise ValueError( + f"field '{self.field_name}': incident_mapping is required when source is 'schema'" + ) + if self.source is not FieldSource.schema and has_mapping: + raise ValueError( + f"field '{self.field_name}': incident_mapping is only allowed when source is 'schema'" + ) + if self.source is FieldSource.static and not has_static: + raise ValueError( + f"field '{self.field_name}': static_text is required when source is 'static'" + ) + if self.source is not FieldSource.static and has_static: + raise ValueError( + f"field '{self.field_name}': static_text is only allowed when source is 'static'" + ) + if self.source is FieldSource.open and not has_description: + raise ValueError( + f"field '{self.field_name}': description is required when source is 'open'. " + "It is the instruction the extractor is given." + ) + if self.field_type == TemplateFieldType.enum and not self.allowed_values: + raise ValueError( + f"field '{self.field_name}': allowed_values is required when field_type is 'enum'" + ) + if ( + self.min_value is not None + and self.max_value is not None + and self.min_value > self.max_value + ): + raise ValueError( + f"field '{self.field_name}': min_value must be <= max_value" + ) + return self + + +class CreateTemplateRequest(BaseModel): + """POST/PUT request body (schemas/template-record.yaml#/CreateTemplateRequest).""" + + form_type: str + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + fields: list[TemplateField] = Field(min_length=1) + source_standard: str | None = None + pdf_template_ref: str | None = None + + @field_validator("form_type") + @classmethod + def _form_type_slug(cls, v: str) -> str: + v = v.strip() + if not _FORM_TYPE.match(v): + raise ValueError( + "form_type must contain only lowercase letters, digits, underscores or hyphens" + ) + return v + + @field_validator("display_name") + @classmethod + def _display_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name must not be empty") + return v + + @model_validator(mode="after") + def _unique_field_names(self) -> "CreateTemplateRequest": + names = [f.field_name for f in self.fields] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + raise ValueError(f"duplicate field_name(s): {', '.join(dupes)}") + return self + + +class TemplateSummary(BaseModel): + """List item (schemas/template-record.yaml#/TemplateSummary).""" + + template_id: UUID + form_type: str + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + version: str + last_updated: date + field_count: int + status: TemplateStatus + + +class TemplateDetail(CreateTemplateRequest): + """Full template definition (schemas/template-record.yaml#/Template). + + Server-generated fields layered on top of CreateTemplateRequest. + """ + + template_id: UUID + version: str + last_updated: date + field_count: int + status: TemplateStatus + created_at: datetime + updated_at: datetime + + +class TemplateFieldsResponse(BaseModel): + """GET /templates/{id}/fields response (path/templates.yaml#/template_fields).""" + + template_id: UUID + form_type: str + total_fields: int + required_fields: int + optional_fields: int + fields: list[TemplateField] + + +# --------------------------------------------------------------------------- +# PDF upload and field detection (path/templates.yaml#/templates_pdf) +# --------------------------------------------------------------------------- +class PageGeometry(BaseModel): + """One page's size in PDF points, the unit every layout box uses.""" + + page: int = Field(ge=0, description="Zero-based page index") + width: float + height: float + + +class MappingSuggestion(BaseModel): + """One ranked incident-contract mapping guess for a detected box.""" + + path: str + label: str | None = None + field_type: str | None = None + section: str | None = None + description: str | None = None + score: float + + +class DraftField(BaseModel): + """A detected box as an editable TemplateField, plus what it was guessed from.""" + + field: TemplateField + # Kept verbatim, not normalized, so the editor can show the user what the + # suggestion was based on. + detected_label: str | None = None + suggestions: list[MappingSuggestion] = Field(default_factory=list) + + +class TemplateDraft(BaseModel): + """Everything the visual editor needs after a PDF upload. + + The PDF is stored the moment the upload returns; only detection is async, + so `status` describes detection alone. A failed detection still leaves a + usable upload, the user just draws every box by hand. + """ + + upload_id: UUID + status: DetectionStatus + pdf_template_ref: str + original_filename: str | None = None + page_count: int + pages: list[PageGeometry] + detected_fields: list[DraftField] | None = None + detection_error: str | None = None + retry_after_seconds: int | None = None + + +class TemplateDraftAccepted(TemplateDraft): + """202 body of POST /templates/pdf: the draft plus where to poll it. + + `job_id` is absent when detection was skipped, since there is no background + work to follow. `poll_url` still resolves, it just answers straight away. + """ + + job_id: str | None = None + poll_url: str diff --git a/app/core/celery.py b/app/core/celery.py index a91146f7..b498e8e7 100644 --- a/app/core/celery.py +++ b/app/core/celery.py @@ -1,4 +1,7 @@ +import sys + from celery import Celery +from celery.signals import celeryd_init from app.core.config import CELERY_BROKER_URL, CELERY_RESULT_BACKEND @@ -16,7 +19,32 @@ result_expires=86400, ) -celery_app.conf.include = ["app.tasks.fill", "app.tasks.purge", "app.tasks.transcribe"] +@celeryd_init.connect +def _check_llm_config(**_kwargs): + """Stop the worker on a bad LLM configuration, before it accepts any task. + + SystemExit rather than letting LLMConfigError travel: Celery catches + anything deriving from Exception in a signal handler, logs it and carries + on, so a raised config error would leave the worker running and picking up + extractions it cannot serve. SystemExit is not an Exception, so it lands. + """ + from app.services import llm + + try: + llm.check_config() + except llm.LLMConfigError as exc: + print(f"FATAL: {exc}", file=sys.stderr, flush=True) + raise SystemExit(1) from exc + + +celery_app.conf.include = [ + "app.tasks.fill", + "app.tasks.purge", + "app.tasks.transcribe", + "app.tasks.extract", + "app.tasks.detect_fields", + "app.tasks.generate_forms", +] # Optional Celery Beat schedule — runs purge_old_submissions once a day. # Enable by running: celery -A app.core.celery beat diff --git a/app/core/config.py b/app/core/config.py index 4a7ca897..183e6828 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -35,8 +35,53 @@ # --- External services ---------------------------------------------------- OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434").rstrip("/") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b") +# Seconds to wait on a single Ollama generate call. One chunk prompt on a small +# local model is slow, so this is generous by design. +OLLAMA_TIMEOUT = int(os.getenv("OLLAMA_TIMEOUT", "600")) +# Ceiling on tokens generated per chunk answer. It stops a small model from +# echoing the whole field skeleton back and spending the timeout on it. +# +# Keep these two in step. A section that runs past OLLAMA_TIMEOUT is lost +# outright, while one that hits this ceiling still keeps every field it +# completed before the cut, so the ceiling should be reachable well inside the +# timeout on the slowest model you run. A small model on CPU manages roughly +# three tokens a second, which is where these defaults come from. +OLLAMA_MAX_TOKENS = int(os.getenv("OLLAMA_MAX_TOKENS", "1200")) WHISPER_HOST = os.getenv("WHISPER_HOST", "http://localhost:9000").rstrip("/") +# --- LLM provider --------------------------------------------------------- +# Which backend answers prompts, one per deployment. "custom" points at any +# endpoint speaking the OpenAI chat completions API and needs LLM_BASE_URL. +# The OLLAMA_* settings above stay the Ollama provider's defaults, so an +# existing .env keeps working untouched. Meanings are documented in +# docker/.env.example. +LLM_PROVIDER = os.getenv("LLM_PROVIDER", "ollama").strip().lower() +LLM_MODEL = os.getenv("LLM_MODEL", "").strip() +LLM_BASE_URL = os.getenv("LLM_BASE_URL", "").strip().rstrip("/") + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip() +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "").strip() +ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip() +LLM_API_KEY = os.getenv("LLM_API_KEY", "").strip() + +LLM_EXTRA_HEADERS = os.getenv("LLM_EXTRA_HEADERS", "").strip() +LLM_TIMEOUT = int(os.getenv("LLM_TIMEOUT", str(OLLAMA_TIMEOUT))) +LLM_MAX_TOKENS = int(os.getenv("LLM_MAX_TOKENS", str(OLLAMA_MAX_TOKENS))) + +# Hosted providers mean incident narratives leave the department's hardware, +# which is the one thing FireForm promises will not happen. They refuse to +# start until this is switched on deliberately. +LLM_ALLOW_EXTERNAL = os.getenv("LLM_ALLOW_EXTERNAL", "false").strip().lower() == "true" + +# A 429 is the provider asking us to wait, so it is worth waiting out. Anything +# still limited after all of these tries is a quota problem no retry fixes. +LLM_RATE_LIMIT_RETRIES = int(os.getenv("LLM_RATE_LIMIT_RETRIES", "10")) +LLM_RATE_LIMIT_WAIT_SECONDS = float(os.getenv("LLM_RATE_LIMIT_WAIT_SECONDS", "10")) +LLM_RATE_LIMIT_MAX_WAIT_SECONDS = float(os.getenv("LLM_RATE_LIMIT_MAX_WAIT_SECONDS", "60")) +LLM_RESPECT_RETRY_AFTER = os.getenv("LLM_RESPECT_RETRY_AFTER", "true").strip().lower() == "true" +LLM_SERVER_RETRIES = int(os.getenv("LLM_SERVER_RETRIES", "2")) +LLM_SERVER_RETRY_WAIT_SECONDS = float(os.getenv("LLM_SERVER_RETRY_WAIT_SECONDS", "2")) + # --- Celery / Redis ------------------------------------------------------- CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") @@ -58,6 +103,44 @@ # or transcribing. Value matches the contract example (contracts/path/input.yaml). INPUT_POLL_INTERVAL_SECONDS = 5 +# Polling hint returned by GET /extract/{id} while an extraction is still +# processing. Matches the contract example (contracts/path/extraction.yaml). +EXTRACTION_POLL_INTERVAL_SECONDS = 5 + +# Advisory estimate returned in the 202 body of POST /extract/{input_id}. +ESTIMATED_EXTRACTION_SECONDS = int(os.getenv("ESTIMATED_EXTRACTION_SECONDS", "60")) + +# --- Extraction worker ---------------------------------------------------- +# How many chunk prompts run at once. A local server queues whatever it cannot +# serve concurrently, so going wider than it buys nothing but memory pressure. +# A hosted provider will rate limit instead, which the retry policy absorbs. +EXTRACTION_MAX_PARALLEL = int( + os.getenv("LLM_MAX_PARALLEL", os.getenv("OLLAMA_NUM_PARALLEL", "4")) +) + +# Extra attempts after a chunk's first result fails validation. One retry, with +# the rejected value named in the prompt; a chunk that misses twice is left for +# manual entry rather than guessed at. +EXTRACTION_CHUNK_RETRIES = int(os.getenv("EXTRACTION_CHUNK_RETRIES", "1")) + +# One input gets one extraction, which makes re-testing the same narrative mean +# re-uploading it. Switching this on lets a repeat run through; every run still +# gets its own extraction and its own draft incident, so the old ones stay put. +# Development only, leave it off anywhere real. +EXTRACTION_ALLOW_RERUN = os.getenv("EXTRACTION_ALLOW_RERUN", "false").strip().lower() == "true" + +# The contract file the chunk registry reads its tiers and triggers from. +INCIDENT_CONTRACT_PATH = Path( + os.getenv("INCIDENT_CONTRACT_PATH", BASE_DIR / "contracts" / "schemas" / "incident-contract.yaml") +) + +# Deployment context the extractor falls back on when the narrative is silent: +# timezone for resolving relative dates, country, currency for Money amounts. +# A request can override any of them through ExtractionRequest.defaults. +DEFAULT_COUNTRY = os.getenv("FIREFORM_DEFAULT_COUNTRY", "US") +DEFAULT_TIMEZONE = os.getenv("FIREFORM_DEFAULT_TIMEZONE", "UTC") +DEFAULT_CURRENCY = os.getenv("FIREFORM_DEFAULT_CURRENCY", "USD") + # --- API Versioning ------------------------------------------------------- API_PREFIX = "/api/v1" @@ -67,6 +150,26 @@ # --- Data Retention -------------------------------------------------------- RETENTION_PERIOD_DAYS = int(os.getenv("RETENTION_PERIOD_DAYS", "30")) +# --- Template PDF storage and field detection ----------------------------- +# Blank agency PDFs uploaded for template authoring land here as +# {TEMPLATE_UPLOAD_DIR}/{upload_id}.pdf. The reference handed back to clients +# is that path taken relative to DATA_DIR, so the two can never drift apart. +TEMPLATE_UPLOAD_DIR = DATA_DIR / "templates" / "uploads" + +MAX_TEMPLATE_PDF_BYTES = 50 * 1024 * 1024 + +# Polling hint returned by the draft endpoints while detection is running. +TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS = 5 + +# Two thresholds govern the mapping suggester. Below the floor nothing is +# offered at all: an empty list next to a good search box beats a wrong guess, +# because a pre-filled mapping is trusted far more than it deserves. At or above +# the auto-apply mark the top hit is written straight into the field as a +# schema mapping; between the two the suggestions are listed and the user picks. +MAPPING_SUGGESTION_FLOOR = float(os.getenv("MAPPING_SUGGESTION_FLOOR", "0.5")) +MAPPING_AUTO_APPLY_SCORE = float(os.getenv("MAPPING_AUTO_APPLY_SCORE", "0.85")) +MAX_MAPPING_SUGGESTIONS = 5 + # --- Audio storage -------------------------------------------------------- # Voice input audio files land here: {AUDIO_DIR}/{input_id}.{ext} AUDIO_DIR = DATA_DIR / "audio" @@ -84,4 +187,17 @@ "ogg": "audio/ogg", "webm": "audio/webm", } -ALLOWED_AUDIO_EXTENSIONS: frozenset[str] = frozenset(AUDIO_CONTENT_TYPES) \ No newline at end of file +ALLOWED_AUDIO_EXTENSIONS: frozenset[str] = frozenset(AUDIO_CONTENT_TYPES) + +# --- Generated form storage ------------------------------------------------- +# Filled form PDFs land here: {FORMS_OUTPUT_DIR}/{form_id}.pdf. Form.pdf_path +# stores this DATA_DIR-relative, same convention as FormTemplate.pdf_template_ref. +FORMS_OUTPUT_DIR = DATA_DIR / "forms" / "generated" + +# Advisory estimate returned in the 202 body of POST /forms/generate. Filling +# is pure lookup-and-draw (no LLM), so this is far below the extraction estimate. +ESTIMATED_FORM_GENERATION_SECONDS = int(os.getenv("ESTIMATED_FORM_GENERATION_SECONDS", "10")) + +# Polling hint returned by GET /forms/{id}/pdf while generation is still in +# progress. Matches the contract example (contracts/path/forms.yaml). +FORM_GENERATION_POLL_INTERVAL_SECONDS = 5 \ No newline at end of file diff --git a/app/core/errors/base.py b/app/core/errors/base.py index d317736a..ad323d51 100644 --- a/app/core/errors/base.py +++ b/app/core/errors/base.py @@ -21,3 +21,23 @@ def __init__( self.status_code = status_code self.error_code = error_code or _default_error_code(status_code) self.detail = detail + + +class ValidationAppError(AppError): + """A 422 raised by a service with field-level issues attached. + + Request-shape problems are caught by FastAPI and rendered from + RequestValidationError. This is for the ones only the service can see, such + as a correction whose merged document no longer matches the contract, and + it renders the same validation_errors list so clients read one shape. + """ + + def __init__( + self, + message: str, + validation_errors: list[dict] | None = None, + error_code: str = "VALIDATION_ERROR", + detail: dict | None = None, + ): + super().__init__(message, status_code=422, error_code=error_code, detail=detail) + self.validation_errors = validation_errors or [] diff --git a/app/core/errors/handlers.py b/app/core/errors/handlers.py index ce8988ad..c57378ed 100644 --- a/app/core/errors/handlers.py +++ b/app/core/errors/handlers.py @@ -1,9 +1,28 @@ +import json + from fastapi import Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from app.core.config import RETRY_AFTER_SECONDS -from app.core.errors.base import AppError +from app.core.errors.base import AppError, ValidationAppError + + +def _jsonable(value): + """Return value untouched if JSON-serializable, else a string form. + + FastAPI puts the offending input on each validation error. When a JSON-body + endpoint is called with the wrong content-type, that input is the raw + request body as bytes, which JSONResponse cannot encode. Coerce anything + non-serializable so the 422 renders instead of turning into a 500. + """ + try: + json.dumps(value) + return value + except (TypeError, ValueError): + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) def register_exception_handlers(app): @@ -16,6 +35,18 @@ async def app_error_handler(request: Request, exc: AppError): body["retry_after_seconds"] = RETRY_AFTER_SECONDS return JSONResponse(status_code=exc.status_code, content=body) + @app.exception_handler(ValidationAppError) + async def validation_app_error_handler(request: Request, exc: ValidationAppError): + body: dict = {"error_code": exc.error_code, "message": exc.message} + if exc.detail is not None: + body["detail"] = exc.detail + if exc.validation_errors: + body["validation_errors"] = [ + {**error, "value": _jsonable(error.get("value"))} + for error in exc.validation_errors + ] + return JSONResponse(status_code=exc.status_code, content=body) + @app.exception_handler(RequestValidationError) async def validation_error_handler(request: Request, exc: RequestValidationError): validation_errors = [] @@ -25,7 +56,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError validation_errors.append({ "field": field or None, "issue": error.get("msg"), - "value": error.get("input"), + "value": _jsonable(error.get("input")), }) return JSONResponse( status_code=422, diff --git a/app/core/lifespan.py b/app/core/lifespan.py index 7f842739..a61cc5b9 100644 --- a/app/core/lifespan.py +++ b/app/core/lifespan.py @@ -6,12 +6,17 @@ from fastapi import FastAPI from app.db.init_db import init_db +from app.services import llm logger = logging.getLogger(__name__) @asynccontextmanager async def lifespan(app: FastAPI): + # A missing key or an unnamed model is a deployment mistake. Better to stop + # here than halfway through someone's incident report. + llm.check_config() + logger.info("Initializing database...") init_db() yield diff --git a/app/db/repositories.py b/app/db/repositories.py index 0935c133..d03df596 100644 --- a/app/db/repositories.py +++ b/app/db/repositories.py @@ -1,24 +1,76 @@ +from datetime import date, datetime, time, timedelta from uuid import UUID +from sqlalchemy import func, nullslast from sqlmodel import Session, select -from app.models import Template, FormSubmission, Job, Input +from app.models import ( + Template, + FormSubmission, + FormTemplate, + Form, + Job, + Input, + Extraction, + Incident, + TemplateUpload, +) +from app.api.schemas.enums import IncidentCategory, ReportStatus -# Templates -def create_template(session: Session, template: Template) -> Template: +# Templates (legacy fill pipeline - read-only lookup, consumed by forms/jobs/tasks) +def get_template(session: Session, template_id: int) -> Template | None: + return session.get(Template, template_id) + +# Form templates (contract Layer 6 registry) +def create_form_template(session: Session, template: FormTemplate) -> FormTemplate: session.add(template) session.commit() session.refresh(template) return template -def get_template(session: Session, template_id: int) -> Template | None: - return session.get(Template, template_id) + +def get_form_template(session: Session, template_id: UUID) -> FormTemplate | None: + return session.get(FormTemplate, template_id) -def list_templates(session: Session) -> list[Template]: - statement = select(Template).order_by(Template.created_at.desc(), Template.id.desc()) +def get_form_template_by_form_type(session: Session, form_type: str) -> FormTemplate | None: + statement = select(FormTemplate).where(FormTemplate.form_type == form_type) + return session.exec(statement).first() + + +def list_form_templates(session: Session) -> list[FormTemplate]: + statement = select(FormTemplate).order_by( + FormTemplate.created_at.desc(), FormTemplate.template_id + ) return list(session.exec(statement)) + +def update_form_template(session: Session, template: FormTemplate) -> FormTemplate: + session.add(template) + session.commit() + session.refresh(template) + return template + + +# Template PDF uploads (field-detection drafts) +def create_template_upload(session: Session, upload: TemplateUpload) -> TemplateUpload: + session.add(upload) + session.commit() + session.refresh(upload) + return upload + + +def get_template_upload(session: Session, upload_id: UUID) -> TemplateUpload | None: + return session.get(TemplateUpload, upload_id) + + +def update_template_upload(session: Session, upload: TemplateUpload) -> TemplateUpload: + session.add(upload) + session.commit() + session.refresh(upload) + return upload + + # Forms def create_form(session: Session, form: FormSubmission) -> FormSubmission: session.add(form) @@ -56,11 +108,6 @@ def update_job(session: Session, job: Job) -> Job: return job -def delete_template(session: Session, template: Template) -> None: - session.delete(template) - session.commit() - - def get_form_submission(session: Session, submission_id: int) -> FormSubmission | None: return session.get(FormSubmission, submission_id) @@ -70,6 +117,32 @@ def delete_form_submission(session: Session, submission: FormSubmission) -> None session.commit() +# Forms (contract Layer 3 — v1 Form model, distinct from the legacy FormSubmission) +def create_generated_form(session: Session, form: Form) -> Form: + session.add(form) + session.commit() + session.refresh(form) + return form + + +def get_form(session: Session, form_id: UUID) -> Form | None: + return session.get(Form, form_id) + + +def list_forms_by_batch(session: Session, batch_id: UUID) -> list[Form]: + statement = select(Form).where(Form.batch_id == batch_id).order_by( + Form.created_at, Form.form_id + ) + return list(session.exec(statement)) + + +def update_form(session: Session, form: Form) -> Form: + session.add(form) + session.commit() + session.refresh(form) + return form + + # Inputs def create_input(session: Session, input_obj: Input) -> Input: session.add(input_obj) @@ -88,3 +161,155 @@ def update_input(session: Session, input_obj: Input) -> Input: session.refresh(input_obj) return input_obj + +# Extractions +def create_extraction(session: Session, extraction: Extraction) -> Extraction: + session.add(extraction) + session.commit() + session.refresh(extraction) + return extraction + + +def get_extraction(session: Session, extract_id: UUID) -> Extraction | None: + return session.get(Extraction, extract_id) + + +def get_extraction_by_input(session: Session, input_id: UUID) -> Extraction | None: + statement = select(Extraction).where(Extraction.input_id == input_id) + return session.exec(statement).first() + + +def update_extraction(session: Session, extraction: Extraction) -> Extraction: + session.add(extraction) + session.commit() + session.refresh(extraction) + return extraction + + +# Incidents +def _day_start(value: date) -> datetime: + """Midnight on the given day, naive. + + incident_datetime is a naive DateTime column and the offset on the value + promoted from the contract is dropped on write, so what is stored is local + wall-clock time. Date bounds are built the same way to match. + """ + return datetime.combine(value, time.min) + + +def create_incident(session: Session, incident: Incident) -> Incident: + session.add(incident) + session.commit() + session.refresh(incident) + return incident + + +def get_incident(session: Session, incident_id: UUID) -> Incident | None: + return session.get(Incident, incident_id) + + +def get_incident_by_extract(session: Session, extract_id: UUID) -> Incident | None: + statement = select(Incident).where(Incident.extract_id == extract_id) + return session.exec(statement).first() + + +def update_incident(session: Session, incident: Incident) -> Incident: + session.add(incident) + session.commit() + session.refresh(incident) + return incident + + +def create_draft_incident(session: Session, extract_id: UUID) -> Incident: + """Create the draft incident row linked to a completed extraction. + + Called when an extraction completes: the new row owns the contract document + and starts in draft status. POST /incidents later finalizes this same row. + """ + incident = Incident(extract_id=extract_id, status=ReportStatus.draft) + return create_incident(session, incident) + + +def get_incident_by_number(session: Session, incident_number: str) -> Incident | None: + """Look up a live incident by its department-assigned number. + + Soft-deleted rows are skipped so a deleted incident does not block its + number from being reused. + """ + statement = select(Incident).where( + Incident.incident_number == incident_number, + Incident.deleted_at.is_(None), + ) + return session.exec(statement).first() + + +def list_incidents( + session: Session, + date_from: date | None = None, + date_to: date | None = None, + incident_category: IncidentCategory | None = None, + status: ReportStatus | None = None, + page: int = 1, + per_page: int = 20, + sort: str = "date_desc", +) -> tuple[list[Incident], int]: + """One page of live incidents plus the total matching the filters. + + Date filters are inclusive and apply to incident_datetime, which is + nullable, so rows without one are excluded whenever a date bound is given + and sort last otherwise. created_at breaks ties, keeping paging stable + across rows that share an incident_datetime. + """ + conditions = [Incident.deleted_at.is_(None)] + if date_from is not None: + conditions.append(Incident.incident_datetime >= _day_start(date_from)) + if date_to is not None: + conditions.append(Incident.incident_datetime < _day_start(date_to) + timedelta(days=1)) + if incident_category is not None: + conditions.append(Incident.incident_category == incident_category) + if status is not None: + conditions.append(Incident.status == status) + + total = session.exec( + select(func.count()).select_from(Incident).where(*conditions) + ).one() + + ascending = sort == "date_asc" + ordering = ( + nullslast(Incident.incident_datetime.asc()) if ascending + else nullslast(Incident.incident_datetime.desc()) + ) + tiebreak = Incident.created_at.asc() if ascending else Incident.created_at.desc() + + statement = ( + select(Incident) + .where(*conditions) + .order_by(ordering, tiebreak, Incident.incident_id) + .offset((page - 1) * per_page) + .limit(per_page) + ) + return list(session.exec(statement)), total + + +def list_forms_by_incident(session: Session, incident_id: UUID) -> list[Form]: + statement = select(Form).where(Form.incident_id == incident_id).order_by( + Form.created_at, Form.form_id + ) + return list(session.exec(statement)) + + +def count_forms_by_incident(session: Session, incident_ids: list[UUID]) -> dict[UUID, int]: + """Form counts for a page of incidents, as one grouped query. + + Counting per row would issue a query per incident on every list request. + Incidents with no forms are absent from the result; callers default to 0. + """ + if not incident_ids: + return {} + statement = ( + select(Form.incident_id, func.count()) + .where(Form.incident_id.in_(incident_ids)) + .group_by(Form.incident_id) + ) + return {incident_id: count for incident_id, count in session.exec(statement)} + diff --git a/app/models/__init__.py b/app/models/__init__.py index bba2eecb..af645112 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,15 +4,19 @@ Extraction, Form, FormSubmission, + FormTemplate, Incident, Input, Job, Report, Template, + TemplateUpload, ) __all__ = [ "Template", + "FormTemplate", + "TemplateUpload", "FormSubmission", "Job", "Input", diff --git a/app/models/models.py b/app/models/models.py index cb9a5ff7..49954db2 100644 --- a/app/models/models.py +++ b/app/models/models.py @@ -2,20 +2,23 @@ from uuid import UUID, uuid4 from datetime import date, datetime, timezone -from sqlalchemy import Column, JSON +from sqlalchemy import Column, Index, JSON, text from sqlmodel import SQLModel, Field from sqlmodel.sql.sqltypes import AutoString from app.api.schemas.enums import ( + DetectionStatus, ExtractionStatus, FormStatus, FormType, + IncidentCategory, InputStatus, InputType, JobStatus, OutputFormat, PeriodType, ReportStatus, + TemplateStatus, ) @@ -91,9 +94,10 @@ class Extraction(SQLModel, table=True): completed_at: datetime | None = None model_used: str | None = None processing_time_seconds: float | None = None - # Full IncidentContract superset blob; stores partial result while processing, - # final canonical JSON when status=completed. - incident_contract: dict | None = Field(default=None, sa_column=Column(JSON)) + # Transient contract blob held only while the job runs. Cleared once the + # extraction completes and the contract is written to the incident row, + # which is the single store. Extractions keep no copy of the final contract. + partial_result: dict | None = Field(default=None, sa_column=Column(JSON)) # Audit trail of manual corrections applied via PATCH /extract/{id}. corrections: list | None = Field(default=None, sa_column=Column(JSON)) error_type: str | None = None @@ -104,6 +108,23 @@ class Extraction(SQLModel, table=True): class Incident(SQLModel, table=True): __tablename__ = "incidents" + __table_args__ = ( + # A department's incident number identifies one live incident. Partial + # so a soft-deleted row does not keep its number reserved forever, and + # so the many rows still awaiting a number do not collide on NULL. + Index( + "ix_incidents_number_live", + "incident_number", + unique=True, + postgresql_where=text("incident_number IS NOT NULL AND deleted_at IS NULL"), + sqlite_where=text("incident_number IS NOT NULL AND deleted_at IS NULL"), + ), + # Covers GET /incidents: every query excludes soft-deleted rows and + # then sorts on incident_datetime. The status and incident_category + # filters are left unindexed on purpose, they are low cardinality and + # a department-sized table does not need them. + Index("ix_incidents_live_datetime", "deleted_at", "incident_datetime"), + ) incident_id: UUID = Field(default_factory=uuid4, primary_key=True) extract_id: UUID = Field(foreign_key="extractions.extract_id") @@ -113,9 +134,35 @@ class Incident(SQLModel, table=True): ) incident_name: str | None = None incident_type: str | None = None - incident_date: date | None = None tags: list | None = Field(default=None, sa_column=Column(JSON)) notes: str | None = None + # The single store of the incident contract. Created as a draft when + # extraction completes; PATCH /extract writes here, form generation reads + # here. Nothing else keeps a copy. + incident_contract: dict | None = Field(default=None, sa_column=Column(JSON)) + # Promoted scalars, recomputed server-side from the contract on every + # document change. All nullable; clients never write them directly. + incident_category: IncidentCategory | None = Field( + default=None, sa_column=Column(AutoString, nullable=True) + ) + incident_datetime: datetime | None = None + city: str | None = None + state: str | None = None + country: str | None = None + civilian_injuries: int | None = None + civilian_fatalities: int | None = None + responder_injuries: int | None = None + responder_fatalities: int | None = None + people_rescued: int | None = None + people_evacuated: int | None = None + structures_destroyed: int | None = None + area_burned_ha: float | None = None + total_loss_amount: float | None = None + total_loss_currency: str | None = None + call_to_arrival_seconds: int | None = None + turnout_seconds_first_unit: int | None = None + travel_seconds_first_unit: int | None = None + on_scene_duration_seconds: int | None = None created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) deleted_at: datetime | None = None @@ -129,8 +176,11 @@ class Form(SQLModel, table=True): status: FormStatus = Field( default=FormStatus.queued, sa_column=Column(AutoString, nullable=False) ) - extract_id: UUID = Field(foreign_key="extractions.extract_id") - incident_id: UUID | None = Field(default=None, foreign_key="incidents.incident_id") + template_id: UUID = Field(foreign_key="form_templates.template_id") + incident_id: UUID = Field(foreign_key="incidents.incident_id") + # Grouping key for a batch generate request. No Batch table — batch status + # is derived on the fly from the Form rows sharing this id. + batch_id: UUID | None = None # Plain UUID, no FK constraint — pending contract Job model resolution (#544 decision A). job_id: UUID | None = None completed_at: datetime | None = None @@ -143,6 +193,66 @@ class Form(SQLModel, table=True): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) +class TemplateUpload(SQLModel, table=True): + """A blank PDF uploaded for template authoring, plus its detection draft. + + The PDF is stored and its page geometry read synchronously, so a row exists + with `page_count`/`pages` filled before detection starts. `status` tracks + detection alone: a failed detection still leaves a usable upload, the user + just draws every box by hand. Rows are drafts, not templates. Registering a + template copies the edited fields into `form_templates` and keeps only the + `pdf_template_ref` pointing back here. + """ + + __tablename__ = "template_uploads" + + upload_id: UUID = Field(default_factory=uuid4, primary_key=True) + status: DetectionStatus = Field( + default=DetectionStatus.processing, sa_column=Column(AutoString, nullable=False) + ) + # Path on disk, and the DATA_DIR-relative reference handed to clients. + pdf_path: str + pdf_template_ref: str + original_filename: str | None = None + page_count: int = Field(default=0) + # List of {page, width, height} in PDF points. + pages: list = Field(default_factory=list, sa_column=Column(JSON, nullable=False)) + # List of DraftField objects (see app/api/schemas/templates.py). + detected_fields: list | None = Field(default=None, sa_column=Column(JSON)) + detection_error: str | None = None + job_id: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class FormTemplate(SQLModel, table=True): + """Contract Layer 6 form template registry (path/templates.yaml). + + Distinct from the legacy prototype `Template` (int PK + uploaded PDF): this + is the standards registry keyed by `form_type`, holding incident-schema field + definitions plus their visual `layout`. `field_count` and `last_updated` are + derived in the response schemas (len(fields) / updated_at.date()), not stored. + """ + + __tablename__ = "form_templates" + + template_id: UUID = Field(default_factory=uuid4, primary_key=True) + form_type: str = Field(sa_column=Column(AutoString, nullable=False, unique=True, index=True)) + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + # List of TemplateField objects (see app/api/schemas/templates.py). + fields: list = Field(sa_column=Column(JSON, nullable=False)) + source_standard: str | None = None + pdf_template_ref: str | None = None + version: str = Field(default="1.0") + status: TemplateStatus = Field( + default=TemplateStatus.active, sa_column=Column(AutoString, nullable=False) + ) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + class Report(SQLModel, table=True): __tablename__ = "reports" diff --git a/app/services/extraction/__init__.py b/app/services/extraction/__init__.py new file mode 100644 index 00000000..c1c5103d --- /dev/null +++ b/app/services/extraction/__init__.py @@ -0,0 +1,11 @@ +"""The extraction layer. + +Split by job: `service` queues a run, `registry` and `router` decide what to ask +the model, `prompts` and `client` do the asking, `runner` validates and retries, +`defaults` covers everything computable without a model, and `worker` stitches +it all into a contract and a draft incident. + +Nothing is re-exported here on purpose. The celery task imports the worker and +the service imports the task, so a package-level import of either would close +that loop at import time. Import the submodule you need. +""" diff --git a/app/services/extraction/defaults.py b/app/services/extraction/defaults.py new file mode 100644 index 00000000..fdd5d034 --- /dev/null +++ b/app/services/extraction/defaults.py @@ -0,0 +1,181 @@ +"""The parts of extraction that need no model at all. + +Deployment context (timezone, country, currency) and anything arithmetic. The +model is told the context so it can resolve "yesterday evening" itself, and +these functions then fill what it left blank and compute what is derivable: +default country and currency, a timezone offset on naive timestamps, and the +per-unit turnout and travel seconds. Plain code beats a prompt every time. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Any +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from app.api.schemas.extraction import ExtractionDefaults, ExtractionHints +from app.core.config import DEFAULT_COUNTRY, DEFAULT_CURRENCY, DEFAULT_TIMEZONE +from app.core.logging import get_logger + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class ExtractionContext: + """Deployment context for one extraction run.""" + + country: str + timezone: str + currency: str + now: datetime + + @property + def zone(self) -> ZoneInfo: + try: + return ZoneInfo(self.timezone) + except (ZoneInfoNotFoundError, ValueError): + logger.warning("unknown timezone %s, falling back to UTC", self.timezone) + return ZoneInfo("UTC") + + +def resolve_context(defaults: ExtractionDefaults | dict | None) -> ExtractionContext: + """Merge the request's defaults over the server's configured ones.""" + if isinstance(defaults, ExtractionDefaults): + defaults = defaults.model_dump(exclude_none=True) + values = defaults or {} + timezone_name = values.get("timezone") or DEFAULT_TIMEZONE + context = ExtractionContext( + country=values.get("country") or DEFAULT_COUNTRY, + timezone=timezone_name, + currency=values.get("currency") or DEFAULT_CURRENCY, + now=datetime.now(), + ) + return ExtractionContext( + country=context.country, + timezone=context.timezone, + currency=context.currency, + now=datetime.now(context.zone), + ) + + +def context_lines(context: ExtractionContext, hints: ExtractionHints | dict | None = None) -> list[str]: + """The context block every chunk prompt carries in its dynamic tail.""" + lines = [ + f"Right now it is {context.now.isoformat(timespec='seconds')} " + f"({context.now.strftime('%A')}), timezone {context.timezone}. " + "Resolve relative times like 'yesterday evening' against it.", + f"Country when the narrative names none: {context.country}.", + f"Currency for money amounts when the narrative names none: {context.currency}.", + ] + if isinstance(hints, ExtractionHints): + hints = hints.model_dump(exclude_none=True) + for key, value in (hints or {}).items(): + if value: + lines.append(f"Hint from the responder, {key.replace('_', ' ')}: {value}.") + return lines + + +def _localize_datetimes(node: Any, context: ExtractionContext) -> Any: + """Attach the deployment offset to any timestamp the model left naive.""" + if isinstance(node, dict): + return {key: _localize_datetimes(value, context) for key, value in node.items()} + if isinstance(node, list): + return [_localize_datetimes(item, context) for item in node] + if isinstance(node, str) and len(node) >= 16 and "T" in node: + try: + parsed = datetime.fromisoformat(node.replace("Z", "+00:00")) + except ValueError: + return node + if parsed.tzinfo is None: + return parsed.replace(tzinfo=context.zone).isoformat() + return node + + +def _apply_currency(node: Any, currency: str) -> Any: + """Stamp the default currency on Money objects that came back without one.""" + if isinstance(node, dict): + filled = {key: _apply_currency(value, currency) for key, value in node.items()} + if isinstance(filled.get("amount"), (int, float)) and not filled.get("currency"): + filled["currency"] = currency + return filled + if isinstance(node, list): + return [_apply_currency(item, currency) for item in node] + return node + + +def _prune_empty(node: Any) -> Any: + """Drop empty strings and empty containers from a contract document. + + Models answer a field they know nothing about with "" or [] rather than + leaving it out. In the contract an absent field means unknown, while an + empty string is a claim that the value is blank, so these are dropped. + Zero is kept: a count of zero is a real answer. + """ + if isinstance(node, dict): + cleaned = {} + for key, value in node.items(): + pruned = _prune_empty(value) + if pruned is None or pruned == "" or pruned == [] or pruned == {}: + continue + cleaned[key] = pruned + return cleaned + if isinstance(node, list): + items = [_prune_empty(item) for item in node] + return [item for item in items if item not in (None, "", [], {})] + return node + + +def _seconds_between(start: Any, end: Any) -> int | None: + """Whole seconds between two RFC 3339 strings, or None if that makes no sense.""" + try: + first = datetime.fromisoformat(str(start).replace("Z", "+00:00")) + second = datetime.fromisoformat(str(end).replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + if first.tzinfo is None or second.tzinfo is None: + return None + delta = (second - first).total_seconds() + return int(delta) if delta >= 0 else None + + +def _derive_unit_timings(contract: dict) -> None: + """Compute each unit's turnout and travel seconds from its own timestamps. + + Arithmetic is not the model's job. When both timestamps are there the + computed value wins, because models do state a duration that contradicts + the times they just gave. The model's number is only kept when the + timestamps are missing and there is nothing to compute from. + """ + units = contract.get("units") + if not isinstance(units, list): + return + for unit in units: + if not isinstance(unit, dict): + continue + turnout = _seconds_between(unit.get("dispatched_datetime"), unit.get("enroute_datetime")) + if turnout is not None: + unit["turnout_seconds"] = turnout + travel = _seconds_between(unit.get("enroute_datetime"), unit.get("arrived_datetime")) + if travel is not None: + unit["travel_seconds"] = travel + + +def apply_context(contract: dict, context: ExtractionContext) -> dict: + """Run every deterministic post-step over a stitched contract.""" + filled = _prune_empty(contract) + filled = _localize_datetimes(filled, context) + filled = _apply_currency(filled, context.currency) + + # Country is deployment truth, not a guess, so it is stamped even when the + # narrative said nothing about where it happened. + location = filled.setdefault("location", {}) + if isinstance(location, dict) and not location.get("country"): + location["country"] = context.country + + incident = filled.get("incident") + if isinstance(incident, dict) and not incident.get("timezone"): + incident["timezone"] = context.timezone + + _derive_unit_timings(filled) + return filled diff --git a/app/services/extraction/prompts.py b/app/services/extraction/prompts.py new file mode 100644 index 00000000..36ee502d --- /dev/null +++ b/app/services/extraction/prompts.py @@ -0,0 +1,154 @@ +"""Chunk prompts. + +Every chunk prompt is a static prefix plus a dynamic suffix. The prefix (the +instructions and the chunk's field skeleton) never changes between incidents, +so Ollama reuses its KV cache for it; only the tail, which carries the +deployment context and the narrative, is new each time. Interpolating the +narrative anywhere but the end would throw that away. + +The field skeleton is rendered from the generated Pydantic model rather than +hand-written, so it cannot drift from the contract. Enum members are spelled +out because small models invent enum values otherwise. +""" + +from __future__ import annotations + +import json +from datetime import date, datetime, time +from enum import Enum +from functools import lru_cache +from typing import Any, get_args, get_origin +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, NaiveDatetime, RootModel + +from app.services.extraction.registry import ChunkSpec + +# How deep the skeleton goes before it stops describing nested shape. Three +# levels covers every chunk in the contract without bloating the prompt. +MAX_DEPTH = 3 + +# Longest enum spelled out in full. Past this the prompt lists the first few +# and tells the model to use exactly one of the contract's values. +MAX_ENUM_MEMBERS = 25 + +INSTRUCTIONS = """You extract fire and emergency incident data from a responder's narrative. + +Rules: +- Return one JSON object and nothing else. No prose, no markdown, no code fence. +- Use only facts the narrative states or clearly implies. Never invent a value. +- Leave out any field the narrative does not support. An absent field is correct; a guess is not. +- Use exactly the enum values listed. If none fits, leave the field out. +- Date-times are RFC 3339 with an offset, for example 2026-04-18T21:14:00-07:00. +- Physical quantities are SI: metres, square metres, hectares, litres, kilometres, Celsius. +""" + + +def _enum_hint(enum_cls: type[Enum]) -> str: + members = [str(member.value) for member in enum_cls] + if len(members) > MAX_ENUM_MEMBERS: + shown = " | ".join(members[:MAX_ENUM_MEMBERS]) + return f"" + return f"" + + +# Pydantic's date-time markers are plain classes at runtime, not datetime +# subclasses, so they need naming before the subclass checks below. +_ALIAS_HINTS: dict[Any, str] = { + AwareDatetime: "", + NaiveDatetime: "", +} + +_SCALAR_HINTS: dict[type, str] = { + str: "", + bool: "", + int: "", + float: "", + datetime: "", + date: "", + time: "", + UUID: "", +} + + +def _skeleton(annotation: Any, depth: int) -> Any: + """Render one field's expected shape as a placeholder value.""" + hint = _ALIAS_HINTS.get(annotation) + if hint is not None: + return hint + + origin = get_origin(annotation) + + if origin is list: + args = [a for a in get_args(annotation) if a is not type(None)] + item = _skeleton(args[0], depth) if args else "" + return [item] + + if origin is not None: + # Optional / Union / Annotated: describe the first real member. + args = [a for a in get_args(annotation) if a is not type(None)] + return _skeleton(args[0], depth) if args else "" + + if isinstance(annotation, type) and issubclass(annotation, Enum): + return _enum_hint(annotation) + + if isinstance(annotation, type) and issubclass(annotation, RootModel): + # A root model is a wrapper around one value; describe the value. + return _skeleton(annotation.model_fields["root"].annotation, depth) + + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + if depth >= MAX_DEPTH: + return "" + return model_skeleton(annotation, depth + 1) + + if isinstance(annotation, type): + for scalar, hint in _SCALAR_HINTS.items(): + if issubclass(annotation, scalar): + return hint + + return "" + + +def model_skeleton(model: type[BaseModel], depth: int = 0) -> dict[str, Any]: + """A JSON-shaped description of every field on a generated contract model.""" + skeleton: dict[str, Any] = {} + for name, info in model.model_fields.items(): + skeleton[name] = _skeleton(info.annotation, depth) + return skeleton + + +@lru_cache(maxsize=None) +def static_prefix(chunk_name: str, model: type[BaseModel], is_list: bool, description: str) -> str: + """The cacheable head of a chunk prompt. Identical for every incident.""" + shape = model_skeleton(model) + body = {chunk_name: [shape] if is_list else shape} + lines = [INSTRUCTIONS] + if description: + lines.append(f"Section: {chunk_name}. {description}") + else: + lines.append(f"Section: {chunk_name}.") + lines.append( + f'Return a JSON object with the single key "{chunk_name}", shaped like this. ' + "The angle-bracket text describes the expected value, it is not a value:" + ) + lines.append(json.dumps(body, indent=2)) + return "\n\n".join(lines) + + +def build_prompt( + spec: ChunkSpec, + text: str, + context_lines: list[str], + retry_note: str | None = None, +) -> str: + """The full prompt for one chunk: cached prefix, then this incident's tail.""" + prefix = static_prefix(spec.name, spec.model, spec.is_list, spec.description) + tail = ["Context for resolving anything the narrative leaves implicit:"] + tail.extend(f"- {line}" for line in context_lines) + if retry_note: + tail.append( + "Your previous answer was rejected. Fix it and return the corrected " + f"JSON object only. Reason: {retry_note}" + ) + tail.append(f"NARRATIVE:\n{text}") + return prefix + "\n\n" + "\n\n".join(tail) diff --git a/app/services/extraction/registry.py b/app/services/extraction/registry.py new file mode 100644 index 00000000..e06b83d3 --- /dev/null +++ b/app/services/extraction/registry.py @@ -0,0 +1,148 @@ +"""The chunk registry. + +The incident contract is the source of truth for how each of its top-level +chunks is extracted. Every chunk carries `x-extraction` (core, gated, +background or manual), gated chunks carry `x-triggers`, and any chunk can carry +`x-extraction-priority`. This module reads those at import time and pairs each +chunk with the generated Pydantic model that validates it, so the rest of the +worker never hardcodes a chunk name or a tier. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import Enum +from functools import lru_cache +from typing import Any, get_args, get_origin + +import yaml +from pydantic import BaseModel + +from app.api.schemas.incident_contract import IncidentContract +from app.core.config import INCIDENT_CONTRACT_PATH +from app.core.logging import get_logger + +logger = get_logger(__name__) + + +class Tier(str, Enum): + """What the worker does with a chunk. + + core always extracted, no gating + gated extracted only when the narrative shows evidence for it + background extracted after everything else, never blocks a form + manual never sent to the model (record ids, signatures, reflections) + """ + + core = "core" + gated = "gated" + background = "background" + manual = "manual" + + +@dataclass(frozen=True) +class ChunkSpec: + """One top-level contract chunk and everything needed to extract it.""" + + name: str + tier: Tier + model: type[BaseModel] | None + is_list: bool + priority: int = 100 + triggers: tuple[str, ...] = () + description: str = "" + trigger_pattern: re.Pattern[str] | None = field(default=None, compare=False) + + def matches(self, text: str) -> bool: + """True when the text carries evidence this chunk applies.""" + if self.trigger_pattern is None: + return False + return self.trigger_pattern.search(text) is not None + + +def _build_trigger_pattern(triggers: tuple[str, ...]) -> re.Pattern[str] | None: + """One case-insensitive word-boundary pattern for a chunk's trigger list.""" + if not triggers: + return None + alternatives = "|".join(re.escape(t) for t in sorted(triggers, key=len, reverse=True)) + return re.compile(rf"\b(?:{alternatives})\b", re.IGNORECASE) + + +def _unwrap(annotation: Any) -> tuple[type[BaseModel] | None, bool]: + """Reduce a generated field annotation to (model, is_list). + + Generated fields are Optional and sometimes list-valued, so this peels off + Optional and list wrappers to find the BaseModel underneath. Returns + (None, False) for scalar chunks like schema_version. + """ + is_list = False + seen: list[Any] = [annotation] + while seen: + current = seen.pop() + if isinstance(current, type) and issubclass(current, BaseModel): + return current, is_list + origin = get_origin(current) + if origin is list: + is_list = True + args = [arg for arg in get_args(current) if arg is not type(None)] + seen.extend(args) + return None, is_list + + +@lru_cache(maxsize=1) +def _contract_properties() -> dict[str, dict]: + """The IncidentContract property map, read from the contract file once.""" + doc = yaml.safe_load(INCIDENT_CONTRACT_PATH.read_text()) + return doc["IncidentContract"]["properties"] + + +@lru_cache(maxsize=1) +def chunk_registry() -> dict[str, ChunkSpec]: + """Every top-level chunk, keyed by name, in tier and priority order.""" + specs: list[ChunkSpec] = [] + for name, spec in _contract_properties().items(): + tier_value = spec.get("x-extraction") + if tier_value is None: + # A chunk with no x-extraction is new and unrouted. Treat it as + # manual so the worker never silently prompts for something the + # contract has not classified. + tier_value = Tier.manual.value + info = IncidentContract.model_fields.get(name) + if info is None: + # The contract grew a chunk the committed models do not have yet. + # Skip it rather than crash, and say so: the fix is to regenerate. + logger.warning( + "contract chunk %s has no generated model, skipping it. " + "Run `make generate-contract-models`.", + name, + ) + continue + triggers = tuple(spec.get("x-triggers") or ()) + model, is_list = _unwrap(info.annotation) + specs.append( + ChunkSpec( + name=name, + tier=Tier(tier_value), + model=model, + is_list=is_list, + priority=int(spec.get("x-extraction-priority", 100)), + triggers=triggers, + description=(spec.get("description") or "").strip(), + trigger_pattern=_build_trigger_pattern(triggers), + ) + ) + + tier_order = {Tier.core: 0, Tier.gated: 1, Tier.background: 2, Tier.manual: 3} + specs.sort(key=lambda s: (tier_order[s.tier], s.priority, s.name)) + return {spec.name: spec for spec in specs} + + +def extractable_chunks() -> list[ChunkSpec]: + """Chunks the model can be asked about: everything but manual, and only + those the generated models can validate.""" + return [ + spec + for spec in chunk_registry().values() + if spec.tier is not Tier.manual and spec.model is not None + ] diff --git a/app/services/extraction/router.py b/app/services/extraction/router.py new file mode 100644 index 00000000..de562d97 --- /dev/null +++ b/app/services/extraction/router.py @@ -0,0 +1,39 @@ +"""Chunk routing. + +Deciding what not to ask the model is most of the speed. Core chunks apply to +every incident and always run. Gated chunks only run when their trigger words +appear in the narrative, so a structure fire never pays for the wildland +prompt. Background chunks run last. Nothing here calls the model. +""" + +from __future__ import annotations + +from app.core.logging import get_logger +from app.services.extraction.registry import ChunkSpec, Tier, extractable_chunks + +logger = get_logger(__name__) + + +def select_chunks(text: str) -> list[ChunkSpec]: + """The chunks worth extracting from this text, in the order to run them. + + Order is tier first (core, then gated, then background) and priority + within a tier, which the registry already applies. + """ + selected: list[ChunkSpec] = [] + skipped: list[str] = [] + + for spec in extractable_chunks(): + if spec.tier is Tier.gated and not spec.matches(text): + skipped.append(spec.name) + continue + selected.append(spec) + + logger.info( + "chunk router selected %d chunks (%s), skipped %d gated (%s)", + len(selected), + ", ".join(spec.name for spec in selected), + len(skipped), + ", ".join(skipped), + ) + return selected diff --git a/app/services/extraction/runner.py b/app/services/extraction/runner.py new file mode 100644 index 00000000..78cfe7a7 --- /dev/null +++ b/app/services/extraction/runner.py @@ -0,0 +1,254 @@ +"""Running the selected chunks. + +Each chunk is one focused prompt, validated against its generated model. A +chunk that comes back malformed gets one retry with the reason named. If it +misses again, the salvage pass keeps whatever fields do validate and throws +away only the offending ones, because one invented enum in a sub-field nobody +mentioned should not cost a whole section. Nothing is ever guessed in the other +direction: fields only come out. Chunks run in waves by tier so the fields a +form needs land first, and inside a wave they run in parallel up to the +provider's concurrency limit. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from functools import lru_cache +from itertools import groupby +from typing import Any, Callable + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from app.core.config import EXTRACTION_CHUNK_RETRIES, EXTRACTION_MAX_PARALLEL +from app.core.logging import get_logger +from app.services import llm +from app.services.extraction.prompts import build_prompt +from app.services.extraction.registry import ChunkSpec + +logger = get_logger(__name__) + + +@dataclass +class ChunkResult: + """What one chunk produced: a validated value, or the reason it has none.""" + + name: str + value: Any = None + error: str | None = None + attempts: int = 0 + # Paths thrown away by the salvage pass, so the review screen can show them + # as gaps rather than pretending the whole section was extracted. + dropped: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return self.error is None + + @property + def has_value(self) -> bool: + return self.value not in (None, {}, []) + + +@lru_cache(maxsize=None) +def _list_adapter(model: type[BaseModel]) -> TypeAdapter: + """Validator for a list-valued chunk, built once per chunk model.""" + return TypeAdapter(list[model]) + + +def _validate(spec: ChunkSpec, payload: dict[str, Any]) -> Any: + """Validate the model's answer against the chunk's contract model. + + The prompt asks for {"chunk_name": {...}}, but models sometimes return the + inner object on its own, so both shapes are accepted. Returns plain JSON + data with unset fields dropped, ready for the contract document. + """ + raw = payload.get(spec.name, payload) + + if spec.is_list: + if isinstance(raw, dict): + raw = [raw] + if not isinstance(raw, list): + raise ValueError(f"expected a list for {spec.name}, got {type(raw).__name__}") + # Validated as a whole list, not item by item, so an error's location + # carries the index the salvage pass needs to find the offending entry. + validated = _list_adapter(spec.model).validate_python(raw) + items = [item.model_dump(mode="json", exclude_none=True) for item in validated] + return [item for item in items if item] + + if not isinstance(raw, dict): + raise ValueError(f"expected an object for {spec.name}, got {type(raw).__name__}") + return spec.model.model_validate(raw).model_dump(mode="json", exclude_none=True) + + +def _drop_path(payload: Any, path: tuple) -> str | None: + """Delete one validation-error path from the raw answer. + + Returns the path it removed, or None when the path does not resolve (a + union tag in the location, an index that moved). Nothing is guessed: only + an exact hit is deleted. + """ + node = payload + for step in path[:-1]: + if isinstance(node, dict) and step in node: + node = node[step] + elif isinstance(node, list) and isinstance(step, int) and step < len(node): + node = node[step] + else: + return None + + last = path[-1] + if isinstance(node, dict) and last in node: + del node[last] + elif isinstance(node, list) and isinstance(last, int) and last < len(node): + del node[last] + else: + return None + return ".".join(str(step) for step in path) + + +def _salvage(spec: ChunkSpec, raw: Any, exc: ValidationError) -> tuple[Any, list[str]]: + """Keep the fields that validate by dropping the ones that do not. + + A small model will invent an enum value in some sub-field nobody mentioned, + and without this one bad field costs the whole section. Dropping the exact + offending paths and validating again keeps the good data. Nothing is + invented here; fields only ever come out. + """ + dropped: list[str] = [] + error = exc + + def deepest_first(path: tuple) -> tuple: + """Sort key: remove the longest paths and the highest list indices + first, so deleting one entry cannot shift another out from under us.""" + return tuple((1, step) if isinstance(step, int) else (0, str(step)) for step in path) + + # Each pass can uncover errors the previous one masked, so try a few times. + for _ in range(3): + removed_any = False + paths = sorted({tuple(err["loc"]) for err in error.errors()}, key=deepest_first, reverse=True) + for path in paths: + removed = _drop_path(raw, path) + if removed: + dropped.append(removed) + removed_any = True + if not removed_any: + break + try: + return _validate(spec, {spec.name: raw}), dropped + except ValidationError as next_error: + error = next_error + except ValueError: + break + + raise error + + +def _failure_reason(exc: Exception) -> str: + """A short, model-readable description of why an answer was rejected.""" + if isinstance(exc, ValidationError): + parts = [] + for err in exc.errors()[:5]: + path = ".".join(str(loc) for loc in err["loc"]) or "value" + parts.append(f"{path}: {err['msg']}") + return "; ".join(parts) + return str(exc) + + +def extract_chunk( + spec: ChunkSpec, + text: str, + context_lines: list[str], + model: str | None = None, + gate: llm.RateLimitGate | None = None, +) -> ChunkResult: + """Extract one chunk: retry a rejected answer once, then salvage what validates.""" + result = ChunkResult(name=spec.name) + retry_note: str | None = None + attempts = 1 + EXTRACTION_CHUNK_RETRIES + + for attempt in range(attempts): + result.attempts = attempt + 1 + last_try = attempt == attempts - 1 + prompt = build_prompt(spec, text, context_lines, retry_note) + try: + payload = llm.generate_json(prompt, model=model, gate=gate) + result.value = _validate(spec, payload) + result.error = None + return result + except (llm.LLMUnavailableError, llm.LLMRateLimitError, llm.LLMAuthError): + # Nothing chunk-specific about these. Every other chunk would hit + # the same wall, so the run gives up rather than working through it. + raise + except llm.LLMTimeoutError as exc: + result.error = str(exc) + logger.warning("chunk %s timed out, not retrying: %s", spec.name, exc) + break + except ValidationError as exc: + retry_note = _failure_reason(exc) + result.error = retry_note + logger.warning( + "chunk %s attempt %d rejected: %s", spec.name, result.attempts, retry_note + ) + if not last_try: + continue + try: + raw = payload.get(spec.name, payload) + result.value, result.dropped = _salvage(spec, raw, exc) + result.error = None + logger.info( + "chunk %s salvaged, dropped %d field(s): %s", + spec.name, + len(result.dropped), + ", ".join(result.dropped), + ) + return result + except (ValidationError, ValueError): + logger.warning("chunk %s could not be salvaged", spec.name) + except (llm.LLMResponseError, ValueError) as exc: + retry_note = _failure_reason(exc) + result.error = retry_note + logger.warning( + "chunk %s attempt %d rejected: %s", spec.name, result.attempts, retry_note + ) + + logger.error( + "chunk %s failed after %d attempt(s), left for manual entry: %s", + spec.name, + result.attempts, + result.error, + ) + result.value = None + return result + + +def run_chunks( + specs: list[ChunkSpec], + text: str, + context_lines: list[str], + model: str | None = None, + on_wave: Callable[[list[ChunkResult]], None] | None = None, +) -> list[ChunkResult]: + """Run every selected chunk, one parallel wave per tier. + + `on_wave` is called with the results of each wave as it lands, which is how + the worker publishes partial results while the long tail is still running. + """ + results: list[ChunkResult] = [] + workers = max(1, EXTRACTION_MAX_PARALLEL) + # Shared by every chunk in this run. The first one to exhaust its rate limit + # retries trips it, and the rest fail immediately instead of each waiting. + gate = llm.RateLimitGate() + + for tier, group in groupby(specs, key=lambda s: s.tier): + wave = list(group) + logger.info("extracting %s wave: %s", tier.value, ", ".join(s.name for s in wave)) + with ThreadPoolExecutor(max_workers=workers) as pool: + wave_results = list( + pool.map(lambda spec: extract_chunk(spec, text, context_lines, model, gate), wave) + ) + results.extend(wave_results) + if on_wave is not None: + on_wave(wave_results) + + return results diff --git a/app/services/extraction/service.py b/app/services/extraction/service.py new file mode 100644 index 00000000..e59dd906 --- /dev/null +++ b/app/services/extraction/service.py @@ -0,0 +1,67 @@ +"""Extraction service. + +Owns the write path that turns a ready input into a queued extraction: it +creates the extraction row, creates the async job, and dispatches the worker. +The route stays a thin HTTP handler and calls straight into here. + +The request's deployment defaults and hints are not stored on the extraction +row, they only shape this one run, so they travel to the worker as task +arguments. +""" + +from datetime import datetime, timezone + +from sqlmodel import Session + +from app.api.schemas.enums import ExtractionStatus +from app.api.schemas.extraction import ExtractionDefaults, ExtractionHints +from app.db.repositories import create_extraction, create_job, update_job +from app.models import Extraction, Job +from app.tasks.extract import extract_task + + +def _as_dict(value: ExtractionDefaults | ExtractionHints | None) -> dict: + """Task arguments have to be plain JSON, so models go over as dicts.""" + return value.model_dump(exclude_none=True) if value is not None else {} + + +class ExtractionService: + def start_extraction( + self, + session: Session, + input_id, + model_override: str | None = None, + defaults: ExtractionDefaults | None = None, + hints: ExtractionHints | None = None, + ) -> tuple[Extraction, Job]: + """Create the extraction row and job, then dispatch the worker. + + The extraction starts in ``processing`` so a poll right after the 202 + sees the in-flight shape. Mirrors the transcription flow: the job row is + created first with a known job_id, dispatched, then its celery_task_id + is backfilled once the broker returns a task id. + """ + now = datetime.now(timezone.utc) + extraction = Extraction( + input_id=input_id, + status=ExtractionStatus.processing, + started_at=now, + model_used=model_override, + created_at=now, + updated_at=now, + ) + extraction = create_extraction(session, extraction) + + job = Job(celery_task_id="", job_type="extraction", status="queued", model=model_override) + job = create_job(session, job) + + result = extract_task.delay( + str(extraction.extract_id), + job.job_id, + _as_dict(defaults), + _as_dict(hints), + ) + job.celery_task_id = result.id + job = update_job(session, job) + + return extraction, job diff --git a/app/services/extraction/worker.py b/app/services/extraction/worker.py new file mode 100644 index 00000000..814c7519 --- /dev/null +++ b/app/services/extraction/worker.py @@ -0,0 +1,240 @@ +"""The extraction run. + +Takes a queued extraction and turns it into a validated contract and a draft +incident. Order of business: read the narrative, pick the chunks worth asking +about, run them in waves, apply the deterministic post-steps, stitch one +document, then write the incident and close out the extraction and its job. + +A chunk that fails does not sink the run. The narrative is the only thing every +chunk shares, so a chunk missing means those fields stay empty and the review +screen shows the gap. The run only fails when the provider itself is the +problem, unreachable or out of quota, or when nothing at all could be extracted. +A run stopped by a rate limit keeps everything the earlier waves published. +""" + +from __future__ import annotations + +import time +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from pydantic import ValidationError +from sqlmodel import Session + +from app.api.schemas.enums import ExtractionStatus +from app.api.schemas.incident_contract import IncidentContract +from app.core.logging import get_logger +from app.db.repositories import ( + create_draft_incident, + get_extraction, + get_input, + get_job_by_uuid, + update_extraction, + update_incident, + update_job, +) +from app.services import llm +from app.services.extraction.defaults import apply_context, context_lines, resolve_context +from app.services.extraction.router import select_chunks +from app.services.extraction.runner import ChunkResult, run_chunks +from app.services.incidents import PROMOTED_COLUMNS, promote + +logger = get_logger(__name__) + +SCHEMA_VERSION = "1.1.0" +SCHEMA_NAME = "fireform_incident_contract" + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _stitch(results: list[ChunkResult]) -> dict[str, Any]: + """Merge the chunk results into one contract document. + + Chunks own disjoint sub-objects, so merging is a plain assignment per chunk. + Chunks with nothing in them are left out: an absent field means unknown. + """ + contract: dict[str, Any] = {"schema_version": SCHEMA_VERSION, "schema_name": SCHEMA_NAME} + for result in results: + if result.has_value: + contract[result.name] = result.value + return contract + + +def _completeness(results: list[ChunkResult]) -> dict[str, Any]: + """Extraction-quality summary for extraction_metadata.""" + filled = [r for r in results if r.has_value] + failed = [r.name for r in results if not r.ok] + empty = [r.name for r in results if r.ok and not r.has_value] + # A salvaged chunk kept its good fields; the ones thrown away are gaps too. + dropped = [f"{r.name}.{path}" for r in results for path in r.dropped] + percent = round(100 * len(filled) / len(results)) if results else 0 + return { + "overall_percent": percent, + "missing_fields": sorted(failed + empty + dropped), + } + + +def _metadata(extraction, input_record, model_used: str, results: list[ChunkResult]) -> dict[str, Any]: + return { + "extract_id": str(extraction.extract_id), + "input_id": str(extraction.input_id), + "input_type": input_record.input_type.value + if hasattr(input_record.input_type, "value") + else str(input_record.input_type), + "extracted_at": _now().isoformat(), + "llm_model": model_used, + "completeness": _completeness(results), + } + + +def _fail(session: Session, extraction, job, error_type: str, detail: str) -> None: + now = _now() + extraction.status = ExtractionStatus.failed + extraction.error_type = error_type + extraction.error_detail = detail + extraction.updated_at = now + update_extraction(session, extraction) + if job: + job.status = "failed" + job.error = {"error_code": error_type, "message": detail} + job.updated_at = now + update_job(session, job) + logger.error("extraction %s failed (%s): %s", extraction.extract_id, error_type, detail) + + +def _write_incident(session: Session, extraction, contract: dict[str, Any]): + """Create the draft incident that owns the contract from here on.""" + incident = create_draft_incident(session, extraction.extract_id) + incident.incident_contract = contract + promoted = promote(contract) + for column in PROMOTED_COLUMNS: + setattr(incident, column, promoted[column]) + incident.updated_at = _now() + return update_incident(session, incident) + + +def run_extraction( + session: Session, + extract_id: UUID, + job_id: str, + defaults: dict | None = None, + hints: dict | None = None, +) -> dict[str, Any]: + """Run one queued extraction to completion. Returns a small result summary.""" + started = time.monotonic() + extraction = get_extraction(session, extract_id) + if extraction is None: + logger.error("extraction %s no longer exists, nothing to run", extract_id) + return {"extract_id": str(extract_id), "status": "missing"} + + job = get_job_by_uuid(session, job_id) + input_record = get_input(session, extraction.input_id) + + now = _now() + extraction.status = ExtractionStatus.processing + extraction.started_at = extraction.started_at or now + extraction.updated_at = now + update_extraction(session, extraction) + if job: + job.status = "processing" + job.updated_at = now + update_job(session, job) + + text = (input_record.transcript if input_record else "") or "" + if not text.strip(): + _fail(session, extraction, job, "EMPTY_INPUT", "The input has no transcript to extract from.") + return {"extract_id": str(extract_id), "status": "failed"} + + model_used = extraction.model_used or llm.get_settings().model + context = resolve_context(defaults) + prompt_context = context_lines(context, hints) + specs = select_chunks(text) + + collected: list[ChunkResult] = [] + + def publish(wave: list[ChunkResult]) -> None: + """Show what has landed so far while the remaining waves run.""" + collected.extend(wave) + extraction.partial_result = _stitch(collected) + extraction.updated_at = _now() + update_extraction(session, extraction) + if job: + job.progress_percent = round(100 * len(collected) / len(specs)) if specs else 100 + job.updated_at = _now() + update_job(session, job) + + try: + results = run_chunks(specs, text, prompt_context, extraction.model_used, on_wave=publish) + except llm.LLMRateLimitError as exc: + # Whatever the waves already published stays on the record, so a run cut + # short by a quota is still worth reviewing rather than starting over. + _fail(session, extraction, job, "LLM_RATE_LIMITED", str(exc)) + return { + "extract_id": str(extract_id), + "status": "failed", + "retry_after_seconds": exc.retry_after_seconds, + } + except (llm.LLMUnavailableError, llm.LLMAuthError) as exc: + _fail(session, extraction, job, "LLM_UNAVAILABLE", str(exc)) + raise + + if results and not any(result.ok for result in results): + _fail( + session, + extraction, + job, + "EXTRACTION_FAILED", + "Every chunk was rejected by validation. Nothing could be extracted.", + ) + return {"extract_id": str(extract_id), "status": "failed"} + + contract = apply_context(_stitch(results), context) + contract["extraction_metadata"] = _metadata(extraction, input_record, model_used, results) + + try: + IncidentContract.model_validate(contract) + except ValidationError as exc: + _fail(session, extraction, job, "EXTRACTION_FAILED", f"stitched contract is invalid: {exc}") + raise + + incident = _write_incident(session, extraction, contract) + + now = _now() + extraction.status = ExtractionStatus.completed + extraction.completed_at = now + extraction.processing_time_seconds = round(time.monotonic() - started, 2) + extraction.model_used = model_used + # The incident row owns the document now; the working copy goes away. + extraction.partial_result = None + extraction.error_type = None + extraction.error_detail = None + extraction.updated_at = now + update_extraction(session, extraction) + + if job: + job.status = "completed" + job.progress_percent = 100 + job.result_url = f"/api/v1/extract/{extract_id}" + job.updated_at = now + update_job(session, job) + + failed = [result.name for result in results if not result.ok] + logger.info( + "extraction %s completed in %.2fs: %d/%d chunks filled, %d failed, incident %s", + extract_id, + extraction.processing_time_seconds, + sum(1 for r in results if r.has_value), + len(results), + len(failed), + incident.incident_id, + ) + return { + "extract_id": str(extract_id), + "job_id": job_id, + "incident_id": str(incident.incident_id), + "status": "completed", + "failed_chunks": failed, + } diff --git a/app/services/extraction_readiness.py b/app/services/extraction_readiness.py new file mode 100644 index 00000000..9f7d4bb0 --- /dev/null +++ b/app/services/extraction_readiness.py @@ -0,0 +1,209 @@ +"""Read-side check of an extraction against the templates that could be filled. + +Once extraction finishes, the responder still has to pick a form. That choice +only makes sense if the screen can say which forms are fillable right now and +what is missing on the ones that are not. Both endpoints answer that from +stored data: one template at a time (validate) or every active template at once +(readiness). No LLM, so it stays cheap to refetch after every correction. + +Plain dicts and repository calls only, no FastAPI. The route stays thin. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +from app.api.schemas.enums import FieldSource, TemplateStatus +from app.api.schemas.extraction import ( + FieldGap, + ReadinessMatrix, + TemplateReadiness, + ValidationResult, +) +from app.api.schemas.templates import TemplateField +from app.models import Extraction, FormTemplate, Incident +from app.services.extraction_review import value_at + +# Open mapping on the contract. Manual and open template fields live here, and +# the extractor writes them under a single flat key, "{form_type}.{field_name}", +# rather than as nested objects. +CUSTOM_FIELDS = "custom_fields" + + +def is_filled(value: Any) -> bool: + """True when the value would actually print something on the form. + + A null, a blank string and an empty container are all a blank box to the + responder, so none of them close a gap. + """ + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, (list, dict, tuple, set)): + return bool(value) + return True + + +def custom_key(field: TemplateField, form_type: str) -> str: + """The custom_fields key a manual or open field is stored under.""" + return f"{form_type}.{field.field_name}" + + +def resolve(contract: dict, field: TemplateField, form_type: str) -> Any: + """The value a template field would be filled with, or None. + + Where to look depends on the field's source. A schema field reads the + contract path it declares. A static field carries its own text. Manual and + open fields sit under custom_fields, which is a flat mapping: the dotted + key is one key, not a path, so it is read directly instead of walked. + """ + if field.source is FieldSource.static: + return field.static_text + if field.source is FieldSource.schema: + return value_at(contract, field.incident_mapping or "") + + custom = contract.get(CUSTOM_FIELDS) + if not isinstance(custom, dict): + return None + return custom.get(custom_key(field, form_type)) + + +def mapping_of(field: TemplateField, form_type: str) -> str | None: + """What to show the UI as the origin of a missing value. + + A schema field points at the contract path to correct. A manual or open + field points at the custom_fields key the typed value goes into. + """ + if field.source is FieldSource.schema: + return field.incident_mapping + if field.source is FieldSource.static: + return None + return f"{CUSTOM_FIELDS}.{custom_key(field, form_type)}" + + +def _gap(field: TemplateField, form_type: str) -> FieldGap: + return FieldGap( + field_name=field.field_name, + source=field.source, + incident_mapping=mapping_of(field, form_type), + description=field.description, + ) + + +def _parse_fields(template: FormTemplate) -> list[TemplateField]: + """The template's stored field definitions as models.""" + return [TemplateField.model_validate(entry) for entry in template.fields or []] + + +def warnings_for(gaps: list[FieldGap]) -> list[str]: + """One readable line per recommended field that has no value. + + Built from the gaps themselves rather than a rule table per form type, so + a newly registered template gets useful warnings without anyone adding to + a list first. + """ + lines = [] + for gap in gaps: + where = gap.incident_mapping or gap.field_name + line = f"{where} has no value. '{gap.field_name}' is optional on this form" + if gap.description: + line = f"{line}: {gap.description}" + lines.append(line) + return lines + + +@dataclass(frozen=True) +class Gaps: + """What one template is missing, and how much of it is filled.""" + + missing_required: list[FieldGap] + missing_recommended: list[FieldGap] + coverage_percent: float + + @property + def ready(self) -> bool: + return not self.missing_required + + +def gaps_for(contract: dict, template: FormTemplate) -> Gaps: + """Compare a contract document against one template's field list.""" + fields = _parse_fields(template) + missing_required: list[FieldGap] = [] + missing_recommended: list[FieldGap] = [] + filled = 0 + + for field in fields: + if is_filled(resolve(contract, field, template.form_type)): + filled += 1 + continue + gap = _gap(field, template.form_type) + if field.required: + missing_required.append(gap) + else: + missing_recommended.append(gap) + + coverage = round(filled / len(fields) * 100, 1) if fields else 0.0 + return Gaps(missing_required, missing_recommended, coverage) + + +def validate_template( + extraction: Extraction, incident: Incident, template: FormTemplate +) -> ValidationResult: + """The single-template answer: can this form be generated right now.""" + gaps = gaps_for(incident.incident_contract or {}, template) + return ValidationResult( + valid=gaps.ready, + template_id=template.template_id, + extract_id=extraction.extract_id, + form_type=template.form_type, + missing_required=gaps.missing_required, + missing_recommended=gaps.missing_recommended, + warnings=warnings_for(gaps.missing_recommended), + field_coverage_percent=gaps.coverage_percent, + ) + + +def readiness_matrix( + extraction: Extraction, incident: Incident, templates: list[FormTemplate] +) -> ReadinessMatrix: + """The same check across every active template, for the selection screen. + + Drafts and legacy templates are left out: nothing on the selection screen + should offer a form that is not in service. + """ + contract = incident.incident_contract or {} + rows = [] + for template in templates: + if template.status != TemplateStatus.active: + continue + gaps = gaps_for(contract, template) + rows.append( + TemplateReadiness( + template_id=template.template_id, + form_type=template.form_type, + display_name=template.display_name, + ready=gaps.ready, + missing_required=gaps.missing_required, + missing_recommended=gaps.missing_recommended, + field_coverage_percent=gaps.coverage_percent, + ) + ) + return ReadinessMatrix( + extract_id=extraction.extract_id, + templates=rows, + computed_at=datetime.now(timezone.utc), + ) + + +__all__ = [ + "Gaps", + "gaps_for", + "is_filled", + "readiness_matrix", + "resolve", + "validate_template", + "warnings_for", +] diff --git a/app/services/extraction_review.py b/app/services/extraction_review.py new file mode 100644 index 00000000..dec7000b --- /dev/null +++ b/app/services/extraction_review.py @@ -0,0 +1,297 @@ +"""Review-screen write path for an extraction. + +A responder fixes a wrong value, adds a missing one or drops one the model +invented. The correction arrives as a JSON Merge Patch (RFC 7396) shaped like +the incident contract, and this module applies it to the contract document on +the linked incident row, which is the single store of incident data. + +Everything here works on plain dicts and repository calls, no FastAPI. The +route stays a thin handler. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from types import UnionType +from typing import Any, Union, get_args, get_origin + +from pydantic import BaseModel, ValidationError +from sqlmodel import Session + +from app.api.schemas.enums import ExtractionStatus, ReportStatus +from app.api.schemas.incident_contract import IncidentContract +from app.core.errors.base import AppError, ValidationAppError +from app.db.repositories import update_extraction, update_incident +from app.models import Extraction, Incident +from app.services.incidents import promote + +# Deleting a key is the whole point of RFC 7396, so a null in the patch is a +# delete, never a value. Kept as a name so the intent reads at the call sites. +DELETE = None + + +# --------------------------------------------------------------------------- +# RFC 7396 merge patch +# --------------------------------------------------------------------------- + +def merge_patch(target: Any, patch: Any) -> Any: + """Apply a JSON Merge Patch to a document (RFC 7396). + + A non-object patch replaces the target outright. Inside an object, a null + removes the key, a nested object merges recursively, anything else + replaces. + """ + if not isinstance(patch, dict): + return patch + + result = dict(target) if isinstance(target, dict) else {} + for key, value in patch.items(): + if value is DELETE: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def patch_paths(patch: dict, prefix: str = "") -> list[tuple[str, Any]]: + """Every leaf of the patch as a (dotted path, value) pair. + + Nested objects are walked so a patch that only touches + ``losses.property_loss.amount`` records that one path and not its parents. + A list, a scalar and an empty object are leaves: the merge replaces them + whole, so that is the level a correction is recorded at. + """ + leaves: list[tuple[str, Any]] = [] + for key, value in patch.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict) and value: + leaves.extend(patch_paths(value, path)) + else: + leaves.append((path, value)) + return leaves + + +def value_at(document: Any, path: str) -> Any: + """Value at a dotted path, or None when any step is missing.""" + current = document + for key in path.split("."): + if not isinstance(current, dict): + return None + current = current.get(key) + return current + + +# --------------------------------------------------------------------------- +# Field path checking +# --------------------------------------------------------------------------- + +def _model_of(annotation: Any) -> type[BaseModel] | None: + """The contract submodel behind an annotation, or None. + + Generated fields are Optional and sometimes list-valued. A list is not + walked into (a patch replaces the whole list), so only a plain object + annotation yields a model. + """ + origin = get_origin(annotation) + if origin in (Union, UnionType): + for arg in get_args(annotation): + if arg is type(None): + continue + return _model_of(arg) + return None + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation + return None + + +def _is_open_mapping(annotation: Any) -> bool: + """True for a free-form object like custom_fields, whose keys are open.""" + origin = get_origin(annotation) + if origin in (Union, UnionType): + return any( + _is_open_mapping(arg) for arg in get_args(annotation) if arg is not type(None) + ) + return origin is dict + + +def unknown_paths(patch: dict, model: type[BaseModel], prefix: str = "") -> list[str]: + """Dotted paths in the patch that the contract has no field for. + + The generated models ignore unknown keys, so without this check a typo + would be accepted and then silently dropped on the way to the database. + Open mappings such as custom_fields accept any key by design. + """ + unknown: list[str] = [] + for key, value in patch.items(): + path = f"{prefix}.{key}" if prefix else key + field = model.model_fields.get(key) + if field is None: + unknown.append(path) + continue + if not isinstance(value, dict) or not value: + continue + if _is_open_mapping(field.annotation): + continue + submodel = _model_of(field.annotation) + if submodel is not None: + unknown.extend(unknown_paths(value, submodel, path)) + return unknown + + +def _validation_errors(exc: ValidationError) -> list[dict]: + """Pydantic errors as the contract's validation_errors entries.""" + errors = [] + for error in exc.errors(): + path = ".".join(str(part) for part in error.get("loc", ())) + errors.append({ + "field": path or None, + "issue": error.get("msg"), + "value": error.get("input"), + }) + return errors + + +# --------------------------------------------------------------------------- +# Corrections +# --------------------------------------------------------------------------- + +def correction_entries( + patch: dict, + before: dict, + after: dict, + corrected_at: datetime, + corrected_by: str | None = None, +) -> list[dict]: + """Audit entries for the paths the patch actually changed. + + A path whose value is the same before and after is a no-op and is not + recorded, so the trail stays a history of real edits. + """ + entries = [] + for path, _ in patch_paths(patch): + original = value_at(before, path) + corrected = value_at(after, path) + if original == corrected: + continue + entries.append({ + "field_path": path, + "original_value": original, + "corrected_value": corrected, + "corrected_at": corrected_at.isoformat(), + "corrected_by": corrected_by, + }) + return entries + + +class ExtractionReviewService: + """Applies manual corrections to a completed extraction.""" + + def apply_patch( + self, + session: Session, + extraction: Extraction, + incident: Incident, + patch: dict, + corrected_by: str | None = None, + ) -> tuple[Extraction, Incident]: + """Merge the patch into the contract, then rewrite everything derived. + + The document, the promoted analytics columns and the corrections trail + all move together in one call so they can never disagree. + """ + self._reject_locked(incident) + self._reject_unknown_paths(patch) + + before = incident.incident_contract or {} + merged = merge_patch(before, patch) + after = self._validated(merged) + + now = datetime.now(timezone.utc) + entries = correction_entries(patch, before, after, now, corrected_by) + + incident.incident_contract = after + for column, value in promote(after).items(): + setattr(incident, column, value) + incident.updated_at = now + incident = update_incident(session, incident) + + if entries: + # Reassign rather than append: SQLModel tracks JSON columns by + # identity, so mutating the list in place would not be persisted. + extraction.corrections = (extraction.corrections or []) + entries + extraction.updated_at = now + extraction = update_extraction(session, extraction) + + return extraction, incident + + def _reject_locked(self, incident: Incident) -> None: + if incident.status == ReportStatus.submitted: + raise AppError( + "Cannot modify extraction incident report has been submitted", + status_code=409, + error_code="EXTRACT_LOCKED", + detail={ + "report_status": incident.status, + "submitted_at": incident.updated_at.isoformat() + if incident.updated_at + else None, + }, + ) + + def _reject_unknown_paths(self, patch: dict) -> None: + unknown = unknown_paths(patch, IncidentContract) + if unknown: + raise ValidationAppError( + "Invalid field path or value in patch", + validation_errors=[ + { + "field": path, + "issue": "Unknown field path in the incident contract", + "value": value_at(patch, path), + } + for path in unknown + ], + ) + + def _validated(self, merged: dict) -> dict: + """Validate the merged document and return it normalized. + + Dumping the validated model back out is what strips the keys a delete + removed and settles types (dates, enums) into their JSON form, so the + stored document always matches the contract. + """ + try: + model = IncidentContract.model_validate(merged) + except ValidationError as exc: + raise ValidationAppError( + "Invalid field path or value in patch", + validation_errors=_validation_errors(exc), + ) from exc + return model.model_dump(mode="json", exclude_none=True) + + +def load_for_review(extraction: Extraction, incident: Incident | None) -> Incident: + """The incident row a correction writes to, or the reason there is none. + + An extraction that has not completed has no contract document yet, so + there is nothing to correct. + """ + if extraction.status != ExtractionStatus.completed or incident is None: + raise AppError( + f"Extraction is in '{extraction.status}' state. Wait until status is 'completed'.", + status_code=409, + error_code="EXTRACT_NOT_COMPLETED", + detail={"current_status": extraction.status}, + ) + return incident + + +__all__ = [ + "ExtractionReviewService", + "correction_entries", + "load_for_review", + "merge_patch", + "patch_paths", + "unknown_paths", + "value_at", +] diff --git a/app/services/field_catalog.py b/app/services/field_catalog.py new file mode 100644 index 00000000..00ba17ce --- /dev/null +++ b/app/services/field_catalog.py @@ -0,0 +1,309 @@ +"""The incident-contract field catalog and its matcher. + +Everything here is built by flattening `contracts/schemas/incident-contract.yaml` +at first use: dotted paths, types, sections, descriptions, enum values, the +`x-pii` flag and the `x-aliases` list. The contract is the only place any of +that is declared, so a schema change moves search and mapping suggestions with +it and nothing is restated in code. + +Two callers share this one index: GET /api/v1/schema/fields (the mapping picker +in the template editor) and the suggester that runs after commonforms field +detection. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from functools import lru_cache +from typing import Any + +import yaml + +from app.core.config import INCIDENT_CONTRACT_PATH +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_ROOT = "IncidentContract" +_PUNCTUATION = re.compile(r"[^a-z0-9]+") + +# Form labels are written for people, not matchers. These are the shortenings +# that show up on nearly every printed incident form. +_ABBREVIATIONS = { + "no": "number", + "num": "number", + "nbr": "number", + "dt": "date", + "addr": "address", + "tel": "phone", + "ph": "phone", + "dob": "date of birth", + "amt": "amount", + "qty": "quantity", + "desc": "description", + "dept": "department", + "apt": "apartment", + "st": "street", + "yr": "year", + "veh": "vehicle", + "inj": "injury", +} + + +@dataclass(frozen=True) +class CatalogEntry: + """One leaf field of the contract, ready to search.""" + + path: str + label: str + field_type: str + section: str + description: str | None = None + enum_values: tuple[str, ...] | None = None + pii: bool = False + aliases: tuple[str, ...] = () + tokens: frozenset[str] = field(default_factory=frozenset, compare=False) + + +# --------------------------------------------------------------------------- +# Text normalization +# --------------------------------------------------------------------------- +def normalize(text: str) -> str: + """Lowercase, drop punctuation, collapse whitespace.""" + return _PUNCTUATION.sub(" ", text.lower()).strip() + + +def normalize_label(text: str) -> str: + """Normalize a label read off a PDF, expanding the usual form shorthand. + + Detected labels are messy ("Incident No.:", "Dt of Loss"), so the words are + expanded before they ever reach the matcher. + """ + words = normalize(text).split() + return " ".join(_ABBREVIATIONS.get(word, word) for word in words) + + +def _humanize(name: str) -> str: + words = name.replace("_", " ").strip() + return words[:1].upper() + words[1:] if words else name + + +def _tokens(*values: str) -> frozenset[str]: + out: set[str] = set() + for value in values: + out.update(normalize(value).split()) + return frozenset(out) + + +# --------------------------------------------------------------------------- +# Catalog construction +# --------------------------------------------------------------------------- +@lru_cache(maxsize=1) +def _contract_doc() -> dict[str, Any]: + return yaml.safe_load(INCIDENT_CONTRACT_PATH.read_text()) + + +@lru_cache(maxsize=1) +def _enums_doc() -> dict[str, Any]: + """The shared enum file the contract points at for closed value lists.""" + enums_path = INCIDENT_CONTRACT_PATH.parent / "enums.yaml" + try: + return yaml.safe_load(enums_path.read_text()) or {} + except OSError: + logger.warning("enum file %s is missing, enum values will be empty", enums_path) + return {} + + +def _resolve(spec: dict[str, Any]) -> dict[str, Any]: + """Follow a $ref one hop, inside the contract or into enums.yaml. + + Anything the ref does not carry (a description written at the reference + site, for example) stays, so both halves survive. + """ + ref = spec.get("$ref") + if not ref: + return spec + + file_part, _, name = ref.partition("#/") + if "enums.yaml" in file_part: + target = _enums_doc().get(name) + elif file_part in ("", "#"): + target = _contract_doc().get(name) + else: + target = None + + if not isinstance(target, dict): + logger.warning("unresolved $ref %s in the incident contract", ref) + return {k: v for k, v in spec.items() if k != "$ref"} + + merged = dict(target) + for key, value in spec.items(): + if key != "$ref": + merged.setdefault(key, value) + return merged + + +def _field_type(spec: dict[str, Any]) -> str: + declared = spec.get("type") + if isinstance(declared, str): + return declared + if spec.get("enum"): + return "string" + if spec.get("properties"): + return "object" + return "string" + + +def _walk( + spec: dict[str, Any], + path: str, + section: str, + name: str, + seen: frozenset[str], + out: list[CatalogEntry], +) -> None: + spec = _resolve(spec) + + properties = spec.get("properties") + if properties: + # A recursive shape would otherwise walk forever. Stop the second time + # the same object type appears on one branch. + marker = spec.get("title") or path + if marker in seen: + return + seen = seen | {marker} + for child_name, child_spec in properties.items(): + if not isinstance(child_spec, dict): + continue + child_path = f"{path}.{child_name}" if path else child_name + _walk(child_spec, child_path, section, child_name, seen, out) + return + + if spec.get("type") == "array": + items = spec.get("items") + if isinstance(items, dict): + resolved = _resolve(items) + if resolved.get("properties"): + _walk(items, f"{path}[]", section, name, seen, out) + return + + enum_values = spec.get("enum") + out.append( + CatalogEntry( + path=path, + label=_humanize(name), + field_type=_field_type(spec), + section=section, + description=(spec.get("description") or "").strip() or None, + enum_values=tuple(str(v) for v in enum_values) if enum_values else None, + pii=bool(spec.get("x-pii")), + aliases=tuple(spec.get("x-aliases") or ()), + tokens=_tokens(name, *(spec.get("x-aliases") or ())), + ) + ) + + +@lru_cache(maxsize=1) +def catalog() -> tuple[CatalogEntry, ...]: + """Every leaf field in the contract, in contract order.""" + properties = _contract_doc()[_ROOT].get("properties", {}) + entries: list[CatalogEntry] = [] + for section, spec in properties.items(): + if isinstance(spec, dict): + _walk(spec, section, section, section, frozenset(), entries) + return tuple(entries) + + +@lru_cache(maxsize=1) +def schema_version() -> str | None: + """The contract version the catalog was built from.""" + properties = _contract_doc()[_ROOT].get("properties", {}) + version = properties.get("schema_version") or {} + example = version.get("example") + return str(example) if example else None + + +# --------------------------------------------------------------------------- +# Matching +# --------------------------------------------------------------------------- +# Scores are banded rather than blended, so the ranking the contract describes +# holds no matter how the fuzzy ratio lands: an exact name beats an exact alias, +# both beat a prefix, and a description-only hit always comes last. +_EXACT_NAME = 1.0 +_EXACT_ALIAS = 0.95 +_NAME_PREFIX = 0.85 +_ALIAS_PREFIX = 0.8 +_FUZZY_CEILING = 0.75 +_FUZZY_FLOOR = 0.6 +_DESCRIPTION_HIT = 0.35 + + +def _leaf_name(path: str) -> str: + return path.rsplit(".", 1)[-1].removesuffix("[]") + + +def score_entry(entry: CatalogEntry, query: str) -> float: + """How well one catalog entry answers a query, 0 when it does not. + + `query` is expected already normalized, since `search` normalizes once and + then scores the whole catalog with it. + """ + if not query: + return 0.0 + + name = normalize(_leaf_name(entry.path)) + aliases = [normalize(a) for a in entry.aliases] + + if query == name: + return _EXACT_NAME + if query in aliases: + return _EXACT_ALIAS + if name.startswith(query): + return _NAME_PREFIX + if any(alias.startswith(query) for alias in aliases): + return _ALIAS_PREFIX + + query_tokens = set(query.split()) + if query_tokens and query_tokens <= entry.tokens: + return _FUZZY_CEILING + + best = max( + (SequenceMatcher(None, query, candidate).ratio() for candidate in [name, *aliases]), + default=0.0, + ) + if best >= _FUZZY_FLOOR: + return round(best * _FUZZY_CEILING, 4) + + if entry.description and query_tokens: + description_tokens = set(normalize(entry.description).split()) + if query_tokens <= description_tokens: + return _DESCRIPTION_HIT + + return 0.0 + + +def search( + query: str | None = None, + section: str | None = None, + limit: int = 20, +) -> list[tuple[CatalogEntry, float | None]]: + """Rank the catalog against `query`, or list it when no query is given. + + `limit` caps search results only. A bare listing returns the whole catalog, + because the editor caches it once per session and filters it locally, and a + truncated catalog would silently hide fields from the mapping picker. + + Ties break toward the shorter path, so the plainest field wins. + """ + entries = [e for e in catalog() if section is None or e.section == section] + + if not query or not query.strip(): + return [(entry, None) for entry in entries] + + normalized = normalize(query) + scored = [(entry, score_entry(entry, normalized)) for entry in entries] + hits = [(entry, score) for entry, score in scored if score > 0] + hits.sort(key=lambda pair: (-pair[1], len(pair[0].path), pair[0].path)) + return [(entry, score) for entry, score in hits[:limit]] diff --git a/app/services/file_manipulator.py b/app/services/file_manipulator.py index 87142e13..37653c50 100644 --- a/app/services/file_manipulator.py +++ b/app/services/file_manipulator.py @@ -1,7 +1,8 @@ import os -from app.services.filler import Filler -from app.services.llm import LLM + from app.core.logging import get_logger +from app.services.filler import Filler +from app.services.form_fill import extract_field_values logger = get_logger(__name__) @@ -9,7 +10,6 @@ class FileManipulator: def __init__(self): self.filler = Filler() - self.llm = LLM() def prepare_fillable(self, pdf_path: str): """ @@ -17,7 +17,6 @@ def prepare_fillable(self, pdf_path: str): fillable PDF. Returns the new path (alongside the original). """ # Disable CUDA to force CPU usage, preventing errors on Mac Silicon / Docker - import os os.environ["CUDA_VISIBLE_DEVICES"] = "" # Monkey patch rfdetr to force CPU usage on Mac Silicon / Docker @@ -50,10 +49,8 @@ def fill_form(self, user_input: str, fields: list, pdf_form_path: str, model: st logger.info("[3] Starting extraction and PDF filling process...") try: - self.llm._target_fields = fields - self.llm._transcript_text = user_input - self.llm._model = model - output_name = self.filler.fill_form(pdf_form=pdf_form_path, llm=self.llm) + values = extract_field_values(user_input, fields, model=model) + output_name = self.filler.fill_form(pdf_form_path, values) logger.info("Process complete. Output saved to: %s", output_name) diff --git a/app/services/filler.py b/app/services/filler.py index aec97560..8c81798c 100644 --- a/app/services/filler.py +++ b/app/services/filler.py @@ -1,34 +1,22 @@ -from pdfrw import PdfReader, PdfWriter -from app.services.llm import LLM from datetime import datetime +from pdfrw import PdfReader, PdfWriter + class Filler: - def __init__(self): - pass + def fill_form(self, pdf_form: str, values: dict[str, str | None]) -> str: + """Write values into a PDF's form widgets and return the new file's path. - def fill_form(self, pdf_form: str, llm: LLM): - """ - Fill a PDF form with values from user_input using LLM. - Fields are filled in the visual order (top-to-bottom, left-to-right). + Widgets are filled in visual order, top to bottom then left to right, + and the values are consumed in the order they were collected. """ output_pdf = ( - pdf_form[:-4] - + "_" - + datetime.now().strftime("%Y%m%d_%H%M%S") - + "_filled.pdf" + pdf_form[:-4] + "_" + datetime.now().strftime("%Y%m%d_%H%M%S") + "_filled.pdf" ) - # Generate dictionary of answers from your original function - t2j = llm.main_loop() - textbox_answers = t2j.get_data() # This is a dictionary - - answers_list = list(textbox_answers.values()) - - # Read PDF + answers = list(values.values()) pdf = PdfReader(pdf_form) - # Loop through pages i = 0 for page in pdf.pages: if page.Annots: @@ -38,15 +26,11 @@ def fill_form(self, pdf_form: str, llm: LLM): for annot in sorted_annots: if annot.Subtype == "/Widget" and annot.T: - if i < len(answers_list): - annot.V = f"{answers_list[i]}" - annot.AP = None - i += 1 - else: - # Stop if we run out of answers + if i >= len(answers): break + annot.V = f"{answers[i]}" + annot.AP = None + i += 1 PdfWriter().write(output_pdf, pdf) - - # Your main.py expects this function to return the path return output_pdf diff --git a/app/services/form_fill.py b/app/services/form_fill.py new file mode 100644 index 00000000..861361d5 --- /dev/null +++ b/app/services/form_fill.py @@ -0,0 +1,52 @@ +"""Reading a PDF template's field values out of a narrative. + +One prompt per field, which is slow but simple, and it is what the template +fill path has always done. The prompts are answered by the shared LLM module, +so this works against whichever provider the deployment is configured for. + +Not to be confused with the extraction layer, which asks about a whole section +of the incident contract at a time. This one only knows about the fields a +particular PDF template happens to have. +""" + +from __future__ import annotations + +import os + +from app.core.logging import get_logger +from app.services import llm + +logger = get_logger(__name__) + +_NOT_FOUND = "-1" + + +def _prompt_template() -> str: + path = os.path.join(os.path.dirname(__file__), "prompt.txt") + with open(path, "r") as handle: + return handle.read() + + +def _clean(answer: str) -> str | None: + """The model's answer as a value, or None when it found nothing.""" + value = answer.strip().replace('"', "") + return None if value == _NOT_FOUND else value + + +def extract_field_values( + text: str, fields: dict[str, str], model: str | None = None +) -> dict[str, str | None]: + """Ask for each template field in turn and collect the answers.""" + template = _prompt_template() + values: dict[str, str | None] = {} + + for index, (field, field_type) in enumerate(fields.items(), 1): + prompt = template.format( + field=field, + type=field_type if isinstance(field_type, str) else "string", + text=text, + ) + values[field] = _clean(llm.generate(prompt, model=model)) + logger.info("[%d/%d] extracted %r", index, len(fields), field) + + return values diff --git a/app/services/form_fill_worker.py b/app/services/form_fill_worker.py new file mode 100644 index 00000000..59bb248f --- /dev/null +++ b/app/services/form_fill_worker.py @@ -0,0 +1,232 @@ +"""Batch form-fill worker. + +Fills every queued Form in a batch: resolve each template field's value from +the incident contract (extraction_readiness.resolve), draw the placed ones +onto a ReportLab overlay at their TemplateFieldLayout coordinates, merge that +overlay onto the template PDF with pypdf, and save the result. Each form is +independently try/excepted — one bad form marks that form failed and moves +on, it never sinks the batch or the job. + +Mirrors app/services/extraction/worker.py's shape (a plain function taking a +session, called by the thin Celery task in app/tasks/generate_forms.py). + +Not to be confused with the legacy app/services/filler.py, which fills +AcroForm widgets by name in visual order — this draws free text at explicit +layout coordinates onto a flat PDF and merges the overlay on top. Layout +coordinates are bottom-left-origin PDF points, the same space ReportLab's +canvas uses natively, so no flip is applied. +""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime, timezone +from io import BytesIO +from pathlib import Path +from uuid import UUID + +from pypdf import PdfReader, PdfWriter +from reportlab.lib.colors import HexColor +from reportlab.pdfbase.pdfmetrics import stringWidth +from reportlab.pdfgen import canvas +from sqlmodel import Session + +from app.api.schemas.enums import FormStatus, TextAlign +from app.api.schemas.templates import TemplateField +from app.core.config import DATA_DIR, FORMS_OUTPUT_DIR +from app.core.logging import get_logger +from app.db.repositories import ( + get_form_template, + get_incident, + get_job_by_uuid, + list_forms_by_batch, + update_form, + update_job, +) +from app.models import Form, FormTemplate +from app.services.extraction_readiness import gaps_for, resolve +from app.services.form_templates import resolve_template_pdf + +logger = get_logger(__name__) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _format_value(value) -> str | None: + """A drawable string for a resolved field value, or None to skip drawing.""" + if value is None: + return None + if isinstance(value, bool): + return "Yes" if value else "No" + if isinstance(value, (dict, list)): + # A composite contract value has no single sane rendering on a form box. + return None + text = str(value).strip() + return text or None + + +def _fit_text(text: str, font: str, size: float, max_width: float) -> str: + """Truncate with an ellipsis if the text is wider than its box.""" + if stringWidth(text, font, size) <= max_width: + return text + while text and stringWidth(f"{text}…", font, size) > max_width: + text = text[:-1] + return f"{text}…" if text else "" + + +def _draw_field(c: canvas.Canvas, field: TemplateField, value: object) -> None: + layout = field.layout + text = _format_value(value) + if not text: + return + + c.setFont(layout.font, layout.font_size) + c.setFillColor(HexColor(layout.color)) + text = _fit_text(text, layout.font, layout.font_size, layout.width) + + if layout.align == TextAlign.center: + c.drawCentredString(layout.x + layout.width / 2, layout.y, text) + elif layout.align == TextAlign.right: + c.drawRightString(layout.x + layout.width, layout.y, text) + else: + c.drawString(layout.x, layout.y, text) + + +def _build_overlay( + template_pdf_path: Path, fields: list[TemplateField], contract: dict, form_type: str +) -> PdfReader: + """One ReportLab page per template page, sized to match, with each placed + field's resolved value drawn at its layout coordinates.""" + template_reader = PdfReader(str(template_pdf_path)) + page_count = len(template_reader.pages) + + by_page: dict[int, list[TemplateField]] = defaultdict(list) + for field in fields: + if field.layout is not None and 0 <= field.layout.page < page_count: + by_page[field.layout.page].append(field) + + buf = BytesIO() + c = canvas.Canvas(buf) + for page_index in range(page_count): + box = template_reader.pages[page_index].mediabox + c.setPageSize((float(box.width), float(box.height))) + for field in by_page.get(page_index, []): + _draw_field(c, field, resolve(contract, field, form_type)) + # showPage() advances to a fresh page — only between pages, never + # after the last one, or save() would emit a trailing blank page. + if page_index < page_count - 1: + c.showPage() + c.save() + buf.seek(0) + return PdfReader(buf) + + +def _merge_overlay(template_pdf_path: Path, overlay: PdfReader) -> PdfWriter: + """Merge the overlay onto a writer already cloned from the template. + + Merging happens on pages already attached to the writer (via + clone_from), not on bare PdfReader pages added afterward — pypdf + deprecated merge-then-add in favor of this order. + """ + writer = PdfWriter(clone_from=str(template_pdf_path)) + for index, page in enumerate(writer.pages): + if index < len(overlay.pages): + page.merge_page(overlay.pages[index], over=True) + return writer + + +def _summary(contract: dict, template: FormTemplate) -> dict: + gaps = gaps_for(contract, template) + total = len(template.fields or []) + blank = len(gaps.missing_required) + len(gaps.missing_recommended) + return { + "total_form_fields": total, + "fields_filled": total - blank, + "fields_blank": blank, + "coverage_percent": gaps.coverage_percent, + } + + +def fill_one(session: Session, form: Form) -> None: + """Fill a single queued form. Raises on any failure — the batch loop + below decides how to record that against the Form row.""" + form.status = FormStatus.generating + form.updated_at = _now() + update_form(session, form) + + incident = get_incident(session, form.incident_id) + if incident is None: + raise ValueError(f"incident {form.incident_id} no longer exists") + + template = get_form_template(session, form.template_id) + if template is None: + raise ValueError(f"template {form.template_id} no longer exists") + + contract = incident.incident_contract or {} + fields = [TemplateField.model_validate(entry) for entry in template.fields or []] + agency_fields = {f.field_name: resolve(contract, f, template.form_type) for f in fields} + + template_pdf_path = resolve_template_pdf(session, form.template_id) + overlay = _build_overlay(template_pdf_path, fields, contract, template.form_type) + writer = _merge_overlay(template_pdf_path, overlay) + + FORMS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + output_path = FORMS_OUTPUT_DIR / f"{form.form_id}.pdf" + with output_path.open("wb") as handle: + writer.write(handle) + + form.pdf_path = str(output_path.relative_to(DATA_DIR)) + form.pdf_ready = True + form.json_data = agency_fields + form.json_ready = True + form.field_mapping_summary = _summary(contract, template) + form.status = FormStatus.completed + form.completed_at = _now() + form.updated_at = _now() + update_form(session, form) + + +def run_batch_fill(session: Session, batch_id: UUID, job_id: str) -> dict: + """Fill every queued form in a batch. Each form is independently + try/excepted: one failure marks that form failed and moves on — it never + sinks the batch or the job, per design.""" + forms = list_forms_by_batch(session, batch_id) + job = get_job_by_uuid(session, job_id) + + if job: + job.status = "processing" + job.updated_at = _now() + update_job(session, job) + + completed = 0 + failed = 0 + for index, form in enumerate(forms, start=1): + try: + fill_one(session, form) + completed += 1 + except Exception: + logger.exception("form %s (batch %s) failed to generate", form.form_id, batch_id) + form.status = FormStatus.failed + form.updated_at = _now() + update_form(session, form) + failed += 1 + + if job: + job.progress_percent = round(100 * index / len(forms)) if forms else 100 + job.updated_at = _now() + update_job(session, job) + + if job: + job.status = "completed" + job.progress_percent = 100 + job.result_url = f"/api/v1/forms/batch/{batch_id}" + job.updated_at = _now() + update_job(session, job) + + logger.info( + "batch %s finished: %d/%d forms completed, %d failed", + batch_id, completed, len(forms), failed, + ) + return {"batch_id": str(batch_id), "completed": completed, "failed": failed} diff --git a/app/services/form_generation.py b/app/services/form_generation.py new file mode 100644 index 00000000..d26cbfd0 --- /dev/null +++ b/app/services/form_generation.py @@ -0,0 +1,268 @@ +"""Form generation service. + +Owns the write path for POST /forms/generate: validates the incident and +every requested template, splits templates into ready/not-ready via the +readiness engine (extraction_readiness.gaps_for), creates one Form row per +queued template, creates the batch Job, and dispatches the fill worker. +Mirrors ExtractionService.start_extraction's shape. The route stays a thin +HTTP handler and calls straight into here. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from io import BytesIO +from pathlib import Path +from uuid import UUID, uuid4 +from zipfile import ZIP_DEFLATED, ZipFile + +from sqlmodel import Session + +from app.api.schemas.enums import FormStatus, TemplateStatus +from app.api.schemas.form_generation import GenerateFormsOptions, GenerateFormsRequest +from app.core.config import DATA_DIR +from app.core.errors.base import AppError +from app.db.repositories import ( + create_generated_form, + create_job, + get_form_template, + get_incident, + list_form_templates, + update_form, + update_job, +) +from app.models import Form, FormTemplate, Job +from app.services.extraction_readiness import gaps_for +from app.services.form_templates import require_template +from app.tasks.generate_forms import generate_forms_batch_task + +# Anything outside this set is replaced in a download filename. Incident +# numbers are free text typed by a responder, so they can carry slashes, +# spaces or quotes, none of which belong in a Content-Disposition header. +# Dots go too: the only one in the name should be the one before "pdf". +_UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9_-]+") + + +@dataclass +class SkippedTemplate: + template_id: UUID + form_type: str + reason: str + + +@dataclass +class GenerationResult: + batch_id: UUID + incident_id: UUID + queued: list[Form] = field(default_factory=list) + skipped: list[SkippedTemplate] = field(default_factory=list) + job: Job | None = None + + +def _skip_reason(gaps) -> str: + """One representative reason, per the agreed format. A template can be + missing more than one required field; this names the first one, the same + way the contract's own example names a single field.""" + gap = gaps.missing_required[0] + return f"Not ready: {gap.field_name} ({gap.source.value}) has no value" + + +def _slug(value: str) -> str: + return _UNSAFE_IN_FILENAME.sub("-", value).strip("-") + + +def _pdf_filename(form: Form, incident_number: str | None) -> str: + number = _slug(incident_number) if incident_number else "" + if not number: + return f"{form.form_id}.pdf" + return f"{_slug(form.form_type)}_{number}.pdf" + + +def download_filename(session: Session, form: Form) -> str: + """The name a downloaded PDF is saved under. + + "{form_type}_{incident_number}.pdf", the same name the batch zip gives its + entries, so a form downloaded on its own and the same form pulled out of a + batch land as one file. Incident numbers are optional and are only assigned + once the department has one, so the form id stands in when it is missing. + """ + incident = get_incident(session, form.incident_id) + return _pdf_filename(form, incident.incident_number if incident else None) + + +def form_version(session: Session, form: Form) -> str | None: + """The version of the template the form was generated from. + + Read live off the template rather than stamped on the form: templates are + versioned in place, so this reports the registry's current version, not the + one in force at fill time. + """ + template = get_form_template(session, form.template_id) + return template.version if template else None + + +def batch_state(forms: list[Form]) -> str: + """processing, completed or failed, derived from the batch's Form rows. + + There is no Batch table, so both the status endpoint and the zip download + read the batch's state from the same place. Per design a single failed form + does not fail the batch: it reads completed as long as every form reached a + terminal state and at least one succeeded, and the per-form list still shows + which ones failed. + """ + total = len(forms) + completed = sum(1 for f in forms if f.status == FormStatus.completed) + failed = sum(1 for f in forms if f.status == FormStatus.failed) + if completed + failed < total: + return "processing" + return "failed" if failed == total else "completed" + + +def resolve_form_pdf(form: Form) -> Path | None: + """The form's PDF on disk, or None if it is not there to serve. + + pdf_path is stored relative to the data directory, so a value that climbs + out of it is refused rather than read. Single downloads turn a None into a + 404; the batch zip just leaves that form out. + """ + if not form.pdf_ready or not form.pdf_path: + return None + path = (DATA_DIR / form.pdf_path).resolve() + if not path.is_relative_to(DATA_DIR) or not path.is_file(): + return None + return path + + +def batch_pdfs(session: Session, forms: list[Form]) -> list[tuple[str, Path]]: + """Every finished PDF in the batch, as (name in the archive, path on disk). + + Forms that failed, or whose file has gone missing under the data directory, + are left out rather than failing the whole download. One incident lookup + covers the batch because every form in it is filled from the same incident. + """ + if not forms: + return [] + + incident = get_incident(session, forms[0].incident_id) + number = incident.incident_number if incident else None + + entries: list[tuple[str, Path]] = [] + for form in forms: + if form.status != FormStatus.completed: + continue + path = resolve_form_pdf(form) + if path is not None: + entries.append((_pdf_filename(form, number), path)) + return entries + + +def batch_zip(session: Session, batch_id: UUID, forms: list[Form]) -> tuple[bytes, str]: + """The batch's PDFs as one zip, with the name to serve it under. + + Built in memory: a batch is one incident's report pack, so it is a handful + of PDFs rather than something worth spooling to disk. + """ + entries = batch_pdfs(session, forms) + incident = get_incident(session, forms[0].incident_id) if forms else None + number = _slug(incident.incident_number) if incident and incident.incident_number else "" + + buffer = BytesIO() + with ZipFile(buffer, "w", ZIP_DEFLATED) as archive: + for name, path in entries: + archive.write(path, arcname=name) + + return buffer.getvalue(), f"fireform_batch_{number or batch_id}.zip" + + +class FormGenerationService: + def _candidates(self, session: Session, request: GenerateFormsRequest) -> list[FormTemplate]: + """The templates this request is about, before readiness is considered. + + An explicit selection is taken as given, including a legacy or draft + template the user deliberately picked. With no selection the candidates + are the active templates, the same set the readiness matrix offers on + the selection screen, so "generate everything ready" cannot pull in a + retired form nobody chose. + + Every requested template is resolved before anything is written: a bad + template_id 404s cleanly instead of leaving a partial batch behind. + """ + if request.template_ids is not None: + return [require_template(session, tid) for tid in request.template_ids] + return [t for t in list_form_templates(session) if t.status == TemplateStatus.active] + + def start_generation(self, session: Session, request: GenerateFormsRequest) -> GenerationResult: + incident = get_incident(session, request.incident_id) + if incident is None: + raise AppError( + f"Incident {request.incident_id} not found", + status_code=404, + error_code="INCIDENT_NOT_FOUND", + ) + + templates = self._candidates(session, request) + + options = request.options or GenerateFormsOptions() + contract = incident.incident_contract or {} + batch_id = uuid4() + now = datetime.now(timezone.utc) + result = GenerationResult(batch_id=batch_id, incident_id=incident.incident_id) + + for template in templates: + gaps = gaps_for(contract, template) + + if not gaps.ready and not options.force_partial: + result.skipped.append( + SkippedTemplate( + template_id=template.template_id, + form_type=template.form_type, + reason=_skip_reason(gaps), + ) + ) + continue + + form = Form( + template_id=template.template_id, + incident_id=incident.incident_id, + batch_id=batch_id, + form_type=template.form_type, + status=FormStatus.queued, + created_at=now, + updated_at=now, + ) + result.queued.append(create_generated_form(session, form)) + + if not result.queued: + # Two ways to end up here, and the caller needs to tell them apart: + # a selection whose every template turned out to be blocked, or no + # selection at all with nothing in the registry ready to generate. + raise AppError( + "None of the selected templates are ready" + if request.template_ids is not None + else "No templates were selected and none are ready", + status_code=422, + error_code="NO_FORMS_TO_GENERATE", + detail={"skipped": [s.reason for s in result.skipped]}, + ) + + job = Job(celery_task_id="", job_type="batch_form_generation", status="queued") + try: + job = create_job(session, job) + task_result = generate_forms_batch_task.delay(str(batch_id), job.job_id) + job.celery_task_id = task_result.id + job = update_job(session, job) + except Exception: + # Dispatch failed after the Form rows were already committed — + # mirrors InputService.process_voice_upload's cleanup discipline: + # nothing is left claiming to be queued with no job behind it. + failed_at = datetime.now(timezone.utc) + for queued_form in result.queued: + queued_form.status = FormStatus.failed + queued_form.updated_at = failed_at + update_form(session, queued_form) + raise + + result.job = job + return result diff --git a/app/services/form_templates.py b/app/services/form_templates.py new file mode 100644 index 00000000..613c3889 --- /dev/null +++ b/app/services/form_templates.py @@ -0,0 +1,310 @@ +"""Business logic for the contract Layer 6 template registry. + +Sits between the route handlers (app/api/routes/form_templates.py) and the +repositories. No FastAPI imports here — handlers do HTTP, this does the work: +validation/conflict checks, ORM construction, and ORM -> response mapping +(including the derived `field_count` / `last_updated`). +""" + +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID, uuid4 + +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, JobType +from app.api.schemas.templates import ( + CreateTemplateRequest, + DraftField, + PageGeometry, + TemplateDetail, + TemplateDraft, + TemplateDraftAccepted, + TemplateField, + TemplateFieldsResponse, + TemplateSummary, +) +from app.core.config import ( + DATA_DIR, + TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS, + TEMPLATE_UPLOAD_DIR, +) +from app.core.errors.base import AppError +from app.db.repositories import ( + create_form_template, + create_job, + create_template_upload, + get_form_template, + get_form_template_by_form_type, + get_template_upload, + list_form_templates, + update_form_template, + update_job, +) +from app.models import FormTemplate, Job, TemplateUpload +from app.services.template_detection import read_pages +from app.tasks.detect_fields import detect_template_fields_task + + +# --------------------------------------------------------------------------- +# Mapping helpers (ORM -> response schema). field_count / last_updated are +# derived here rather than stored on the model. +# --------------------------------------------------------------------------- +def _field_count(template: FormTemplate) -> int: + return len(template.fields or []) + + +def _to_summary(template: FormTemplate) -> TemplateSummary: + return TemplateSummary( + template_id=template.template_id, + form_type=template.form_type, + display_name=template.display_name, + jurisdiction=template.jurisdiction, + agency_type=template.agency_type, + version=template.version, + last_updated=template.updated_at.date(), + field_count=_field_count(template), + status=template.status, + ) + + +def _to_detail(template: FormTemplate) -> TemplateDetail: + return TemplateDetail( + template_id=template.template_id, + form_type=template.form_type, + display_name=template.display_name, + jurisdiction=template.jurisdiction, + agency_type=template.agency_type, + fields=template.fields, + source_standard=template.source_standard, + pdf_template_ref=template.pdf_template_ref, + version=template.version, + last_updated=template.updated_at.date(), + field_count=_field_count(template), + status=template.status, + created_at=template.created_at, + updated_at=template.updated_at, + ) + + +def require_template(db: Session, template_id: UUID) -> FormTemplate: + template = get_form_template(db, template_id) + if not template: + raise AppError( + f"Template {template_id} not found", + status_code=404, + error_code="TEMPLATE_NOT_FOUND", + ) + return template + + +# --------------------------------------------------------------------------- +# Operations +# --------------------------------------------------------------------------- +def list_templates(db: Session) -> list[TemplateSummary]: + return [_to_summary(t) for t in list_form_templates(db)] + + +def create_template(db: Session, body: CreateTemplateRequest) -> TemplateDetail: + if get_form_template_by_form_type(db, body.form_type): + raise AppError( + f"Template with form_type '{body.form_type}' already exists", + status_code=409, + error_code="TEMPLATE_EXISTS", + ) + + template = FormTemplate( + form_type=body.form_type, + display_name=body.display_name, + jurisdiction=body.jurisdiction, + agency_type=body.agency_type, + fields=[f.model_dump(mode="json") for f in body.fields], + source_standard=body.source_standard, + pdf_template_ref=body.pdf_template_ref, + ) + return _to_detail(create_form_template(db, template)) + + +def get_template(db: Session, template_id: UUID) -> TemplateDetail: + return _to_detail(require_template(db, template_id)) + + +def replace_template( + db: Session, template_id: UUID, body: CreateTemplateRequest +) -> TemplateDetail: + template = require_template(db, template_id) + + # form_type is unique in the DB, so a rename onto a form_type another + # template already holds has to be answered here. Without this the insert + # fails deep in the session and the client gets a bare 500. + clash = get_form_template_by_form_type(db, body.form_type) + if clash and clash.template_id != template_id: + raise AppError( + f"Template with form_type '{body.form_type}' already exists", + status_code=409, + error_code="TEMPLATE_EXISTS", + ) + + # Contract defines a 409 TEMPLATE_IN_USE when submitted incidents reference + # this template. The contract forms/incidents layers tie records to + # extract_id + form_type, never template_id, so there is no linkage to query + # yet. Once a form_type<->submission link exists, gate the update here. + # TODO(contract): enforce 409 TEMPLATE_IN_USE. + + template.form_type = body.form_type + template.display_name = body.display_name + template.jurisdiction = body.jurisdiction + template.agency_type = body.agency_type + template.fields = [f.model_dump(mode="json") for f in body.fields] + template.source_standard = body.source_standard + template.pdf_template_ref = body.pdf_template_ref + template.updated_at = datetime.now(timezone.utc) + return _to_detail(update_form_template(db, template)) + + +def resolve_template_pdf(db: Session, template_id: UUID) -> Path: + """On-disk path of a template's source PDF. + + `pdf_template_ref` is client-supplied, so the resolved path is checked to + still sit under the data directory before anything is served from it. + """ + template = require_template(db, template_id) + if not template.pdf_template_ref: + raise AppError( + f"Template {template_id} has no source PDF", + status_code=404, + error_code="TEMPLATE_PDF_NOT_FOUND", + ) + + path = (DATA_DIR / template.pdf_template_ref).resolve() + if not path.is_relative_to(DATA_DIR) or not path.is_file(): + raise AppError( + f"Source PDF for template {template_id} is missing", + status_code=404, + error_code="TEMPLATE_PDF_NOT_FOUND", + ) + return path + + +# --------------------------------------------------------------------------- +# PDF upload and field detection +# --------------------------------------------------------------------------- +def _to_draft(upload: TemplateUpload) -> TemplateDraft: + return TemplateDraft( + upload_id=upload.upload_id, + status=upload.status, + pdf_template_ref=upload.pdf_template_ref, + original_filename=upload.original_filename, + page_count=upload.page_count, + pages=[PageGeometry(**page) for page in upload.pages], + detected_fields=( + [DraftField(**field) for field in upload.detected_fields] + if upload.detected_fields is not None + else None + ), + detection_error=upload.detection_error, + retry_after_seconds=( + TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS + if upload.status == DetectionStatus.processing + else None + ), + ) + + +def store_upload( + db: Session, content: bytes, filename: str | None, detect_fields: bool +) -> tuple[TemplateUpload, Job | None]: + """Store a blank PDF, read its page geometry, and queue field detection. + + The PDF and its geometry are saved before returning, so the editor can + render pages immediately. Only detection runs in the background, and with + `detect_fields` off the draft is complete the moment it is created. + """ + upload_id = uuid4() + TEMPLATE_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + pdf_path = TEMPLATE_UPLOAD_DIR / f"{upload_id}.pdf" + pdf_path.write_bytes(content) + + try: + pages = read_pages(pdf_path) + except Exception as exc: + pdf_path.unlink(missing_ok=True) + raise AppError( + "Uploaded file could not be read as a PDF", + status_code=415, + error_code="INVALID_PDF", + detail={"reason": str(exc)}, + ) + + upload = TemplateUpload( + upload_id=upload_id, + status=DetectionStatus.processing if detect_fields else DetectionStatus.completed, + pdf_path=str(pdf_path), + pdf_template_ref=str(pdf_path.relative_to(DATA_DIR)), + original_filename=filename, + page_count=len(pages), + pages=[page.model_dump() for page in pages], + detected_fields=None if detect_fields else [], + ) + + if not detect_fields: + return create_template_upload(db, upload), None + + # Same order as the voice upload: create the job, dispatch, then backfill + # the celery id. A failure anywhere after the file write takes the file + # with it rather than leaving an upload nobody can finish. + try: + job = create_job( + db, + Job(celery_task_id="", job_type=JobType.template_field_detection, status="queued"), + ) + upload.job_id = job.job_id + upload = create_template_upload(db, upload) + result = detect_template_fields_task.delay(str(upload.upload_id), job.job_id) + job.celery_task_id = result.id + update_job(db, job) + except Exception: + pdf_path.unlink(missing_ok=True) + raise + + return upload, job + + +def get_draft(db: Session, upload_id: UUID) -> TemplateDraft: + upload = get_template_upload(db, upload_id) + if not upload: + raise AppError( + f"Upload {upload_id} not found", + status_code=404, + error_code="UPLOAD_NOT_FOUND", + ) + return _to_draft(upload) + + +def draft_response(upload: TemplateUpload, job: Job | None) -> TemplateDraftAccepted: + """The 202 body: the draft the editor can already render, plus where to poll.""" + draft = _to_draft(upload) + return TemplateDraftAccepted( + **draft.model_dump(), + job_id=job.job_id if job else None, + poll_url=f"/api/v1/templates/pdf/{upload.upload_id}", + ) + + +def get_template_fields( + db: Session, template_id: UUID, required_only: bool +) -> TemplateFieldsResponse: + template = require_template(db, template_id) + + fields = [TemplateField(**f) for f in template.fields] + required = [f for f in fields if f.required] + selected = required if required_only else fields + + return TemplateFieldsResponse( + template_id=template.template_id, + form_type=template.form_type, + total_fields=len(fields), + required_fields=len(required), + optional_fields=len(fields) - len(required), + fields=selected, + ) diff --git a/app/services/incident_crud.py b/app/services/incident_crud.py new file mode 100644 index 00000000..ba854c1f --- /dev/null +++ b/app/services/incident_crud.py @@ -0,0 +1,173 @@ +"""Contract Layer 4 incident CRUD (contracts/path/incidents.yaml). + +Read/write operations on the incident row itself. The promoted analytics +columns are not touched here: they are derived from the contract document by +`app.services.incidents.promote`, which runs on the extraction path and on +PATCH /extract. This module only moves the metadata a user owns, so the two +can never fight over the same column. +""" + +from datetime import date, datetime, timezone +from uuid import UUID + +from sqlmodel import Session + +from app.api.schemas.enums import IncidentCategory, ReportStatus +from app.api.schemas.incidents import CreateIncidentRequest, UpdateIncidentRequest +from app.core.errors.base import AppError +from app.db.repositories import ( + count_forms_by_incident, + get_extraction, + get_incident, + get_incident_by_extract, + get_incident_by_number, + list_forms_by_incident, + list_incidents, + update_incident, +) +from app.models import Form, Incident + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +class IncidentService: + """Business logic for the five incident endpoints.""" + + def finalize(self, session: Session, body: CreateIncidentRequest) -> Incident: + """POST /incidents: finalize the draft created when extraction completed. + + Never creates a second row. Calling it again for the same extraction + just reapplies the number and tags, so a client that retries after a + dropped response gets the same incident back. + """ + incident = get_incident_by_extract(session, body.extract_id) + if incident is None: + # Distinguish "no such extraction" from "extraction exists but has + # not produced its draft yet", because only the second is worth + # retrying. + if get_extraction(session, body.extract_id) is None: + raise AppError( + f"Extract {body.extract_id} not found", + status_code=404, + error_code="EXTRACT_NOT_FOUND", + ) + raise AppError( + "Extraction has not completed yet, so it has no incident to finalize", + status_code=409, + error_code="EXTRACTION_NOT_COMPLETED", + detail={"extract_id": str(body.extract_id)}, + ) + + if body.incident_number is not None: + self._require_number_free(session, body.incident_number, incident.incident_id) + incident.incident_number = body.incident_number + if body.tags is not None: + incident.tags = body.tags + + incident.updated_at = _now() + return update_incident(session, incident) + + def get(self, session: Session, incident_id: UUID) -> Incident: + """A single incident, soft-deleted ones included. + + Reads stay open on a deleted incident: the DELETE response promises the + row is recoverable, which is meaningless if it cannot be read back. + """ + incident = get_incident(session, incident_id) + if incident is None: + raise AppError( + f"Incident {incident_id} not found", + status_code=404, + error_code="INCIDENT_NOT_FOUND", + ) + return incident + + # Not named `list`: that would shadow the builtin for the rest of the class + # body, breaking every later `list[...]` annotation. + def list_page( + self, + session: Session, + date_from: date | None = None, + date_to: date | None = None, + incident_category: IncidentCategory | None = None, + status: ReportStatus | None = None, + page: int = 1, + per_page: int = 20, + sort: str = "date_desc", + ) -> tuple[list[Incident], dict[UUID, int], int]: + """One page of live incidents, their form counts, and the total.""" + if date_from is not None and date_to is not None and date_from > date_to: + raise AppError( + "date_from must not be later than date_to", + status_code=422, + error_code="VALIDATION_ERROR", + detail={"date_from": date_from.isoformat(), "date_to": date_to.isoformat()}, + ) + + rows, total = list_incidents( + session, + date_from=date_from, + date_to=date_to, + incident_category=incident_category, + status=status, + page=page, + per_page=per_page, + sort=sort, + ) + counts = count_forms_by_incident(session, [row.incident_id for row in rows]) + return rows, counts, total + + def forms(self, session: Session, incident_id: UUID) -> list[Form]: + return list_forms_by_incident(session, incident_id) + + def update( + self, session: Session, incident_id: UUID, body: UpdateIncidentRequest + ) -> Incident: + """PATCH /incidents/{id}: update the metadata a user owns. + + Only fields present in the request body are applied, so omitting one + leaves it alone rather than clearing it. The contract document and the + columns promoted from it are untouched; correcting those is PATCH + /extract. + """ + incident = self.get(session, incident_id) + changes = body.model_dump(exclude_unset=True) + + if "incident_number" in changes and changes["incident_number"] is not None: + self._require_number_free(session, changes["incident_number"], incident_id) + + for field, value in changes.items(): + setattr(incident, field, value) + + incident.updated_at = _now() + return update_incident(session, incident) + + def soft_delete(self, session: Session, incident_id: UUID) -> Incident: + """DELETE /incidents/{id}: stamp deleted_at. Data is never removed.""" + incident = self.get(session, incident_id) + if incident.deleted_at is not None: + raise AppError( + "Incident has already been deleted", + status_code=409, + error_code="ALREADY_DELETED", + detail={"deleted_at": incident.deleted_at.isoformat()}, + ) + + incident.deleted_at = _now() + incident.updated_at = incident.deleted_at + return update_incident(session, incident) + + def _require_number_free( + self, session: Session, incident_number: str, incident_id: UUID + ) -> None: + """Reject a number already held by a different live incident.""" + existing = get_incident_by_number(session, incident_number) + if existing is not None and existing.incident_id != incident_id: + raise AppError( + f"Incident number {incident_number} already exists", + status_code=409, + error_code="DUPLICATE_INCIDENT_NUMBER", + detail={"existing_incident_id": str(existing.incident_id)}, + ) diff --git a/app/services/incidents.py b/app/services/incidents.py new file mode 100644 index 00000000..73081052 --- /dev/null +++ b/app/services/incidents.py @@ -0,0 +1,193 @@ +"""Incident analytics recompute. + +`promote` is the single function that derives the promoted analytics columns +from the incident contract. The routes and the async worker both call it after +any change to the contract, so the columns can never drift from the document +they are derived from. It takes a plain contract dict and returns a plain dict +of column values; it never touches the database. +""" + +from datetime import datetime +from typing import Any + +# Keys of the value dict returned by ``promote``. Kept explicit so callers can +# apply the result to an Incident row without guessing field names. +PROMOTED_COLUMNS = ( + "incident_name", + "incident_type", + "incident_category", + "incident_datetime", + "city", + "state", + "country", + "civilian_injuries", + "civilian_fatalities", + "responder_injuries", + "responder_fatalities", + "people_rescued", + "people_evacuated", + "structures_destroyed", + "area_burned_ha", + "total_loss_amount", + "total_loss_currency", + "call_to_arrival_seconds", + "turnout_seconds_first_unit", + "travel_seconds_first_unit", + "on_scene_duration_seconds", +) + + +def _obj(contract: dict, key: str) -> dict: + """Return contract[key] if it is a dict, else an empty dict.""" + value = contract.get(key) + return value if isinstance(value, dict) else {} + + +def _parse_dt(value: Any) -> datetime | None: + """Parse an RFC 3339 date-time string; return None if absent or unparseable.""" + if isinstance(value, datetime): + return value + if not isinstance(value, str) or not value: + return None + try: + # fromisoformat handles the 'Z' suffix from Python 3.11 onwards. + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _seconds_between(start: Any, end: Any) -> int | None: + """Whole seconds from start to end; None unless both parse and end >= start.""" + a = _parse_dt(start) + b = _parse_dt(end) + if a is None or b is None: + return None + delta = (b - a).total_seconds() + if delta < 0: + return None + return int(delta) + + +def _primary_type(incident: dict) -> dict: + """The incident type flagged primary, else the first one, else empty. + + Both `incident_category` and `incident_type` come off this single entry, so + they can never describe two different types. + """ + types = incident.get("types") + if not isinstance(types, list): + return {} + entries = [t for t in types if isinstance(t, dict)] + if not entries: + return {} + for entry in entries: + if entry.get("primary"): + return entry + return entries[0] + + +def _incident_datetime(incident: dict, dispatch: dict) -> datetime | None: + """Alarm, else start, else dispatch call-received, parsed to a datetime.""" + raw = ( + incident.get("alarm_datetime") + or incident.get("start_datetime") + or dispatch.get("call_received_datetime") + ) + return _parse_dt(raw) + + +def _total_loss(losses: dict) -> tuple[float | None, str | None]: + """Sum property and contents loss; currency from whichever is present first.""" + prop = losses.get("property_loss") if isinstance(losses.get("property_loss"), dict) else {} + contents = losses.get("contents_loss") if isinstance(losses.get("contents_loss"), dict) else {} + amounts = [v for v in (prop.get("amount"), contents.get("amount")) if isinstance(v, (int, float))] + total = sum(amounts) if amounts else None + currency = prop.get("currency") or contents.get("currency") + return total, currency + + +def _first_unit(contract: dict) -> dict: + """The unit that arrived first (earliest arrived_datetime), else empty.""" + units = contract.get("units") + if not isinstance(units, list): + return {} + arrived = [] + for unit in units: + if not isinstance(unit, dict): + continue + at = _parse_dt(unit.get("arrived_datetime")) + if at is not None: + arrived.append((at, unit)) + if not arrived: + return {} + arrived.sort(key=lambda pair: pair[0]) + return arrived[0][1] + + +def _unit_turnout(unit: dict) -> int | None: + """Precomputed turnout_seconds, else dispatched to enroute.""" + precomputed = unit.get("turnout_seconds") + if isinstance(precomputed, int): + return precomputed + return _seconds_between(unit.get("dispatched_datetime"), unit.get("enroute_datetime")) + + +def _unit_travel(unit: dict) -> int | None: + """Precomputed travel_seconds, else enroute to arrived.""" + precomputed = unit.get("travel_seconds") + if isinstance(precomputed, int): + return precomputed + return _seconds_between(unit.get("enroute_datetime"), unit.get("arrived_datetime")) + + +def _count(rescues: Any) -> int | None: + """Number of entries in a list field, or None when it is absent.""" + return len(rescues) if isinstance(rescues, list) else None + + +def promote(contract: dict | None) -> dict[str, Any]: + """Derive the promoted analytics columns from the incident contract. + + Every value is nullable; an empty or partial contract yields None for the + fields it does not cover. This is the only place these columns are computed. + """ + contract = contract or {} + incident = _obj(contract, "incident") + dispatch = _obj(contract, "dispatch") + location = _obj(contract, "location") + casualties = _obj(contract, "casualties") + evacuation = _obj(contract, "evacuation_displacement") + structure = _obj(contract, "structure") + wildland = _obj(contract, "wildland") + losses = _obj(contract, "losses") + + total_loss_amount, total_loss_currency = _total_loss(losses) + + call_received = dispatch.get("call_received_datetime") or incident.get("alarm_datetime") + first_arrival = incident.get("first_arrival_datetime") + first_unit = _first_unit(contract) + primary_type = _primary_type(incident) + + return { + "incident_name": incident.get("name"), + "incident_type": primary_type.get("subcategory"), + "incident_category": primary_type.get("category"), + "incident_datetime": _incident_datetime(incident, dispatch), + "city": location.get("city"), + "state": location.get("state"), + "country": location.get("country"), + "civilian_injuries": casualties.get("total_civilian_injuries"), + "civilian_fatalities": casualties.get("total_civilian_fatalities"), + "responder_injuries": casualties.get("total_responder_injuries"), + "responder_fatalities": casualties.get("total_responder_fatalities"), + "people_rescued": _count(contract.get("rescues")), + "people_evacuated": evacuation.get("total_people_evacuated"), + "structures_destroyed": structure.get("structures_destroyed"), + "area_burned_ha": wildland.get("area_burned_ha"), + "total_loss_amount": total_loss_amount, + "total_loss_currency": total_loss_currency, + "call_to_arrival_seconds": _seconds_between(call_received, first_arrival), + "turnout_seconds_first_unit": _unit_turnout(first_unit), + "travel_seconds_first_unit": _unit_travel(first_unit), + "on_scene_duration_seconds": _seconds_between(first_arrival, incident.get("cleared_datetime")), + } diff --git a/app/services/llm.py b/app/services/llm.py deleted file mode 100644 index e2d1639d..00000000 --- a/app/services/llm.py +++ /dev/null @@ -1,99 +0,0 @@ -import json -import os -import requests -from requests.exceptions import Timeout, RequestException - -from app.core.config import OLLAMA_HOST, OLLAMA_MODEL -from app.core.logging import get_logger - -logger = get_logger(__name__) - - -class LLM: - def __init__(self, transcript_text: str=None, target_fields: list=None, json_dict: dict=None, model: str=None): - self._transcript_text = transcript_text - self._target_fields = target_fields - self._json = json_dict if json_dict is not None else {} - # Optional per-request model override; falls back to OLLAMA_MODEL env. - self._model = model - - def build_prompt(self, current_field: str, current_type: str = "string"): - """ - This method is in charge of the prompt engineering. It creates a specific prompt for each target field. - @params: current_field -> represents the current element of the json that is being prompted. - @params: current_type -> hint to the LLM about the expected value shape (date, number, etc.). - """ - prompt_path = os.path.join(os.path.dirname(__file__), "prompt.txt") - with open(prompt_path, "r") as f: - template = f.read() - - return template.format(field=current_field, type=current_type, text=self._transcript_text) - - def main_loop(self): - timeout = 45 - max_retries = 3 - - total_fields = len(self._target_fields) - for i, (field, field_type) in enumerate(self._target_fields.items(), 1): - prompt = self.build_prompt(field, field_type if isinstance(field_type, str) else "string") - ollama_url = f"{OLLAMA_HOST}/api/generate" - ollama_model = self._model or OLLAMA_MODEL - - payload = { - "model": ollama_model, - "prompt": prompt, - "stream": False, - } - - json_data = None - try: - for attempt in range(max_retries): - try: - response = requests.post(ollama_url, json=payload, timeout=timeout) - response.raise_for_status() - json_data = response.json() - break - except Timeout: - logger.warning("Ollama request timed out (attempt %d) for field '%s'. Retrying...", attempt + 1, field) - except RequestException as e: - logger.error("Ollama request failed: %s", e) - except requests.exceptions.ConnectionError: - raise ConnectionError( - f"Could not connect to Ollama at {ollama_url}. " - "Please ensure Ollama is running and accessible." - ) - except requests.exceptions.HTTPError as e: - raise RuntimeError(f"Ollama returned an error: {e}") - - if json_data is None: - raise RuntimeError("Failed to get response from Ollama after retries.") - else: - parsed_response = json_data["response"] - self.add_response_to_json(field, parsed_response) - logger.info("[%d/%d] Extracted data for field '%s' successfully.", i, total_fields, field) - - logger.info("Resulting JSON created from the input text:\n%s", json.dumps(self._json, indent=2)) - - return self - - def add_response_to_json(self, field: str, value: str): - """ - this method adds the following value under the specified field, - or under a new field if the field doesn't exist, to the json dict - """ - value = value.strip().replace('"', "") - parsed_value = None - - if value != "-1": - parsed_value = value - - if field in self._json.keys(): - self._json[field].append(parsed_value) - else: - self._json[field] = parsed_value - - return - - - def get_data(self): - return self._json diff --git a/app/services/llm/__init__.py b/app/services/llm/__init__.py new file mode 100644 index 00000000..3f1dd5f5 --- /dev/null +++ b/app/services/llm/__init__.py @@ -0,0 +1,60 @@ +"""Talking to a language model. + +This is the only part of FireForm that sends a prompt anywhere. Import from +here, never from the modules underneath, so that swapping a provider or changing +how retries work stays a change to one package. + + from app.services import llm + + fields = llm.generate_json(prompt) + summary = llm.generate(prompt) + +Which backend answers is a deployment setting, LLM_PROVIDER, not a per request +choice. Ollama, OpenAI, Gemini and Claude are supported by name, and "custom" +covers anything else that serves the OpenAI chat completions API. +""" + +from app.services.llm.client import ( + check_config, + generate, + generate_json, + get_settings, + health, + list_models, + reset, +) +from app.services.llm.errors import ( + LLMAuthError, + LLMConfigError, + LLMError, + LLMRateLimitError, + LLMResponseError, + LLMTimeoutError, + LLMUnavailableError, +) +from app.services.llm.gate import RateLimitGate +from app.services.llm.models import LLMSettings, ModelInfo, Provider, ProviderHealth +from app.services.llm.providers import PROVIDERS + +__all__ = [ + "PROVIDERS", + "LLMAuthError", + "LLMConfigError", + "LLMError", + "LLMRateLimitError", + "LLMResponseError", + "LLMSettings", + "LLMTimeoutError", + "LLMUnavailableError", + "ModelInfo", + "Provider", + "ProviderHealth", + "RateLimitGate", + "check_config", + "generate", + "generate_json", + "get_settings", + "health", + "list_models", + "reset", +] diff --git a/app/services/llm/client.py b/app/services/llm/client.py new file mode 100644 index 00000000..add230b1 --- /dev/null +++ b/app/services/llm/client.py @@ -0,0 +1,351 @@ +"""The one place FireForm talks to a model. + +Every provider we support speaks the OpenAI chat completions API, so there is +one request shape, one response shape and one set of failures to handle. What +differs between providers lives in the table in providers.py, not here. + +Two things in this file are load bearing and easy to undo by accident. The SDK +client is built with max_retries=0, because the SDK's own retries would sit +underneath the rate limit policy here and quietly turn ten attempts into thirty. +And the rate limit wait is deliberately long, because a 429 is the provider +asking us to wait, not telling us we are wrong. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Callable, TypeVar + +import openai +from openai import OpenAI + +from app.core.logging import get_logger +from app.services.llm.errors import ( + LLMAuthError, + LLMConfigError, + LLMRateLimitError, + LLMResponseError, + LLMTimeoutError, + LLMUnavailableError, +) +from app.services.llm.gate import RateLimitGate +from app.services.llm.models import LLMSettings, ModelInfo, ProviderHealth +from app.services.llm.parsing import extract_json_object +from app.services.llm.providers import resolve + +logger = get_logger(__name__) + +T = TypeVar("T") + +_OVERLOADED_STATUS = 529 +_OPTIONAL_PARAMS = ("response_format", "max_tokens", "temperature") + +_lock = threading.Lock() +_settings: LLMSettings | None = None +_client: OpenAI | None = None + + +def get_settings() -> LLMSettings: + """Resolved provider settings, worked out once per process.""" + global _settings + with _lock: + if _settings is None: + _settings = resolve() + return _settings + + +def get_client() -> OpenAI: + """The SDK client, built once per process.""" + global _client + settings = get_settings() + with _lock: + if _client is None: + _client = OpenAI( + api_key=settings.api_key, + base_url=settings.base_url, + timeout=settings.timeout, + default_headers=settings.extra_headers, + max_retries=0, + ) + return _client + + +def reset() -> None: + """Drop the cached settings and client. Used by tests and after a reconfig.""" + global _settings, _client + with _lock: + _settings = None + _client = None + + +def check_config() -> LLMSettings: + """Resolve settings now so a bad configuration stops the process at startup. + + Raises LLMConfigError with a message naming the setting to fix. + """ + settings = get_settings() + where = settings.base_url or "the provider's default endpoint" + logger.info( + "LLM provider: %s, model %s, endpoint %s%s", + settings.label, + settings.model, + where, + ", prompts leave this machine" if settings.external else "", + ) + return settings + + +def _retry_after(exc: Exception, fallback: float, ceiling: float) -> float: + """How long to wait, preferring what the provider asked for.""" + response = getattr(exc, "response", None) + header = None + if response is not None: + try: + header = response.headers.get("retry-after") + except Exception: + header = None + if header: + try: + asked = float(header) + except ValueError: + asked = fallback + return min(max(asked, fallback), ceiling) + return fallback + + +def _with_retries(call: Callable[[], T], *, what: str, gate: RateLimitGate | None = None) -> T: + """Run one provider call, applying the rate limit and server error policy.""" + settings = get_settings() + attempts = settings.rate_limit_retries + 1 + server_attempts = settings.server_retries + 1 + server_tries = 0 + + for attempt in range(1, attempts + 1): + if gate is not None: + gate.check() + try: + return call() + except openai.RateLimitError as exc: + wait = _retry_after(exc, settings.rate_limit_wait, settings.rate_limit_max_wait) + if attempt == attempts: + error = LLMRateLimitError( + f"{settings.label} is still rate limiting after {attempts} attempts " + f"over about {int(wait * attempts)}s ({what})", + retry_after_seconds=wait, + ) + if gate is not None: + gate.trip(error) + logger.error("%s", error) + raise error from exc + logger.warning( + "%s rate limited (attempt %d of %d), waiting %.0fs", + settings.label, + attempt, + attempts, + wait, + ) + time.sleep(wait) + except (openai.AuthenticationError, openai.PermissionDeniedError) as exc: + raise LLMAuthError( + f"{settings.label} rejected the API key. Check the key for this provider." + ) from exc + except openai.APITimeoutError as exc: + raise LLMTimeoutError( + f"{settings.label} did not answer within {settings.timeout}s ({what})" + ) from exc + except openai.APIConnectionError as exc: + raise LLMUnavailableError( + f"could not reach {settings.label} at " + f"{settings.base_url or 'its default endpoint'}: {exc}" + ) from exc + except openai.APIStatusError as exc: + status = getattr(exc, "status_code", None) + if status == _OVERLOADED_STATUS: + if attempt == attempts: + error = LLMRateLimitError( + f"{settings.label} reported itself overloaded on every attempt ({what})", + retry_after_seconds=settings.rate_limit_wait, + ) + if gate is not None: + gate.trip(error) + raise error from exc + time.sleep( + _retry_after(exc, settings.rate_limit_wait, settings.rate_limit_max_wait) + ) + continue + if status is not None and status >= 500: + server_tries += 1 + if server_tries >= server_attempts: + raise LLMUnavailableError( + f"{settings.label} returned {status} on {server_tries} attempts ({what})" + ) from exc + time.sleep(settings.server_retry_wait) + continue + raise LLMResponseError(f"{settings.label} rejected the request ({what}): {exc}") from exc + + raise LLMUnavailableError(f"{settings.label} could not be called ({what})") + + +def _named_param(message: str) -> str | None: + """Which optional parameter an error message is complaining about.""" + lowered = message.lower() + for name in _OPTIONAL_PARAMS: + if name in lowered: + return name + return None + + +def _create(payload: dict[str, Any]) -> Any: + """One chat completion, dropping any optional parameter the model refuses. + + Some hosted models reject a parameter instead of ignoring it, and which ones + do changes with every release. Reading the complaint and trying again + without that parameter is cheaper than keeping a compatibility matrix. + """ + body = dict(payload) + for _ in range(len(_OPTIONAL_PARAMS)): + try: + return get_client().chat.completions.create(**body) + except openai.BadRequestError as exc: + name = _named_param(str(exc)) + if name is None or name not in body: + raise + logger.warning("provider rejected %s, retrying without it", name) + body.pop(name) + return get_client().chat.completions.create(**body) + + +def _answer_text(response: Any, prefilled: bool) -> str: + """The answer as text, with a prefilled opening brace put back.""" + choices = getattr(response, "choices", None) or [] + if not choices: + raise LLMResponseError("the provider returned no answer") + text = choices[0].message.content or "" + if prefilled and not text.lstrip().startswith("{"): + text = "{" + text + return text + + +def generate( + prompt: str, + *, + model: str | None = None, + max_tokens: int | None = None, + timeout: int | None = None, + json_object: bool = False, + gate: RateLimitGate | None = None, +) -> str: + """Send one prompt and return the answer as text. + + `json_object` asks the provider for JSON where it supports that, and + otherwise falls back to starting the answer with an opening brace. Callers + that want a parsed object should use generate_json. + """ + settings = get_settings() + prefill = json_object and settings.json_prefill + + messages: list[dict[str, str]] = [{"role": "user", "content": prompt}] + if prefill: + messages.append({"role": "assistant", "content": "{"}) + + payload: dict[str, Any] = { + "model": model or settings.model, + "messages": messages, + "temperature": 0, + "max_tokens": max_tokens or settings.max_tokens, + "timeout": timeout or settings.timeout, + } + if json_object and settings.json_mode: + payload["response_format"] = {"type": "json_object"} + + what = f"model {payload['model']}" + response = _with_retries(lambda: _create(payload), what=what, gate=gate) + return _answer_text(response, prefill) + + +def generate_json( + prompt: str, + *, + model: str | None = None, + max_tokens: int | None = None, + timeout: int | None = None, + gate: RateLimitGate | None = None, +) -> dict[str, Any]: + """Send one prompt and return the JSON object the model answered with.""" + text = generate( + prompt, + model=model, + max_tokens=max_tokens, + timeout=timeout, + json_object=True, + gate=gate, + ) + return extract_json_object(text) + + +def list_models() -> list[ModelInfo]: + """Models the provider will serve, with the configured one marked. + + A provider that will not list them, because the key lacks the permission or + the endpoint does not implement it, still gets an answer: the model this + deployment is configured to use. + """ + settings = get_settings() + try: + names = [model.id for model in get_client().models.list()] + except Exception as exc: + logger.warning("%s would not list models: %s", settings.label, exc) + names = [] + + if settings.model not in names: + names.insert(0, settings.model) + return [ModelInfo(name=name, default=name == settings.model) for name in names] + + +def health() -> ProviderHealth: + """Whether the model backend is usable, without spending money to find out. + + A local provider is cheap to ask, so it gets asked. A hosted one is not + probed: a listing call on every health check costs quota and rate limit + headroom to answer a question the configuration already answers. + """ + try: + settings = get_settings() + except LLMConfigError as exc: + return ProviderHealth( + provider="unknown", + label="unknown", + model="", + external=False, + status="unhealthy", + probed=False, + detail=str(exc), + ) + + base = { + "provider": settings.provider, + "label": settings.label, + "model": settings.model, + "external": settings.external, + } + + if settings.external: + return ProviderHealth( + **base, + status="healthy", + probed=False, + detail="hosted provider, not probed to avoid spending quota", + ) + + started = time.monotonic() + try: + get_client().models.list() + except Exception as exc: + return ProviderHealth(**base, status="unhealthy", probed=True, detail=str(exc)) + return ProviderHealth( + **base, + status="healthy", + probed=True, + response_time_ms=int((time.monotonic() - started) * 1000), + ) diff --git a/app/services/llm/errors.py b/app/services/llm/errors.py new file mode 100644 index 00000000..0b504402 --- /dev/null +++ b/app/services/llm/errors.py @@ -0,0 +1,45 @@ +"""What can go wrong when talking to a model. + +Every provider reaches the rest of the app through these, so a caller never has +to know whether it was Ollama refusing a connection or Gemini refusing a key. +The split is by what the caller should do about it: give up now, wait and try +later, or treat the answer as unusable and move on. +""" + +from __future__ import annotations + + +class LLMError(RuntimeError): + """Base for everything this module raises.""" + + +class LLMConfigError(LLMError): + """The provider settings do not make sense. Raised at startup, not mid request.""" + + +class LLMUnavailableError(LLMError): + """The provider could not be reached, so nothing is extractable right now.""" + + +class LLMAuthError(LLMError): + """The key was rejected. Never retried, since it will be rejected again.""" + + +class LLMRateLimitError(LLMError): + """Still rate limited after every retry was used up. + + Carries the wait the provider last asked for, so the caller can pass a + useful Retry-After to whoever asked for the extraction. + """ + + def __init__(self, message: str, retry_after_seconds: float | None = None): + super().__init__(message) + self.retry_after_seconds = retry_after_seconds + + +class LLMResponseError(LLMError): + """The call went through but the answer is unusable. One prompt's problem.""" + + +class LLMTimeoutError(LLMResponseError): + """The call ran past the timeout. Rarely worth retrying, the second try takes as long.""" diff --git a/app/services/llm/gate.py b/app/services/llm/gate.py new file mode 100644 index 00000000..f1eb58f2 --- /dev/null +++ b/app/services/llm/gate.py @@ -0,0 +1,46 @@ +"""A shared stop signal for one batch of prompts. + +Ten retries ten seconds apart is a sensible wait for one call. It is a terrible +wait repeated across twenty prompts running four at a time, which is what an +extraction is: the first prompt waits its hundred seconds, and then so does +every prompt behind it, for a limit that is clearly not going to lift. + +So the batch shares a gate. The first prompt to run out of retries trips it, and +everything still queued fails immediately with the same error instead of paying +the wait again. Callers that pass no gate are unaffected. +""" + +from __future__ import annotations + +import threading + +from app.services.llm.errors import LLMRateLimitError + + +class RateLimitGate: + """Trip once, and every later check through this gate fails the same way.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._tripped: LLMRateLimitError | None = None + + @property + def tripped(self) -> bool: + with self._lock: + return self._tripped is not None + + def trip(self, error: LLMRateLimitError) -> None: + """Record the first rate limit failure. Later ones change nothing.""" + with self._lock: + if self._tripped is None: + self._tripped = error + + def check(self) -> None: + """Raise straight away if this batch already gave up on rate limits.""" + with self._lock: + tripped = self._tripped + if tripped is not None: + raise LLMRateLimitError( + f"skipped, the provider is rate limiting this run: {tripped}", + retry_after_seconds=tripped.retry_after_seconds, + ) diff --git a/app/services/llm/models.py b/app/services/llm/models.py new file mode 100644 index 00000000..284795a8 --- /dev/null +++ b/app/services/llm/models.py @@ -0,0 +1,75 @@ +"""Data shapes used across the LLM module. + +Plain dataclasses, not Pydantic and not ORM models. Nothing here is an HTTP +body or a table; the API schemas that wrap these live in app/api/schemas/. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Provider: + """One backend, described by what makes it different from the others. + + `external=None` means it depends on the URL, which is the custom endpoint + case. `json_prefill` starts the answer with an opening brace, the only lever + available on a provider that has no JSON mode. + """ + + name: str + label: str + key_setting: str | None + key_required: bool + default_base_url: str | None + base_url_required: bool + json_mode: bool + json_prefill: bool + external: bool | None + model_required: bool + + +@dataclass(frozen=True) +class LLMSettings: + """Everything one call needs, resolved once and reused.""" + + provider: str + label: str + model: str + base_url: str | None + api_key: str + timeout: int + max_tokens: int + json_mode: bool + json_prefill: bool + external: bool + extra_headers: dict[str, str] | None + rate_limit_retries: int + rate_limit_wait: float + rate_limit_max_wait: float + respect_retry_after: bool + server_retries: int + server_retry_wait: float + + +@dataclass(frozen=True) +class ModelInfo: + """One model the provider will answer to.""" + + name: str + default: bool = False + + +@dataclass(frozen=True) +class ProviderHealth: + """Whether the model backend is usable, for the health endpoint.""" + + provider: str + label: str + model: str + external: bool + status: str + probed: bool + detail: str | None = None + response_time_ms: int | None = None diff --git a/app/services/llm/parsing.py b/app/services/llm/parsing.py new file mode 100644 index 00000000..d6822994 --- /dev/null +++ b/app/services/llm/parsing.py @@ -0,0 +1,126 @@ +"""Reading a JSON object out of whatever the model actually said. + +Even with JSON mode on, a small model will wrap its answer in a code fence or +put a sentence in front of it, and no provider offers JSON mode on every model +we support. So the parser tolerates both rather than throwing away a good answer +over formatting, and rebuilds an answer that was cut off at the token ceiling +instead of losing the whole thing. + +Lifted from the extraction chunk client, which is where it earned its keep. +Every provider needs it now, so it lives here. +""" + +from __future__ import annotations + +import json +from typing import Any + +from app.core.logging import get_logger +from app.services.llm.errors import LLMResponseError + +logger = get_logger(__name__) + + +def close_truncated(text: str) -> str | None: + """Rebuild a JSON object that was cut off mid answer, or return None. + + An answer that hits the token ceiling ends in the middle of a value and + parses as nothing, losing a section that was mostly fine. This trims back to + the last point the text was known to be complete, a closing bracket or a + comma between values, then closes whatever is still open. + + Two properties make this safe to run on a model's answer. It never writes a + value, so the worst it can do is drop fields, never invent one. And it only + cuts at a boundary outside a string, so a comma or a brace inside a value + cannot be mistaken for the end of a field. + + It runs only after normal parsing has already failed, so when it fails too + the caller raises the same error it would have raised anyway. + """ + depth: list[str] = [] + in_string = False + escaped = False + last_complete = None + + for index, char in enumerate(text): + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "{[": + depth.append("}" if char == "{" else "]") + elif char in "}]": + if depth: + depth.pop() + last_complete = index + 1 + elif char == "," and depth: + last_complete = index + + if last_complete is None: + return None + + head = text[:last_complete] + stack: list[str] = [] + in_string = False + escaped = False + for char in head: + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "{[": + stack.append("}" if char == "{" else "]") + elif char in "}]" and stack: + stack.pop() + + return head + "".join(reversed(stack)) + + +def extract_json_object(raw: str) -> dict[str, Any]: + """Parse the model's answer, ignoring a code fence or surrounding prose.""" + text = raw.strip() + if text.startswith("```"): + text = text.split("```")[1] if "```" in text[3:] else text[3:] + text = text.removeprefix("json").strip() + + try: + parsed = json.loads(text) + except json.JSONDecodeError as first_error: + start = text.find("{") + end = text.rfind("}") + candidate = text[start : end + 1] if start != -1 and end > start else None + try: + parsed = json.loads(candidate) if candidate else None + except json.JSONDecodeError: + parsed = None + + if parsed is None: + repaired = close_truncated(text[start:] if start != -1 else text) + if repaired is None: + raise LLMResponseError( + f"no JSON object in the model's answer: {raw[:200]!r}" + ) from first_error + try: + parsed = json.loads(repaired) + except json.JSONDecodeError as exc: + raise LLMResponseError(f"unparseable JSON from the model: {exc}") from exc + logger.warning( + "the model's answer was cut off; kept the %d complete field(s) before the cut", + len(parsed) if isinstance(parsed, dict) else 0, + ) + + if not isinstance(parsed, dict): + raise LLMResponseError(f"expected a JSON object, got {type(parsed).__name__}") + return parsed diff --git a/app/services/llm/providers.py b/app/services/llm/providers.py new file mode 100644 index 00000000..be4a28e1 --- /dev/null +++ b/app/services/llm/providers.py @@ -0,0 +1,211 @@ +"""Which providers exist, and turning the environment into settings. + +Every provider here speaks the OpenAI chat completions API, so the table below +holds only what differs: where to send the request, which key opens it, and +whether the provider can be made to answer in JSON. Adding a provider is a row. + +Nothing here makes a network call. Resolving settings is pure, which is what +lets the backend refuse to start on a bad configuration instead of finding out +halfway through someone's incident report. +""" + +from __future__ import annotations + +import ipaddress +import json +from urllib.parse import urlparse + +from app.core import config as app_config +from app.services.llm.errors import LLMConfigError +from app.services.llm.models import LLMSettings, Provider + +PROVIDERS: dict[str, Provider] = { + "ollama": Provider( + name="ollama", + label="Ollama", + key_setting=None, + key_required=False, + default_base_url=None, + base_url_required=False, + json_mode=True, + json_prefill=False, + external=False, + model_required=False, + ), + "openai": Provider( + name="openai", + label="OpenAI", + key_setting="OPENAI_API_KEY", + key_required=True, + default_base_url=None, + base_url_required=False, + json_mode=True, + json_prefill=False, + external=True, + model_required=True, + ), + "gemini": Provider( + name="gemini", + label="Google Gemini", + key_setting="GEMINI_API_KEY", + key_required=True, + default_base_url="https://generativelanguage.googleapis.com/v1beta/openai/", + base_url_required=False, + json_mode=True, + json_prefill=False, + external=True, + model_required=True, + ), + "anthropic": Provider( + name="anthropic", + label="Anthropic Claude", + key_setting="ANTHROPIC_API_KEY", + key_required=True, + default_base_url="https://api.anthropic.com/v1/", + base_url_required=False, + # Their compatibility layer documents response_format as ignored, so + # asking for it would be a silent no-op. The prefill does the job. + json_mode=False, + json_prefill=True, + external=True, + model_required=True, + ), + "custom": Provider( + name="custom", + label="Custom endpoint", + key_setting="LLM_API_KEY", + key_required=False, + default_base_url=None, + base_url_required=True, + json_mode=True, + json_prefill=False, + external=None, + model_required=True, + ), +} + +# The SDK rejects an empty key before it sends anything, so an endpoint that +# needs no auth still gets a placeholder. +_NO_KEY_PLACEHOLDER = "not-needed" + + +def _is_local(url: str) -> bool: + """True when a URL points at this machine or a private network. + + A hostname that does not parse as an IP is treated as remote, because + guessing wrong in that direction is the safe way to guess wrong. + """ + host = urlparse(url).hostname + if not host: + return False + if host in {"localhost", "host.docker.internal"}: + return True + if "." not in host and ":" not in host: + return True + try: + address = ipaddress.ip_address(host) + except ValueError: + return False + return address.is_loopback or address.is_private + + +def _parse_headers(raw: str) -> dict[str, str] | None: + if not raw: + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise LLMConfigError( + f"LLM_EXTRA_HEADERS is not valid JSON: {exc}. " + 'It should look like {"X-My-Header": "value"}.' + ) from exc + if not isinstance(parsed, dict) or not all( + isinstance(key, str) and isinstance(value, str) for key, value in parsed.items() + ): + raise LLMConfigError( + 'LLM_EXTRA_HEADERS must be a JSON object of strings, such as {"X-My-Header": "value"}.' + ) + return parsed + + +def _resolve_base_url(provider: Provider, cfg) -> str | None: + """Where requests go, preferring an explicit override.""" + if cfg.LLM_BASE_URL: + return cfg.LLM_BASE_URL + if provider.name == "ollama": + return f"{cfg.OLLAMA_HOST}/v1" + return provider.default_base_url + + +def resolve(cfg=None) -> LLMSettings: + """Build the settings for the configured provider, or explain what is wrong. + + `cfg` is the config module, read at call time rather than bound as a default + so tests can hand over a stand-in instead of editing the environment. + """ + cfg = cfg or app_config + provider = PROVIDERS.get(cfg.LLM_PROVIDER) + if provider is None: + known = ", ".join(sorted(PROVIDERS)) + raise LLMConfigError( + f"LLM_PROVIDER is set to {cfg.LLM_PROVIDER!r}, which is not a provider. " + f"Pick one of: {known}." + ) + + base_url = _resolve_base_url(provider, cfg) + if provider.base_url_required and not base_url: + raise LLMConfigError( + f"LLM_PROVIDER={provider.name} needs LLM_BASE_URL set to an endpoint " + "that serves the OpenAI chat completions API, such as " + "http://localhost:8001/v1." + ) + + model = cfg.LLM_MODEL or (cfg.OLLAMA_MODEL if provider.name == "ollama" else "") + if not model: + raise LLMConfigError( + f"LLM_PROVIDER={provider.name} needs LLM_MODEL set. " + f"{provider.label} has no default here on purpose, because a model name " + "pinned in source goes stale." + ) + + api_key = getattr(cfg, provider.key_setting, "") if provider.key_setting else "" + if provider.key_required and not api_key: + raise LLMConfigError(f"LLM_PROVIDER={provider.name} needs {provider.key_setting} set.") + + external = provider.external + if external is None: + external = not _is_local(base_url or "") + + if external and not cfg.LLM_ALLOW_EXTERNAL: + raise LLMConfigError( + f"LLM_PROVIDER={provider.name} sends incident narratives to {provider.label}, " + "which means names, addresses and medical detail leave this machine. " + "Set LLM_ALLOW_EXTERNAL=true to allow that, or use a local provider." + ) + + if cfg.LLM_RATE_LIMIT_RETRIES < 0 or cfg.LLM_RATE_LIMIT_WAIT_SECONDS < 0: + raise LLMConfigError( + "LLM_RATE_LIMIT_RETRIES and LLM_RATE_LIMIT_WAIT_SECONDS cannot be negative." + ) + if cfg.LLM_TIMEOUT <= 0 or cfg.LLM_MAX_TOKENS <= 0: + raise LLMConfigError("LLM_TIMEOUT and LLM_MAX_TOKENS must be greater than zero.") + + return LLMSettings( + provider=provider.name, + label=provider.label, + model=model, + base_url=base_url, + api_key=api_key or _NO_KEY_PLACEHOLDER, + timeout=cfg.LLM_TIMEOUT, + max_tokens=cfg.LLM_MAX_TOKENS, + json_mode=provider.json_mode, + json_prefill=provider.json_prefill, + external=external, + extra_headers=_parse_headers(cfg.LLM_EXTRA_HEADERS), + rate_limit_retries=cfg.LLM_RATE_LIMIT_RETRIES, + rate_limit_wait=cfg.LLM_RATE_LIMIT_WAIT_SECONDS, + rate_limit_max_wait=cfg.LLM_RATE_LIMIT_MAX_WAIT_SECONDS, + respect_retry_after=cfg.LLM_RESPECT_RETRY_AFTER, + server_retries=cfg.LLM_SERVER_RETRIES, + server_retry_wait=cfg.LLM_SERVER_RETRY_WAIT_SECONDS, + ) diff --git a/app/services/template_detection.py b/app/services/template_detection.py new file mode 100644 index 00000000..634bf220 --- /dev/null +++ b/app/services/template_detection.py @@ -0,0 +1,401 @@ +"""Field detection for an uploaded template PDF. + +commonforms finds the boxes, this turns them into editable template fields. +Widget rectangles come out of the PDF already in points with a bottom-left +origin, which is exactly what `TemplateFieldLayout` stores, so no coordinate +conversion happens anywhere on the backend. The editor converts to pixels for +display and back again on save. + +Detection is best effort by design. A box whose label cannot be read still +comes back with its geometry, and geometry alone is a fine starting point for +someone drawing the rest by hand. +""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from pypdf import PdfReader, PdfWriter +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, FieldSource, TemplateFieldType +from app.api.schemas.templates import ( + DraftField, + MappingSuggestion, + PageGeometry, + TemplateField, + TemplateFieldLayout, +) +from app.core.config import ( + MAPPING_AUTO_APPLY_SCORE, + MAPPING_SUGGESTION_FLOOR, + MAX_MAPPING_SUGGESTIONS, +) +from app.core.logging import get_logger +from app.db.repositories import ( + get_job_by_uuid, + get_template_upload, + update_job, + update_template_upload, +) +from app.services import field_catalog + +logger = get_logger(__name__) + +_NAME_CLEANUP = re.compile(r"[^a-z0-9]+") + +# How far from a box a piece of text can sit and still be taken as its label. +# Both are in points: roughly two characters of slack to the left, and a little +# over one line height above. +_LABEL_GAP_LEFT = 160.0 +_LABEL_GAP_ABOVE = 26.0 + + +# --------------------------------------------------------------------------- +# Reading the PDF +# --------------------------------------------------------------------------- +def read_pages(pdf_path: str | Path) -> list[PageGeometry]: + """Per-page size in PDF points, index 0 first.""" + reader = PdfReader(str(pdf_path)) + pages = [] + for index, page in enumerate(reader.pages): + box = page.mediabox + pages.append( + PageGeometry( + page=index, + width=float(box.width), + height=float(box.height), + ) + ) + return pages + + +def _text_positions(page) -> list[tuple[str, float, float]]: + """Every text fragment on the page as (text, x, y) in points. + + pypdf hands the text matrix to the visitor, whose last two entries are the + drawing position. Fragments are kept whole rather than merged into lines: + form labels are nearly always drawn in one go, and merging risks gluing a + neighbouring column onto the label. + """ + found: list[tuple[str, float, float]] = [] + + def visit(text, _cm, tm, _font_dict, _font_size): + cleaned = text.strip() + if cleaned: + found.append((cleaned, float(tm[4]), float(tm[5]))) + + try: + page.extract_text(visitor_text=visit) + except Exception as exc: + # A PDF whose content stream will not parse still has usable widgets. + logger.warning("could not read text for label detection: %s", exc) + return found + + +def _nearest_label( + layout: TemplateFieldLayout, texts: list[tuple[str, float, float]] +) -> str | None: + """The text most likely to be this box's label. + + Printed forms put the label to the left of the box, or directly above it. + Left wins when both exist, because a label above is often the column + heading for a whole run of boxes. + """ + top = layout.y + layout.height + + left = [ + (layout.x - x, text) + for text, x, y in texts + if x < layout.x + and layout.x - x <= _LABEL_GAP_LEFT + and layout.y - 2 <= y <= top + 2 + ] + if left: + return min(left)[1] + + above = [ + (y - top, text) + for text, x, y in texts + if top <= y <= top + _LABEL_GAP_ABOVE + and layout.x - 4 <= x <= layout.x + layout.width + ] + if above: + return min(above)[1] + return None + + +def _widgets(pdf_path: str | Path) -> list[tuple[TemplateFieldLayout, str | None, list]]: + """Every form widget in the PDF as (layout, widget name, page texts). + + The page index lives on the layout, so it is not repeated in the tuple. + """ + reader = PdfReader(str(pdf_path)) + out = [] + for index, page in enumerate(reader.pages): + annotations = page.get("/Annots") + if not annotations: + continue + texts = _text_positions(page) + # MediaBox does not have to start at the origin. Subtracting its corner + # keeps every stored coordinate relative to the page the editor draws. + origin_x = float(page.mediabox.left) + origin_y = float(page.mediabox.bottom) + + for annotation in annotations: + try: + obj = annotation.get_object() + except Exception: + continue + if obj.get("/Subtype") != "/Widget": + continue + rect = obj.get("/Rect") + if not rect or len(rect) != 4: + continue + x0, y0, x1, y1 = (float(v) for v in rect) + width, height = abs(x1 - x0) or 1.0, abs(y1 - y0) or 1.0 + layout = TemplateFieldLayout( + page=index, + x=max(min(x0, x1) - origin_x, 0), + y=max(min(y0, y1) - origin_y, 0), + width=width, + height=height, + ) + name = obj.get("/T") + out.append((layout, str(name) if name else None, texts)) + + # Reading order: top of the page down, then left to right. + out.sort(key=lambda item: (item[0].page, -item[0].y, item[0].x)) + return out + + +# --------------------------------------------------------------------------- +# Turning widgets into draft fields +# --------------------------------------------------------------------------- +def suggest_mappings(label: str | None) -> list[MappingSuggestion]: + """Ranked contract paths for a detected label, best first. + + Returns nothing when the label is missing or nothing scores above the + floor. An empty list is the honest answer, and the editor's search box + covers it. + """ + if not label: + return [] + + query = field_catalog.normalize_label(label) + hits = field_catalog.search(query, limit=MAX_MAPPING_SUGGESTIONS) + return [ + MappingSuggestion( + path=entry.path, + label=entry.label, + field_type=entry.field_type, + section=entry.section, + description=entry.description, + score=round(score, 4), + ) + for entry, score in hits + if score is not None and score >= MAPPING_SUGGESTION_FLOOR + ] + + +def _field_name(raw: str | None, used: set[str], position: int) -> str: + """A unique, slug-shaped name for a detected box.""" + base = _NAME_CLEANUP.sub("_", (raw or "").lower()).strip("_") + if not base: + base = f"field_{position}" + name = base + suffix = 2 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +def _apply_suggestion( + field: TemplateField, suggestions: list[MappingSuggestion] +) -> TemplateField: + """Pre-apply the top suggestion when it clears the auto-apply mark.""" + if not suggestions or suggestions[0].score < MAPPING_AUTO_APPLY_SCORE: + return field + + top = suggestions[0] + update = {"source": FieldSource.schema, "incident_mapping": top.path} + + entry = next((e for e in field_catalog.catalog() if e.path == top.path), None) + if entry and entry.enum_values: + update["field_type"] = TemplateFieldType.enum + update["allowed_values"] = list(entry.enum_values) + + return field.model_copy(update=update) + + +def _drafts_from_widgets(widgets: list) -> list[DraftField]: + used: set[str] = set() + drafts: list[DraftField] = [] + + for position, (layout, widget_name, texts) in enumerate(widgets, start=1): + label = _nearest_label(layout, texts) + # A widget's own name is often the best clue a fillable PDF gives + # ("incident_number"), so it stands in when no text sits near the box. + # `detected_label` still reports only what was read off the page. + suggestions = suggest_mappings(label or widget_name) + field = TemplateField( + field_name=_field_name(widget_name or label, used, position), + field_type=TemplateFieldType.string, + # Nothing is assumed about an unmapped box: manual means a person + # types the value, which is always safe to change to something else. + source=FieldSource.manual, + required=False, + layout=layout, + ) + drafts.append( + DraftField( + field=_apply_suggestion(field, suggestions), + detected_label=label, + suggestions=suggestions, + ) + ) + return drafts + + +def build_draft_fields(pdf_path: str | Path) -> list[DraftField]: + """Read a PDF's form widgets and turn each into a draft field.""" + return _drafts_from_widgets(_widgets(pdf_path)) + + +def _pad_with_blank_page(pdf_path: Path) -> Path: + """Write a copy of a one-page PDF with a blank second page appended. + + commonforms cannot read a single-page document. It wraps the detector's + output a second time when the page count is 1 (inference.py), and the + rfdetr versions it now installs with already hand back a list, so the run + dies with "'list' object has no attribute 'with_nms'". One blank page keeps + us off that branch. Detections on the padding are dropped afterwards. + """ + reader = PdfReader(str(pdf_path)) + page = reader.pages[0] + writer = PdfWriter() + writer.add_page(page) + writer.add_blank_page(width=page.mediabox.width, height=page.mediabox.height) + + padded = pdf_path.with_name(f"{pdf_path.stem}_padded.pdf") + with padded.open("wb") as handle: + writer.write(handle) + return padded + + +def detect_fields(pdf_path: str | Path) -> list[DraftField]: + """Run commonforms over a PDF, then draft a field per detected box. + + A PDF that already carries form widgets is used as-is. commonforms only + has to run on flat scans, and it is the slow part. + """ + # Imported here, not at module scope: the Controller pulls in the whole + # detection stack, which is far too heavy for an API process to import. + from app.services.controller import Controller + + pdf_path = Path(pdf_path) + widgets = _widgets(pdf_path) + if widgets: + logger.info("%s already has form widgets, skipping detection", pdf_path) + return _drafts_from_widgets(widgets) + + padded = None + if len(PdfReader(str(pdf_path)).pages) == 1: + padded = _pad_with_blank_page(pdf_path) + logger.info("%s is one page, padding it for commonforms", pdf_path) + + fillable_path = None + try: + fillable_path = Path(Controller().prepare_fillable(str(padded or pdf_path))) + drafts = build_draft_fields(fillable_path) + finally: + # Both files are scratch. The boxes live in the draft from here on, and + # the upload the user later registers points at the original PDF. + if padded: + padded.unlink(missing_ok=True) + if fillable_path: + fillable_path.unlink(missing_ok=True) + + if padded: + # Nothing should land on the blank page, but a stray box there would + # otherwise become a field on a page the real PDF does not have. + drafts = [d for d in drafts if d.field.layout and d.field.layout.page == 0] + return drafts + + +# --------------------------------------------------------------------------- +# The background run +# --------------------------------------------------------------------------- +def _finish_job(session: Session, job_id: str | None, status: str, error: dict | None = None) -> None: + if not job_id: + return + job = get_job_by_uuid(session, job_id) + if not job: + return + job.status = status + if status == "completed": + job.progress_percent = 100 + job.error = error + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + +def run_detection(session: Session, upload_id: UUID, job_id: str | None = None) -> dict: + """Detect an upload's fields and write the draft back. Returns a summary. + + Failure is not exceptional here. The PDF and its page geometry are already + stored, so a detection that falls over still leaves the editor able to draw + every box by hand, and that is what the failed status tells it to do. + """ + upload = get_template_upload(session, upload_id) + if upload is None: + logger.warning("upload %s vanished before detection ran", upload_id) + _finish_job( + session, + job_id, + "failed", + {"error_code": "UPLOAD_NOT_FOUND", "message": "Upload no longer exists"}, + ) + return {"upload_id": str(upload_id), "status": "failed"} + + job = get_job_by_uuid(session, job_id) if job_id else None + if job: + job.status = "processing" + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + now = datetime.now(timezone.utc) + try: + drafts = detect_fields(upload.pdf_path) + except Exception as exc: + logger.exception("field detection failed for upload %s", upload_id) + upload.status = DetectionStatus.failed + upload.detection_error = str(exc) + upload.updated_at = now + update_template_upload(session, upload) + _finish_job( + session, job_id, "failed", {"error_code": "DETECTION_FAILED", "message": str(exc)} + ) + return {"upload_id": str(upload_id), "status": "failed"} + + upload.detected_fields = [draft.model_dump(mode="json") for draft in drafts] + upload.status = DetectionStatus.completed + upload.detection_error = None + upload.updated_at = now + update_template_upload(session, upload) + + if job: + job.result_url = f"/api/v1/templates/pdf/{upload_id}" + update_job(session, job) + _finish_job(session, job_id, "completed") + + return { + "upload_id": str(upload_id), + "status": "completed", + "detected_fields": len(drafts), + } diff --git a/app/tasks/detect_fields.py b/app/tasks/detect_fields.py new file mode 100644 index 00000000..21598a69 --- /dev/null +++ b/app/tasks/detect_fields.py @@ -0,0 +1,29 @@ +"""Celery glue for template field detection. + +The work lives in app/services/template_detection.py. Detection runs here +rather than in the request because commonforms loads a vision model and can +take minutes on a scanned form. The upload row already holds the stored PDF +and its page geometry, so the editor is usable the whole time this runs. +""" + +import logging +from uuid import UUID + +from app.core.celery import celery_app +from app.db.database import get_session + +logger = logging.getLogger(__name__) + + +@celery_app.task(name="detect_template_fields") +def detect_template_fields_task(upload_id_str: str, job_id_str: str | None = None) -> dict: + """Detect the fields of one uploaded template PDF.""" + # Imported inside the task so the API process never pulls the detection + # stack in just by importing this module. + from app.services.template_detection import run_detection + + session = next(get_session()) + try: + return run_detection(session, UUID(upload_id_str), job_id_str) + finally: + session.close() diff --git a/app/tasks/extract.py b/app/tasks/extract.py new file mode 100644 index 00000000..fa8690c5 --- /dev/null +++ b/app/tasks/extract.py @@ -0,0 +1,29 @@ +"""Celery glue for the extraction worker. + +All the work lives in app/services/extraction/worker.py. This is only the +broker entry point: open a session, run the extraction, close the session. +""" + +import logging +from uuid import UUID + +from app.core.celery import celery_app +from app.db.database import get_session +from app.services.extraction.worker import run_extraction + +logger = logging.getLogger(__name__) + + +@celery_app.task(name="extract_incident") +def extract_task( + extract_id_str: str, + job_id_str: str, + defaults: dict | None = None, + hints: dict | None = None, +) -> dict: + """Run the chunked extraction for one queued extraction row.""" + session = next(get_session()) + try: + return run_extraction(session, UUID(extract_id_str), job_id_str, defaults, hints) + finally: + session.close() diff --git a/app/tasks/generate_forms.py b/app/tasks/generate_forms.py new file mode 100644 index 00000000..a74b0138 --- /dev/null +++ b/app/tasks/generate_forms.py @@ -0,0 +1,25 @@ +"""Celery glue for the batch form-fill worker. + +All the work lives in app/services/form_fill_worker.py. This is only the +broker entry point: open a session, run the batch, close the session. Mirrors +app/tasks/extract.py's split — not app/tasks/fill.py, the legacy prototype task. +""" + +import logging +from uuid import UUID + +from app.core.celery import celery_app +from app.db.database import get_session +from app.services.form_fill_worker import run_batch_fill + +logger = logging.getLogger(__name__) + + +@celery_app.task(name="generate_forms_batch") +def generate_forms_batch_task(batch_id_str: str, job_id_str: str) -> dict: + """Fill every queued form in one batch.""" + session = next(get_session()) + try: + return run_batch_fill(session, UUID(batch_id_str), job_id_str) + finally: + session.close() diff --git a/contracts/openapi.yaml b/contracts/openapi.yaml index 16931f98..6fe241ad 100644 --- a/contracts/openapi.yaml +++ b/contracts/openapi.yaml @@ -18,7 +18,7 @@ tags: - name: input description: Submit voice or text incident narratives - name: extraction - description: AI-powered data extraction from narratives into canonical JSON + description: AI-powered data extraction from narratives into the incident contract - name: forms description: Generate agency-specific PDF forms from extracted data - name: incidents @@ -50,14 +50,14 @@ paths: $ref: "path/extraction.yaml#/extract_by_input" /api/v1/extract/{extract_id}: $ref: "path/extraction.yaml#/extract_by_id" + /api/v1/extract/{extract_id}/readiness: + $ref: "path/extraction.yaml#/readiness" /api/v1/extract/{extract_id}/validate: $ref: "path/extraction.yaml#/validate" # ── Layer 3: Form Generation ─────────────────────────────────── - /api/v1/forms/generate/all: - $ref: "path/forms.yaml#/generate_all" - /api/v1/forms/generate/{form_type}: - $ref: "path/forms.yaml#/generate_single" + /api/v1/forms/generate: + $ref: "path/forms.yaml#/generate" /api/v1/forms/{form_id}: $ref: "path/forms.yaml#/form_by_id" /api/v1/forms/{form_id}/pdf: @@ -66,6 +66,8 @@ paths: $ref: "path/forms.yaml#/form_json" /api/v1/forms/batch/{batch_id}: $ref: "path/forms.yaml#/batch_by_id" + /api/v1/forms/batch/{batch_id}/download: + $ref: "path/forms.yaml#/batch_download" # ── Layer 4: Incident Management ─────────────────────────────── /api/v1/incidents: @@ -88,8 +90,12 @@ paths: $ref: "path/templates.yaml#/template_by_id" /api/v1/templates/{template_id}/fields: $ref: "path/templates.yaml#/template_fields" + /api/v1/templates/{template_id}/pdf: + $ref: "path/templates.yaml#/template_source_pdf" /api/v1/templates/pdf: $ref: "path/templates.yaml#/templates_pdf" + /api/v1/templates/pdf/{upload_id}: + $ref: "path/templates.yaml#/template_pdf_draft" # ── Layer 7: System ──────────────────────────────────────────── /api/v1/health: @@ -98,6 +104,8 @@ paths: $ref: "path/system.yaml#/schema_incident" /api/v1/schema/incident/versions: $ref: "path/system.yaml#/schema_versions" + /api/v1/schema/fields: + $ref: "path/system.yaml#/schema_fields" # ── Layer 8: Async Jobs ──────────────────────────────────────── /api/v1/jobs/{job_id}: diff --git a/contracts/path/extraction.yaml b/contracts/path/extraction.yaml index 865e20c4..e6cdc040 100644 --- a/contracts/path/extraction.yaml +++ b/contracts/path/extraction.yaml @@ -2,6 +2,7 @@ # POST /api/v1/extract/{input_id} # GET /api/v1/extract/{extract_id} # PATCH /api/v1/extract/{extract_id} +# GET /api/v1/extract/{extract_id}/readiness # POST /api/v1/extract/{extract_id}/validate extract_by_input: @@ -9,10 +10,36 @@ extract_by_input: operationId: createExtraction summary: Start AI extraction from input narrative description: | - Sends the narrative (from a previously submitted input) to the local Ollama - LLM with a structured prompt to extract all incident fields into the canonical - FireForm JSON schema. This is an asynchronous operation the LLM may take - 30–120 seconds. Returns an extract_id and job_id for polling. + Extracts the narrative (from a previously submitted input) into the + incident contract. The contract is far too large for one LLM + call, so the extractor splits it into small field groups and runs them + as parallel calls against the configured provider (bounded by + LLM_MAX_PARALLEL and available RAM), validates each group against + the schema with per-group retry, and stitches the validated pieces + into one document. + + A provider that rate limits the run is retried, and if the limit does not + lift the extraction is recorded as failed with error_type + LLM_RATE_LIMITED. Everything the earlier waves already produced is kept + in partial_result rather than thrown away. + + Open fields from registered templates (source=open) are extracted in + the same run as extra groups, so their values reach the review screen + together with everything else; they are stored under the contract's + custom_fields keyed "{form_type}.{field_name}". + + Deterministic context is applied without the LLM where possible: + relative dates resolve against the configured timezone, and country + and currency defaults come from the request or server config. + + The moment extraction completes, the server creates a draft incident + row holding the stitched contract document. That row is the single + store of incident data: review corrections write into it, and form + generation reads from it by incident_id. The extraction row itself + keeps only job metadata and the corrections audit trail. + + This is an asynchronous operation and may take 30-120 seconds. + Returns an extract_id and job_id for polling. tags: - extraction parameters: @@ -34,6 +61,10 @@ extract_by_input: extraction_hints: incident_type: "wildland_fire" state: "CA" + defaults: + country: "US" + timezone: "America/Los_Angeles" + currency: "USD" responses: "202": description: Extraction job queued successfully @@ -81,26 +112,27 @@ extract_by_input: detail: existing_extract_id: "550e8400-e29b-41d4-a716-446655440020" "503": - description: Ollama LLM service unavailable + description: The configured LLM provider is unavailable content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: error_code: "LLM_UNAVAILABLE" - message: "Ollama LLM service is not available" + message: "The Ollama LLM service is not available" retry_after_seconds: 30 detail: - ollama_status: "connection_refused" + provider: "ollama" + reason: "connection refused" extract_by_id: get: operationId: getExtraction summary: Get extraction result by ID description: | - Returns the full canonical FireForm JSON when extraction is complete, or + Returns the full incident contract when extraction is complete, or the current job status while still processing. When status is "completed", - the response body contains the entire canonical incident schema. When still + the response body contains the entire incident contract. When still processing, includes a retry_after_seconds hint for polling. tags: - extraction @@ -123,10 +155,11 @@ extract_by_id: - $ref: "../schemas/extraction-record.yaml#/ExtractionProcessing" examples: completed: - summary: Extraction completed with canonical JSON + summary: Extraction completed with the incident contract value: extract_id: "550e8400-e29b-41d4-a716-446655440020" input_id: "550e8400-e29b-41d4-a716-446655440001" + incident_id: "550e8400-e29b-41d4-a716-446655440050" status: "completed" completed_at: "2024-07-15T14:31:05Z" incident_contract: @@ -159,10 +192,20 @@ extract_by_id: operationId: updateExtraction summary: Manually correct extracted fields description: | - Allows a responder to correct any field in the canonical JSON after LLM - extraction. Uses JSON Merge Patch (RFC 7396) only send the fields that - changed. The server records an audit trail of all changes vs the original - LLM output and recalculates completeness scores and applicable_forms. + The review-screen write path. Lets a responder correct any field in the + contract after extraction: fix a wrong value, add a missing one, or + remove a hallucinated one (RFC 7396: sending null deletes the field). + Manual template fields are entered the same way, and open-field values + corrected the same way, both under custom_fields keyed + "{form_type}.{field_name}". Only send the fields that changed. + + Corrections are applied to the contract document on the linked + incident row (the single store of incident data), so form generation + and analytics always see the reviewed values with no copy to sync. + The server records an audit trail of all changes vs the original LLM + output on the extraction and recalculates completeness, analytics + columns and readiness, so a corrected field immediately flips + dependent templates to ready in the readiness matrix. tags: - extraction parameters: @@ -181,13 +224,15 @@ extract_by_id: schema: $ref: "../schemas/incident-contract.yaml#/IncidentContract" example: - fire: - estimated_damage_usd: 250000 + losses: + property_loss: + amount: 250000 + currency: "USD" casualties: total_responder_injuries: 2 responses: "200": - description: Extraction updated returns full updated canonical JSON + description: Extraction updated returns the full updated incident contract content: application/json: schema: @@ -199,17 +244,27 @@ extract_by_id: schema: $ref: "../schemas/common.yaml#/ErrorResponse" "409": - description: Extraction is locked (already submitted) + description: Conflict extraction locked or not yet completed content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "EXTRACT_LOCKED" - message: "Cannot modify extraction incident report has been submitted" - detail: - report_status: "submitted" - submitted_at: "2024-07-15T18:00:00Z" + examples: + locked: + summary: Report already submitted + value: + error_code: "EXTRACT_LOCKED" + message: "Cannot modify extraction incident report has been submitted" + detail: + report_status: "submitted" + submitted_at: "2024-07-15T18:00:00Z" + not_completed: + summary: Extraction still running, no document to correct yet + value: + error_code: "EXTRACT_NOT_COMPLETED" + message: "Extraction is in 'processing' state. Wait until status is 'completed'." + detail: + current_status: "processing" "422": description: Invalid field path or value content: @@ -224,15 +279,90 @@ extract_by_id: issue: "Must be one of: confirmed, probable, suspected, undetermined" value: "maybe" +readiness: + get: + operationId: getExtractionReadiness + summary: Fill readiness of every registered template for this extraction + description: | + The form-selection matrix. Compares the extracted contract against the + field list of every active template and reports, per template, whether + it can be generated right now and which fields block it. Pure lookup + over stored data, no LLM, so it is cheap to refetch after every correction. + + The frontend renders ready templates green and selectable, the rest + greyed out; hovering or clicking a grey one lists its gaps from + missing_required. Typing a manual value or correcting the contract via + PATCH and refetching flips templates to ready. + tags: + - extraction + parameters: + - name: extract_id + in: path + required: true + description: Unique identifier of the extraction + schema: + type: string + format: uuid + responses: + "200": + description: Readiness of every active template + content: + application/json: + schema: + $ref: "../schemas/extraction-record.yaml#/ReadinessMatrix" + example: + extract_id: "550e8400-e29b-41d4-a716-446655440020" + computed_at: "2026-07-15T14:35:00Z" + templates: + - template_id: "550e8400-e29b-41d4-a716-446655440070" + form_type: "neris" + display_name: "NERIS Incident Report" + ready: true + missing_required: [] + missing_recommended: + - field_name: "smoke_alarm_presence" + source: "schema" + incident_mapping: "risk_reduction.smoke_alarm.presence" + field_coverage_percent: 94 + - template_id: "550e8400-e29b-41d4-a716-446655440073" + form_type: "state_texas" + display_name: "Texas State Fire Marshal Incident Report" + ready: false + missing_required: + - field_name: "marshal_signature_name" + source: "manual" + incident_mapping: "custom_fields.state_texas.marshal_signature_name" + description: "Reviewing marshal's printed name, entered per incident" + - field_name: "fire_cause" + source: "schema" + incident_mapping: "fire.cause_category" + missing_recommended: [] + field_coverage_percent: 78 + "404": + description: Extraction not found + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + "409": + description: Extraction not yet completed + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + example: + error_code: "EXTRACT_NOT_COMPLETED" + message: "Extraction is still processing. Wait until status is 'completed'." + validate: post: operationId: validateExtraction - summary: Validate extraction against a form's requirements + summary: Validate extraction against one template's requirements description: | - Validates the canonical JSON against a specific form type's field requirements. - Returns whether the extraction has all required fields, which recommended - fields are missing, and any warnings. Useful for checking "can I generate - a NERIS report with what I have?" before triggering form generation. + Single-template version of the readiness matrix: checks the contract + against one registered template and returns the gaps in detail. Useful + for re-checking just the template the user is looking at instead of + recomputing the whole matrix. tags: - extraction parameters: @@ -250,12 +380,13 @@ validate: schema: type: object required: - - form_type + - template_id properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" + template_id: + type: string + format: uuid example: - form_type: "neris" + template_id: "550e8400-e29b-41d4-a716-446655440070" responses: "200": description: Validation result @@ -265,49 +396,35 @@ validate: $ref: "../schemas/extraction-record.yaml#/ValidationResult" example: valid: true + template_id: "550e8400-e29b-41d4-a716-446655440070" form_type: "neris" extract_id: "550e8400-e29b-41d4-a716-446655440020" missing_required: [] missing_recommended: - - "fire.detector_present" - - "fire.detector_operated" + - field_name: "smoke_alarm_presence" + source: "schema" + incident_mapping: "risk_reduction.smoke_alarm.presence" + - field_name: "smoke_alarm_operation" + source: "schema" + incident_mapping: "risk_reduction.smoke_alarm.operation" warnings: - - "fire.estimated_damage_usd is null NERIS recommends providing damage estimates" + - "losses.property_loss is null. NERIS recommends providing damage estimates" field_coverage_percent: 94 "404": - description: Extraction not found + description: Extraction or template not found content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" - "422": - description: Unknown form type + example: + error_code: "TEMPLATE_NOT_FOUND" + message: "Template with ID 550e8400-e29b-41d4-a716-446655440099 not found" + "409": + description: Extraction not yet completed content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: - error_code: "UNKNOWN_FORM_TYPE" - message: "Form type 'xyz' is not recognized" - detail: - valid_form_types: - - neris - - nemsis_epcr - - nibrs - - nfirs_basic - - nfirs_fire - - nfirs_structure - - nfirs_wildland - - nfirs_ems - - nfirs_hazmat - - nfirs_apparatus - - nfirs_personnel - - nfirs_arson - - nfirs_casualty_civilian - - nfirs_casualty_responder - - cal_fire_ics209 - - osha_301 - - un_ssirs - - state_georgia - - state_california - - state_new_york + error_code: "EXTRACT_NOT_COMPLETED" + message: "Extraction is still processing. Wait until status is 'completed'." diff --git a/contracts/path/forms.yaml b/contracts/path/forms.yaml index b920b41a..a44dbd4a 100644 --- a/contracts/path/forms.yaml +++ b/contracts/path/forms.yaml @@ -1,20 +1,29 @@ # Layer 3 Form Generation Endpoints -# POST /api/v1/forms/generate/all -# POST /api/v1/forms/generate/{form_type} +# POST /api/v1/forms/generate # GET /api/v1/forms/{form_id} # GET /api/v1/forms/{form_id}/pdf # GET /api/v1/forms/{form_id}/json # GET /api/v1/forms/batch/{batch_id} +# GET /api/v1/forms/batch/{batch_id}/download -generate_all: +generate: post: - operationId: generateAllForms - summary: Generate all applicable forms from an extraction + operationId: generateForms + summary: Generate forms for the selected templates description: | - Triggers batch generation of ALL forms listed in the extraction's - extraction_metadata.applicable_forms. This is an async batch job. - Use skip_incomplete to skip forms that fail validation, or force_partial - to generate forms with blank fields where data is missing. + Generates one form per requested template from the incident's contract + document, the single store of incident data, referenced by incident_id + (returned by the extraction endpoints once extraction completes). Send + the template_ids the user selected on the readiness screen, or omit + template_ids to generate everything the readiness matrix reports as + ready. Filling is pure lookup and draw: schema fields read their + contract path, static fields stamp their saved text, manual and open + fields read custom_fields. No LLM runs here, so a whole batch finishes + in seconds. + + Templates that are not ready are skipped with a reason unless + force_partial is set, in which case they generate with blanks. + Always returns a batch, even for a single template. tags: - forms requestBody: @@ -22,11 +31,14 @@ generate_all: content: application/json: schema: - $ref: "../schemas/form-record.yaml#/GenerateAllRequest" + $ref: "../schemas/form-record.yaml#/GenerateFormsRequest" example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" + incident_id: "550e8400-e29b-41d4-a716-446655440050" + template_ids: + - "550e8400-e29b-41d4-a716-446655440070" + - "550e8400-e29b-41d4-a716-446655440072" options: - skip_incomplete: true + output_format: "both" force_partial: false responses: "202": @@ -38,118 +50,46 @@ generate_all: example: batch_id: "550e8400-e29b-41d4-a716-446655440030" status: "processing" - extract_id: "550e8400-e29b-41d4-a716-446655440020" + incident_id: "550e8400-e29b-41d4-a716-446655440050" forms_queued: - - "neris" - - "nfirs_basic" - - "nfirs_wildland" + - form_id: "550e8400-e29b-41d4-a716-446655440040" + template_id: "550e8400-e29b-41d4-a716-446655440070" + form_type: "neris" + - form_id: "550e8400-e29b-41d4-a716-446655440041" + template_id: "550e8400-e29b-41d4-a716-446655440072" + form_type: "nfirs_basic" forms_skipped: - - form_type: "nemsis_epcr" - reason: "Missing required fields: ems.patients[0].date_of_birth" - estimated_seconds: 45 + - template_id: "550e8400-e29b-41d4-a716-446655440073" + form_type: "state_texas" + reason: "Not ready: marshal_signature_name (manual) has no value" + estimated_seconds: 10 poll_url: "/api/v1/forms/batch/550e8400-e29b-41d4-a716-446655440030" "404": - description: Extract ID not found + description: Incident ID or a template ID not found. The incident row + exists as soon as extraction completes, so a missing incident means + extraction has not finished (or the id is wrong). content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" - "409": - description: Extraction not yet completed - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "EXTRACT_NOT_COMPLETED" - message: "Extraction is still processing. Wait until status is 'completed'." - detail: - current_status: "processing" "422": - description: No applicable forms found + description: Nothing to generate content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: - error_code: "NO_APPLICABLE_FORMS" - message: "No forms are applicable for this extraction's incident type" - -generate_single: - post: - operationId: generateSingleForm - summary: Generate one specific form type - description: | - Generates a single agency-specific form from the canonical extraction data. - The form_type path parameter specifies which form to generate. If the extraction - is missing required fields for this form, returns 422 unless force_partial is true. - tags: - - forms - parameters: - - name: form_type - in: path - required: true - description: Type of form to generate - schema: - $ref: "../schemas/enums.yaml#/FormType" - requestBody: - required: true - content: - application/json: - schema: - $ref: "../schemas/form-record.yaml#/GenerateSingleRequest" - example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - options: - output_format: "pdf" - force_partial: false - responses: - "202": - description: Form generation job accepted - content: - application/json: - schema: - $ref: "../schemas/form-record.yaml#/FormGenerateResponse" - example: - form_id: "550e8400-e29b-41d4-a716-446655440040" - form_type: "neris" - status: "processing" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - job_id: "550e8400-e29b-41d4-a716-446655440097" - estimated_seconds: 15 - poll_url: "/api/v1/forms/550e8400-e29b-41d4-a716-446655440040" - "404": - description: Extract ID not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "422": - description: Validation failure or form not in applicable list - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "FORM_VALIDATION_FAILED" - message: "Extraction is missing required fields for NERIS form" - detail: - form_type: "neris" - missing_fields: - - "incident.types[0].neris_code" - - "location.coordinates" - validation_errors: - - field: "incident.types[0].neris_code" - issue: "Required field is null" + error_code: "NO_FORMS_TO_GENERATE" + message: "No templates were selected and none are ready" form_by_id: get: operationId: getForm summary: Get form metadata and status description: | - Returns the form record including generation status, associated extract and - incident IDs, and a field_mapping_summary showing how canonical fields were - mapped to the form's agency-specific fields (for audit/transparency). + Returns the form record including generation status, the incident it + was filled from, and a field_mapping_summary showing how incident-contract + fields were mapped to the form's agency-specific fields (for audit/transparency). tags: - forms parameters: @@ -169,10 +109,11 @@ form_by_id: $ref: "../schemas/form-record.yaml#/FormRecord" example: form_id: "550e8400-e29b-41d4-a716-446655440040" + template_id: "550e8400-e29b-41d4-a716-446655440070" form_type: "neris" status: "completed" - extract_id: "550e8400-e29b-41d4-a716-446655440020" incident_id: "550e8400-e29b-41d4-a716-446655440050" + batch_id: "550e8400-e29b-41d4-a716-446655440030" created_at: "2024-07-15T14:32:00Z" completed_at: "2024-07-15T14:32:12Z" pdf_ready: true @@ -196,7 +137,9 @@ form_pdf: description: | Returns the filled PDF binary file for the specified form. If the form generation is still in progress, returns 202 Accepted with a retry_after - hint. The Content-Disposition header is set for browser download. + hint. The Content-Disposition header is set for browser download, naming + the file "{form_type}_{incident_number}.pdf", the same name the batch zip + uses for its entries. Incidents without a number fall back to the form id. tags: - forms parameters: @@ -262,7 +205,7 @@ form_json: summary: Get form-specific field-mapped JSON description: | Returns the form-specific JSON with fields mapped to the agency's expected - format not the canonical FireForm schema, but the actual field names and + format not the incident contract, but the actual field names and structure that the target agency system expects. Useful for future direct API submission to agency systems. tags: @@ -286,19 +229,48 @@ form_json: form_type: "neris" form_version: "2.0" form_id: "550e8400-e29b-41d4-a716-446655440040" - extract_id: "550e8400-e29b-41d4-a716-446655440020" + template_id: "550e8400-e29b-41d4-a716-446655440070" + incident_id: "550e8400-e29b-41d4-a716-446655440050" agency_fields: incident_type: "wildland-fire" incident_date: "2024-07-10" alarm_time: "13:52" acres_burned: 1247 cause: "natural" + "202": + description: Form generation still in progress + content: + application/json: + schema: + type: object + properties: + message: + type: string + status: + type: string + retry_after_seconds: + type: integer + example: + message: "Form generation is still in progress" + status: "processing" + retry_after_seconds: 5 "404": description: Form not found content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" + "500": + description: Form generation failed, so there is no JSON to return + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + example: + error_code: "FORM_GENERATION_FAILED" + message: "Failed to generate form" + detail: + reason: "Template file corrupted or missing" batch_by_id: get: @@ -306,7 +278,7 @@ batch_by_id: summary: Get batch form generation status description: | Returns the status of a batch form generation job including progress - for each individual form. Poll this endpoint after POST /forms/generate/all. + for each individual form. Poll this endpoint after POST /forms/generate. tags: - forms parameters: @@ -332,17 +304,89 @@ batch_by_id: failed: 0 forms: - form_id: "550e8400-e29b-41d4-a716-446655440040" + template_id: "550e8400-e29b-41d4-a716-446655440070" form_type: "neris" status: "completed" - form_id: "550e8400-e29b-41d4-a716-446655440041" + template_id: "550e8400-e29b-41d4-a716-446655440072" form_type: "nfirs_basic" status: "completed" - form_id: "550e8400-e29b-41d4-a716-446655440042" + template_id: "550e8400-e29b-41d4-a716-446655440074" form_type: "nfirs_wildland" status: "completed" + download_url: "/api/v1/forms/batch/550e8400-e29b-41d4-a716-446655440030/download" + "404": + description: Batch job not found + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + +batch_download: + get: + operationId: downloadBatchZip + summary: Download all PDFs of a batch as one zip + description: | + Returns a zip archive of every completed PDF in the batch, named + "{form_type}_{incident_number}.pdf" inside the archive. Available once + the batch status is completed; while forms are still generating, + returns 202 with a retry hint. + tags: + - forms + parameters: + - name: batch_id + in: path + required: true + description: Unique identifier of the batch job + schema: + type: string + format: uuid + responses: + "200": + description: Zip archive of the batch's PDFs + content: + application/zip: + schema: + type: string + format: binary + headers: + Content-Disposition: + description: Attachment filename for download + schema: + type: string + example: 'attachment; filename="fireform_batch_FF-2024-CA-0157.zip"' + "202": + description: Batch still generating + content: + application/json: + schema: + type: object + properties: + message: + type: string + status: + type: string + retry_after_seconds: + type: integer + example: + message: "Batch is still generating" + status: "processing" + retry_after_seconds: 5 "404": description: Batch job not found content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" + "500": + description: Every form in the batch failed, so there is nothing to bundle + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + example: + error_code: "FORM_GENERATION_FAILED" + message: "Every form in the batch failed to generate" + detail: + reason: "No PDFs were produced for this batch" diff --git a/contracts/path/incidents.yaml b/contracts/path/incidents.yaml index 810a311a..a050d8d8 100644 --- a/contracts/path/incidents.yaml +++ b/contracts/path/incidents.yaml @@ -8,11 +8,14 @@ incidents: post: operationId: createIncident - summary: Create a full incident record + summary: Finalize the incident record for an extraction description: | - Creates a permanent incident record that links input, extraction, and - generated forms into a single coherent unit. Assigns a permanent incident_id - and stores the complete incident for future retrieval and reporting. + A draft incident row is created automatically the moment extraction + completes; it owns the contract document (the single store of incident + data) and its incident_id is returned by the extraction endpoints. + This endpoint finalizes that draft: assigns the department's incident + number and tags, and promotes the record for retrieval and reporting. + It never creates a second row for the same extraction. tags: - incidents requestBody: @@ -40,6 +43,7 @@ incidents: extract_id: "550e8400-e29b-41d4-a716-446655440020" incident_number: "CA-SQF-2024-0421" status: "draft" + incident_datetime: "2024-07-10T13:52:00-07:00" forms_generated: - form_id: "550e8400-e29b-41d4-a716-446655440040" form_type: "neris" @@ -79,19 +83,21 @@ incidents: parameters: - name: date_from in: query - description: Start date filter (inclusive, ISO 8601) + description: Start date filter (inclusive, ISO 8601), applied to + incident_datetime schema: type: string format: date - name: date_to in: query - description: End date filter (inclusive, ISO 8601) + description: End date filter (inclusive, ISO 8601), applied to + incident_datetime schema: type: string format: date - - name: incident_type + - name: incident_category in: query - description: Filter by incident category + description: Filter by the incident's primary category schema: $ref: "../schemas/enums.yaml#/IncidentCategory" - name: status @@ -136,8 +142,9 @@ incidents: incident_number: "CA-SQF-2024-0421" status: "draft" incident_name: "Bear Creek Wildfire" - incident_type: "fire" - incident_date: "2024-07-10" + incident_type: "wildland_fire" + incident_category: "fire" + incident_datetime: "2024-07-10T13:52:00-07:00" forms_count: 3 created_at: "2024-07-15T14:35:00Z" pagination: @@ -166,7 +173,7 @@ incident_by_id: operationId: getIncident summary: Get full incident record description: | - Returns the complete incident record including the linked canonical + Returns the complete incident record including the linked incident extraction, all generated forms and their statuses, submission log, and audit trail. tags: diff --git a/contracts/path/jobs.yaml b/contracts/path/jobs.yaml index 2ac042f6..1ea0858d 100644 --- a/contracts/path/jobs.yaml +++ b/contracts/path/jobs.yaml @@ -6,7 +6,7 @@ job_by_id: operationId: getJobStatus summary: Get async job status description: | - Universal job status endpoint for any asynchronous operation in FireForm — + Universal job status endpoint for any asynchronous operation in FireForm transcription, LLM extraction, form generation, and report generation. Returns the current status, progress percentage, and a result URL when the job completes. Poll this endpoint for long-running operations. diff --git a/contracts/path/system.yaml b/contracts/path/system.yaml index cd00901c..e6a31c2f 100644 --- a/contracts/path/system.yaml +++ b/contracts/path/system.yaml @@ -2,13 +2,14 @@ # GET /api/v1/health # GET /api/v1/schema/incident # GET /api/v1/schema/incident/versions +# GET /api/v1/schema/fields health: get: operationId: getHealth summary: System health check description: | - Returns the health status of all FireForm components — database, Ollama LLM + Returns the health status of all FireForm components - database, Ollama LLM (including loaded models, GPU status, and current load), Whisper transcription, and file storage. Returns 200 even when degraded (so load balancers don't kill it), 503 only if the system is truly unhealthy and cannot serve any requests. @@ -32,23 +33,16 @@ health: database: status: "healthy" response_time_ms: 2 - ollama: + llm: status: "healthy" response_time_ms: 15 - model_loaded: "llama3:8b" - ollama_version: "0.3.0" + provider: "ollama" + model: "llama3:8b" + external: false + probed: true models_available: - - name: "llama3:8b" - size_gb: 4.7 - quantization: "Q4_K_M" - loaded: true - - name: "mistral:7b" - size_gb: 4.1 - quantization: "Q4_0" - loaded: false - current_load: - active_requests: 1 - queued_requests: 0 + - "llama3:8b" + - "mistral:7b" whisper: status: "healthy" response_time_ms: 10 @@ -65,13 +59,14 @@ health: database: status: "healthy" response_time_ms: 2 - ollama: + llm: status: "degraded" response_time_ms: 5000 - detail: "High latency model may be loading" - current_load: - active_requests: 3 - queued_requests: 2 + provider: "ollama" + model: "llama3:8b" + external: false + probed: true + detail: "High latency, the model may still be loading" whisper: status: "unhealthy" detail: "Whisper model not loaded" @@ -92,16 +87,20 @@ health: database: status: "unhealthy" detail: "Connection refused" - ollama: + llm: status: "unhealthy" + provider: "ollama" + model: "llama3:8b" + external: false + probed: true detail: "Connection refused" schema_incident: get: operationId: getIncidentSchema - summary: Get the canonical FireForm JSON Schema + summary: Get the incident contract as a JSON Schema description: | - Returns the full canonical FireForm incident JSON Schema (JSON Schema + Returns the full incident contract as a JSON Schema document (JSON Schema draft-07). Clients can use this to validate incident data locally before submitting. This is a schema-as-API pattern for interoperability. tags: @@ -113,21 +112,99 @@ schema_incident: application/json: schema: type: object - description: JSON Schema draft-07 document for the canonical incident model + description: JSON Schema draft-07 document for the incident contract example: $schema: "http://json-schema.org/draft-07/schema#" - title: "FireForm Canonical Incident" + title: "FireForm Incident Contract" type: "object" properties: schema_version: type: "string" +schema_fields: + get: + operationId: searchSchemaFields + summary: Search or list the incident-contract field catalog + description: | + Returns the flattened, searchable catalog of every leaf field in the + incident contract: dotted path, type, section, description, enum values, + aliases and PII flag. The catalog is generated from the contract itself, + so it always matches the running schema version. + + The catalog (paths, types, descriptions, aliases, PII flags) is built + by flattening incident-contract.yaml at startup; aliases come from each + field's x-aliases entry in that file. The contract is the single source + of truth, nothing is duplicated in code, and a schema upgrade updates + search and suggestions automatically. + + Without `q` the full catalog is returned; the editor can cache it for + the whole session since it only changes with a schema upgrade and + filter locally as the user types. With `q` the server ranks matches + with plain fuzzy scoring. No LLM is involved, results are deterministic + and instant. Ranking order, best first: exact field-name match, exact + alias match, field-name prefix, alias prefix, token-based fuzzy match + on name and aliases, word match in the description. Ties break toward + the shorter path. Short queries stay usable because name and alias hits + always outrank description-only hits. + + This powers two things: the type-ahead mapping picker in the template + editor, and the automatic mapping suggestions produced after commonforms + field detection (same index, same scorer). + tags: + - system + parameters: + - name: q + in: query + required: false + description: Search text. Matches field names, aliases and descriptions. + schema: + type: string + - name: section + in: query + required: false + description: Restrict results to one top-level contract section + schema: + type: string + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + maximum: 100 + responses: + "200": + description: Ranked matches, or the full catalog when q is omitted + content: + application/json: + schema: + $ref: "../schemas/template-record.yaml#/SchemaFieldSearchResponse" + example: + query: "zip" + total: 2 + schema_version: "1.1.0" + fields: + - path: "location.postal_code" + label: "Postal code" + field_type: "string" + section: "location" + aliases: ["zip", "zipcode", "pincode"] + pii: false + score: 0.97 + - path: "persons_involved[].address" + label: "Person address" + field_type: "string" + section: "persons_involved" + description: "Owner, occupant or other party's address" + pii: true + score: 0.41 + schema_versions: get: operationId: getSchemaVersions summary: Get schema version history description: | - Returns the version history of the canonical FireForm incident schema, + Returns the version history of the incident contract, including changelogs and whether each version introduced breaking changes. Useful for clients tracking schema evolution across FireForm updates. tags: @@ -148,5 +225,5 @@ schema_versions: breaking_changes: true - version: "1.0.0" released_at: "2024-01-01T00:00:00Z" - changelog: "Initial canonical schema with NFIRS support." + changelog: "Initial incident contract with NFIRS support." breaking_changes: false diff --git a/contracts/path/templates.yaml b/contracts/path/templates.yaml index c2621cac..da433d68 100644 --- a/contracts/path/templates.yaml +++ b/contracts/path/templates.yaml @@ -4,7 +4,9 @@ # POST /api/v1/templates # PUT /api/v1/templates/{template_id} # GET /api/v1/templates/{template_id}/fields +# GET /api/v1/templates/{template_id}/pdf # POST /api/v1/templates/pdf +# GET /api/v1/templates/pdf/{upload_id} templates: get: @@ -25,7 +27,7 @@ templates: schema: type: array items: - $ref: "../schemas/template.yaml#/TemplateSummary" + $ref: "../schemas/template-record.yaml#/TemplateSummary" example: - template_id: "550e8400-e29b-41d4-a716-446655440070" form_type: "neris" @@ -61,8 +63,11 @@ templates: description: | Registers a new form template for a jurisdiction or agency not yet supported. This is how FireForm extends to new states, countries, or custom agency forms - without code changes. The template defines all fields, their types, validation - rules, and how each maps from the FireForm incident schema. + without code changes. The template defines all fields, their placement on the + uploaded PDF, and where each value comes from (see FieldSource): a contract + lookup, a constant, a per-incident manual entry, or an open field the LLM + extracts. Typically the fields array starts from the detected_fields draft + returned by the PDF upload and carries the user's edits. tags: - templates requestBody: @@ -70,16 +75,17 @@ templates: content: application/json: schema: - $ref: "../schemas/template.yaml#/CreateTemplateRequest" + $ref: "../schemas/template-record.yaml#/CreateTemplateRequest" example: form_type: "state_texas" display_name: "Texas State Fire Marshal Incident Report" jurisdiction: "US-TX" agency_type: "fire_department" - pdf_template_ref: "templates/state_texas.pdf" + pdf_template_ref: "templates/uploads/550e8400-e29b-41d4-a716-446655440080.pdf" fields: - field_name: "incident_number" field_type: "string" + source: "schema" required: true max_length: 20 description: "State-assigned incident number" @@ -96,6 +102,7 @@ templates: align: "left" - field_name: "fire_cause" field_type: "enum" + source: "schema" required: true allowed_values: - "accidental" @@ -109,11 +116,33 @@ templates: y: 560.0 width: 200.0 height: 18.0 + - field_name: "insurance_company" + field_type: "string" + source: "open" + required: false + description: "Name of the insurance company covering the property" + layout: + page: 0 + x: 188.33 + y: 520.0 + width: 200.0 + height: 18.0 + - field_name: "marshal_signature_name" + field_type: "string" + source: "manual" + required: true + description: "Reviewing marshal's printed name, entered per incident" + layout: + page: 1 + x: 100.0 + y: 120.0 + width: 180.0 + height: 18.0 - field_name: "report_footer" field_type: "string" + source: "static" required: false static_text: "Generated by FireForm" - incident_mapping: null layout: page: 0 x: 72.0 @@ -128,7 +157,7 @@ templates: content: application/json: schema: - $ref: "../schemas/template.yaml#/Template" + $ref: "../schemas/template-record.yaml#/Template" "409": description: Template with this form_type already exists content: @@ -166,7 +195,7 @@ template_by_id: content: application/json: schema: - $ref: "../schemas/template.yaml#/Template" + $ref: "../schemas/template-record.yaml#/Template" "404": description: Template not found content: @@ -196,14 +225,14 @@ template_by_id: content: application/json: schema: - $ref: "../schemas/template.yaml#/CreateTemplateRequest" + $ref: "../schemas/template-record.yaml#/CreateTemplateRequest" responses: "200": description: Template updated content: application/json: schema: - $ref: "../schemas/template.yaml#/Template" + $ref: "../schemas/template-record.yaml#/Template" "404": description: Template not found content: @@ -269,7 +298,7 @@ template_fields: fields: type: array items: - $ref: "../schemas/template.yaml#/TemplateField" + $ref: "../schemas/template-record.yaml#/TemplateField" example: template_id: "550e8400-e29b-41d4-a716-446655440070" form_type: "neris" @@ -309,20 +338,27 @@ template_fields: templates_pdf: post: operationId: uploadTemplatePdf - summary: Upload a blank PDF for a template before defining its fields + summary: Upload a blank PDF and auto-detect its fields description: | - Uploads the blank agency PDF that a template's field coordinates will be - written onto, and returns a `pdf_template_ref` plus the page geometry. + First step of the template-authoring flow. Stores the blank agency PDF, + reads its page geometry, and kicks off asynchronous field detection with + commonforms. Returns 202 immediately with the stored `pdf_template_ref`, + the page dimensions, and an `upload_id` to poll. + + Poll `GET /api/v1/templates/pdf/{upload_id}` until detection completes. + The completed draft carries one TemplateField per detected box, each with + its layout already set and, where the fuzzy matcher scored a confident + match, a suggested `incident_mapping` plus ranked alternatives. Detection + is best effort: boxes with no interpretable label come back as plain + coordinate boxes with no mapping, and a detection failure still leaves + the upload fully usable for manual box drawing. - This is the first step of the template-authoring flow. The visual editor - renders the returned pages, the user draws a box per field, and the box - coordinates become each field's `layout`. The resulting `pdf_template_ref` - is then sent in the `POST /api/v1/templates` body. Upload precedes create - because the layout coordinates only have meaning relative to this PDF. + The editor renders the pages, the user adjusts boxes, sources and + mappings, and the resulting fields array goes into `POST /api/v1/templates` + together with the returned `pdf_template_ref`. - Page dimensions are returned in PDF points (1/72 inch, origin bottom-left) - so the editor can map canvas positions to the same coordinate system the - `layout` fields use. + Page dimensions are in PDF points (1/72 inch, origin bottom-left), the + same coordinate system the `layout` fields use. tags: - templates requestBody: @@ -338,44 +374,29 @@ templates_pdf: type: string format: binary description: Blank fillable or flat PDF form. Max 50MB. + detect_fields: + type: boolean + default: true + description: Set false to skip field detection and only store + the PDF (draft completes immediately with no detected_fields) responses: - "201": - description: PDF stored and ready to be referenced by a template + "202": + description: PDF stored, field detection queued content: application/json: schema: - type: object - required: - - pdf_template_ref - - original_filename - - page_count - - pages - properties: - pdf_template_ref: - type: string - description: Opaque reference to the stored PDF, passed back in - the template body as pdf_template_ref - original_filename: - type: string - page_count: - type: integer - pages: - type: array - description: Per-page geometry in PDF points, index 0 = first page - items: - type: object - required: - - page - - width - - height - properties: - page: - type: integer - width: - type: number - height: - type: number + allOf: + - $ref: "../schemas/template-record.yaml#/TemplateDraft" + - type: object + properties: + job_id: + type: string + format: uuid + poll_url: + type: string example: + upload_id: "550e8400-e29b-41d4-a716-446655440080" + status: "processing" pdf_template_ref: "templates/uploads/550e8400-e29b-41d4-a716-446655440080.pdf" original_filename: "texas_sfm_incident.pdf" page_count: 2 @@ -386,6 +407,9 @@ templates_pdf: - page: 1 width: 612.0 height: 792.0 + retry_after_seconds: 5 + job_id: "550e8400-e29b-41d4-a716-446655440095" + poll_url: "/api/v1/templates/pdf/550e8400-e29b-41d4-a716-446655440080" "400": description: Missing file or malformed multipart request content: @@ -404,3 +428,141 @@ templates_pdf: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" + +template_pdf_draft: + get: + operationId: getTemplateDraft + summary: Get the template draft for an uploaded PDF + description: | + Returns the TemplateDraft for a previous PDF upload: page geometry plus, + once detection finishes, the auto-detected fields with their mapping + suggestions. Poll while status is "processing". A "failed" status means + detection failed but the PDF is stored and usable; the editor falls back + to manual box drawing. + tags: + - templates + parameters: + - name: upload_id + in: path + required: true + description: Upload identifier returned by POST /api/v1/templates/pdf + schema: + type: string + format: uuid + responses: + "200": + description: Current draft state + content: + application/json: + schema: + $ref: "../schemas/template-record.yaml#/TemplateDraft" + examples: + completed: + summary: Detection finished with suggestions + value: + upload_id: "550e8400-e29b-41d4-a716-446655440080" + status: "completed" + pdf_template_ref: "templates/uploads/550e8400-e29b-41d4-a716-446655440080.pdf" + original_filename: "texas_sfm_incident.pdf" + page_count: 2 + pages: + - page: 0 + width: 612.0 + height: 792.0 + - page: 1 + width: 612.0 + height: 792.0 + detected_fields: + - field: + field_name: "incident_number" + field_type: "string" + source: "schema" + required: false + incident_mapping: "report_metadata.incident_number" + layout: + page: 0 + x: 188.33 + y: 621.33 + width: 127.33 + height: 28.67 + detected_label: "Incident No." + suggestions: + - path: "report_metadata.incident_number" + label: "Incident number" + field_type: "string" + section: "report_metadata" + description: "Agency-assigned incident number" + score: 0.93 + - path: "dispatch.dispatch_number" + label: "Dispatch number" + field_type: "string" + section: "dispatch" + score: 0.61 + - field: + field_name: "field_7" + field_type: "string" + source: "manual" + required: false + layout: + page: 1 + x: 100.0 + y: 120.0 + width: 180.0 + height: 18.0 + detected_label: null + suggestions: [] + processing: + summary: Detection still running + value: + upload_id: "550e8400-e29b-41d4-a716-446655440080" + status: "processing" + pdf_template_ref: "templates/uploads/550e8400-e29b-41d4-a716-446655440080.pdf" + original_filename: "texas_sfm_incident.pdf" + page_count: 2 + pages: + - page: 0 + width: 612.0 + height: 792.0 + - page: 1 + width: 612.0 + height: 792.0 + retry_after_seconds: 5 + "404": + description: Upload not found + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" + +template_source_pdf: + get: + operationId: downloadTemplatePdf + summary: Download a template's source PDF + description: | + Returns the blank source PDF a template's layout coordinates refer to. + The editor calls this when reopening a saved template so the user sees + the exact same pages and boxes they saved, ready to edit again. + tags: + - templates + parameters: + - name: template_id + in: path + required: true + description: Unique identifier of the template + schema: + type: string + format: uuid + responses: + "200": + description: The template's source PDF + content: + application/pdf: + schema: + type: string + format: binary + "404": + description: Template not found or has no source PDF + content: + application/json: + schema: + $ref: "../schemas/common.yaml#/ErrorResponse" diff --git a/contracts/schemas/common.yaml b/contracts/schemas/common.yaml index 0798f85d..4fa2f051 100644 --- a/contracts/schemas/common.yaml +++ b/contracts/schemas/common.yaml @@ -82,6 +82,7 @@ AsyncJobResponse: - form_generation - batch_form_generation - report_generation + - template_field_detection status: $ref: "enums.yaml#/JobStatus" estimated_seconds: @@ -109,6 +110,7 @@ Job: - form_generation - batch_form_generation - report_generation + - template_field_detection status: $ref: "enums.yaml#/JobStatus" progress_percent: diff --git a/contracts/schemas/enums.yaml b/contracts/schemas/enums.yaml index b79bedae..327dd991 100644 --- a/contracts/schemas/enums.yaml +++ b/contracts/schemas/enums.yaml @@ -46,45 +46,59 @@ JobStatus: description: Status of any async job FormType: + type: string + description: | + Stable string identifier for a form type. An open string, not a closed + enum, because users register their own templates (any jurisdiction, any + agency) and each carries its own form_type label. Generation and + validation are keyed by template_id; form_type is a human-friendly label + and grouping key. + + Well-known built-in values: neris, nemsis_epcr, nibrs, nfirs_basic, + nfirs_fire, nfirs_structure, nfirs_wildland, nfirs_ems, nfirs_hazmat, + nfirs_apparatus, nfirs_personnel, nfirs_arson, nfirs_casualty_civilian, + nfirs_casualty_responder, cal_fire_ics209, osha_301, un_ssirs, + state_georgia, state_california, state_new_york. + example: "neris" + +FieldSource: type: string enum: - - neris - - nemsis_epcr - - nibrs - - nfirs_basic - - nfirs_fire - - nfirs_structure - - nfirs_wildland - - nfirs_ems - - nfirs_hazmat - - nfirs_apparatus - - nfirs_personnel - - nfirs_arson - - nfirs_casualty_civilian - - nfirs_casualty_responder - - cal_fire_ics209 - - osha_301 - - un_ssirs - - state_georgia - - state_california - - state_new_york + - schema + - static + - manual + - open description: | - Stable string identifier for a form type. Includes the new NERIS standard - (replacing NFIRS as of Feb 2026), legacy NFIRS modules, NEMSIS, NIBRS, - OSHA, state-specific, and international (UN SSIRS) forms. + Where a template field's value comes from at generation time. + - schema: looked up from the incident contract via incident_mapping. + No LLM at fill time. + - static: constant text saved in the template (static_text), stamped + on every generated form. Station name, footer, checkbox marks. + - manual: intentionally per-incident. Left blank until the user types + the value on the review screen; stored in the contract's + custom_fields. Counts as missing in the readiness matrix until + filled. + - open: not covered by the incident contract. The LLM extracts it + during the extraction layer using the field's description as the + instruction (one extra grouped call per template with open fields). + The value lands in the contract's custom_fields, passes the same + human review as schema fields, and is filled by lookup afterwards. IncidentCategory: type: string enum: - fire + - overpressure_explosion - ems - rescue - hazardous_conditions - service_call - good_intent - false_alarm + - natural_disaster - law_enforcement - description: High-level incident category + - special_incident + description: High-level incident category (aligned with NFIRS series 100-900 and UK IRS generic types) CauseCertainty: type: string @@ -101,8 +115,10 @@ InjurySeverity: - minor - moderate - severe + - life_threatening - fatal - description: Severity classification for injuries + - undetermined + description: Severity classification for injuries (NFIRS 5-level scale plus undetermined) RateOfSpread: type: string diff --git a/contracts/schemas/extraction-record.yaml b/contracts/schemas/extraction-record.yaml index ae0fd487..c175b51e 100644 --- a/contracts/schemas/extraction-record.yaml +++ b/contracts/schemas/extraction-record.yaml @@ -15,17 +15,41 @@ ExtractionRequest: description: Hint about the incident type (e.g. "wildland_fire", "structure_fire") state: type: string - description: US state code to apply state-specific extraction rules + description: State or region code to apply region-specific extraction rules agency_type: type: string description: Agency type hint for form selection additionalProperties: true + defaults: + type: object + description: | + Deployment context the extractor uses when the narrative does not say + otherwise: resolving relative dates ("yesterday evening") against the + local timezone, defaulting country, and picking the currency for + Money amounts. Server config supplies these when omitted. + properties: + country: + type: string + description: ISO 3166-1 alpha-2 + timezone: + type: string + description: IANA timezone name (e.g. "Asia/Kolkata") + currency: + type: string + description: ISO 4217 currency for extracted Money amounts ExtractionCompleted: type: object + description: | + A completed extraction. The contract document itself is stored once, on + the incident row this extraction links to (a draft incident is created + automatically the moment extraction completes). The extraction row keeps + only job metadata and the corrections audit trail; this response embeds + the contract read from the incident so the review screen needs one call. required: - extract_id - input_id + - incident_id - status - incident_contract properties: @@ -35,6 +59,11 @@ ExtractionCompleted: input_id: type: string format: uuid + incident_id: + type: string + format: uuid + description: The draft incident row created when extraction completed. + It holds the contract document and is what form generation targets. status: type: string enum: @@ -48,7 +77,10 @@ ExtractionCompleted: processing_time_seconds: type: number incident_contract: - $ref: "incident-contract.yaml#/IncidentContract" + description: The contract document, read from the linked incident row + (the single store; extractions hold no copy of it) + allOf: + - $ref: "incident-contract.yaml#/IncidentContract" corrections: type: array description: Audit trail of manual corrections applied via PATCH @@ -103,12 +135,15 @@ ValidationResult: type: object required: - valid - - form_type + - template_id - extract_id properties: valid: type: boolean - description: Whether all required fields for this form type are present + description: Whether all required fields for this template are present + template_id: + type: string + format: uuid form_type: $ref: "enums.yaml#/FormType" extract_id: @@ -117,13 +152,12 @@ ValidationResult: missing_required: type: array items: - type: string - description: JSON paths of required fields that are missing + $ref: "#/FieldGap" missing_recommended: type: array items: - type: string - description: JSON paths of recommended fields that are missing + $ref: "#/FieldGap" + description: Optional template fields that have no value yet warnings: type: array items: @@ -132,3 +166,82 @@ ValidationResult: field_coverage_percent: type: number description: Percentage of form fields that have values + +FieldGap: + type: object + description: One template field that has no value yet, with enough context + for the UI to explain it and offer the right fix (type it in for manual + fields, correct the contract for schema fields). + required: + - field_name + - source + properties: + field_name: + type: string + description: The template's own field name + source: + $ref: "enums.yaml#/FieldSource" + incident_mapping: + type: string + nullable: true + description: Contract path the value would come from (source=schema), + or the custom_fields key (source=manual/open) + description: + type: string + nullable: true + +ReadinessMatrix: + type: object + description: | + Per-template fill readiness for one extraction, computed by comparing the + contract (including custom_fields) against every registered template's + field list. Pure lookup, no LLM. Drives the form-selection screen: ready + templates show green and selectable, the rest grey with their gaps + listed. Filling a gap (typing a manual value, correcting the contract) + and refetching flips the template to ready. + required: + - extract_id + - templates + properties: + extract_id: + type: string + format: uuid + computed_at: + type: string + format: date-time + templates: + type: array + items: + $ref: "#/TemplateReadiness" + +TemplateReadiness: + type: object + required: + - template_id + - form_type + - display_name + - ready + properties: + template_id: + type: string + format: uuid + form_type: + $ref: "enums.yaml#/FormType" + display_name: + type: string + ready: + type: boolean + description: True when every required field resolves to a value + missing_required: + type: array + items: + $ref: "#/FieldGap" + description: Required fields with no value. Empty when ready. + missing_recommended: + type: array + items: + $ref: "#/FieldGap" + description: Optional fields with no value; the form can still generate + with these left blank + field_coverage_percent: + type: number diff --git a/contracts/schemas/form-record.yaml b/contracts/schemas/form-record.yaml index a83ec41a..57245cd9 100644 --- a/contracts/schemas/form-record.yaml +++ b/contracts/schemas/form-record.yaml @@ -1,33 +1,29 @@ # Form generation related schemas -GenerateAllRequest: +GenerateFormsRequest: type: object + description: | + One request generates any number of forms. The normal flow: the user + reviews the extraction, looks at the readiness matrix, selects the ready + templates they want, and submits their template_ids here. Every value is + read from the incident row's contract document, the single store of + incident data (incident_id comes back in the extraction response). + Generation is pure lookup and draw (no LLM), so even a large batch + completes in seconds. required: - - extract_id + - incident_id properties: - extract_id: - type: string - format: uuid - options: - type: object - properties: - skip_incomplete: - type: boolean - default: true - description: Skip forms that fail validation - force_partial: - type: boolean - default: false - description: Generate forms even with missing fields (leaving blanks) - -GenerateSingleRequest: - type: object - required: - - extract_id - properties: - extract_id: + incident_id: type: string format: uuid + description: Incident whose contract document the forms are filled from + template_ids: + type: array + items: + type: string + format: uuid + description: Templates to generate. When omitted, every template the + readiness matrix reports as ready is generated. options: type: object properties: @@ -36,17 +32,15 @@ GenerateSingleRequest: force_partial: type: boolean default: false - force: - type: boolean - default: false - description: Allow generation even if form_type is not in applicable_forms + description: Also generate templates that are not ready, leaving + their missing fields blank on the PDF BatchGenerateResponse: type: object required: - batch_id - status - - extract_id + - incident_id properties: batch_id: type: string @@ -54,18 +48,30 @@ BatchGenerateResponse: status: type: string enum: [processing] - extract_id: + incident_id: type: string format: uuid forms_queued: type: array items: - $ref: "../schemas/enums.yaml#/FormType" + type: object + properties: + form_id: + type: string + format: uuid + template_id: + type: string + format: uuid + form_type: + $ref: "../schemas/enums.yaml#/FormType" forms_skipped: type: array items: type: object properties: + template_id: + type: string + format: uuid form_type: $ref: "../schemas/enums.yaml#/FormType" reason: @@ -75,50 +81,32 @@ BatchGenerateResponse: poll_url: type: string -FormGenerateResponse: +FormRecord: type: object required: - form_id + - template_id - form_type - status + - incident_id properties: form_id: type: string format: uuid - form_type: - $ref: "../schemas/enums.yaml#/FormType" - status: - type: string - enum: [processing, completed] - extract_id: - type: string - format: uuid - job_id: - type: string - format: uuid - estimated_seconds: - type: integer - poll_url: - type: string - -FormRecord: - type: object - required: - - form_id - - form_type - - status - properties: - form_id: + template_id: type: string format: uuid + description: Template this form was generated from form_type: $ref: "../schemas/enums.yaml#/FormType" status: $ref: "../schemas/enums.yaml#/FormStatus" - extract_id: + incident_id: type: string format: uuid - incident_id: + description: Incident this form was filled from. The extraction is + reachable through the incident; forms do not link it directly. + batch_id: type: string format: uuid nullable: true @@ -158,7 +146,10 @@ FormMappedJson: form_id: type: string format: uuid - extract_id: + template_id: + type: string + format: uuid + incident_id: type: string format: uuid agency_fields: @@ -192,7 +183,16 @@ BatchStatus: form_id: type: string format: uuid + template_id: + type: string + format: uuid form_type: $ref: "../schemas/enums.yaml#/FormType" status: $ref: "../schemas/enums.yaml#/FormStatus" + download_url: + type: string + nullable: true + description: Zip bundle of all completed PDFs, present once the batch + finishes with at least one PDF to bundle + (GET /api/v1/forms/batch/{batch_id}/download) diff --git a/contracts/schemas/incident-contract.yaml b/contracts/schemas/incident-contract.yaml index 1a129df1..fa412a96 100644 --- a/contracts/schemas/incident-contract.yaml +++ b/contracts/schemas/incident-contract.yaml @@ -1,67 +1,520 @@ -# Canonical FireForm Incident Schema -# This is the master superset schema single source of truth for all downstream forms +# FireForm Incident Contract +# +# Master superset schema, single source of truth for all downstream forms. +# Built from a survey of NERIS, NFIRS 5.0, ICS-209, NEMSIS, OSHA 301, UK IRS, +# Australia AIRS, Canada NFID / Ontario SIR, CTIF, UN SSIRS, and Indian state +# fire service proformas (see incident_contract_research.md). +# +# Conventions: +# - One contract field per concept. Form mappers rename per target form. +# - Physical quantities are stored in SI units, unit in the field name +# (_c, _kph, _m, _m2, _ha, _l, _km). Mappers convert for the target form. +# - Money is always {amount, currency} (ISO 4217). +# - Standard-specific codes attach through CodeRef arrays {scheme, code}, +# never as per-standard fields. +# - Absent field = unknown / not extracted. Enums carry explicit `none` and +# `undetermined` members where source standards distinguish them. +# - Fields holding personal data are marked x-pii: true. +# - Fields carry x-aliases: alternate names a user might type when searching +# for the field (synonyms, abbreviations, regional terms - "zip" and +# "pincode" for postal_code). The schema-field catalog behind +# GET /api/v1/schema/fields and the mapping suggester load them from this +# file at startup; the contract is the only place aliases are defined, +# nothing re-declares them in code. Seeded on the high-traffic fields; +# grows over time (including non-English terms) without schema changes. +# - All date-times are RFC 3339 with UTC offset; incident.timezone holds the +# IANA zone for local wall-clock rendering. +# - Each top-level chunk carries x-extraction, telling the extraction worker +# how to treat it: core runs on every incident, gated runs only when the +# narrative shows evidence for it, background runs after the rest, manual is +# never sent to the model (signatures, record ids, authored reflections). +# Gated chunks list the evidence words in x-triggers, and x-extraction-priority +# orders chunks inside a tier. The worker reads all three from this file at +# startup, so retuning the router means editing the contract, not the code. IncidentContract: type: object description: | - The canonical FireForm incident data model. This is the superset schema containing - every field any downstream form could need. Form-specific mappers select only the - relevant fields for each agency template. + The FireForm incident contract. This is the superset schema + containing every field any downstream form could need. Form-specific + mappers select only the relevant fields for each agency template. + Stored as a single JSONB document; queryable stats are promoted to + IncidentRecord columns server-side. properties: schema_version: + x-extraction: manual type: string description: Schema version identifier example: "1.1.0" schema_name: + x-extraction: manual type: string description: Schema name identifier enum: - fireform_incident_contract extraction_metadata: + x-extraction: manual $ref: "#/ExtractionMetadata" report_metadata: + x-extraction: manual $ref: "#/ReportMetadata" incident: + x-extraction: core + x-extraction-priority: 1 $ref: "#/Incident" + dispatch: + x-extraction: core + x-extraction-priority: 2 + $ref: "#/Dispatch" location: + x-extraction: core + x-extraction-priority: 3 $ref: "#/Location" + actions_taken: + x-extraction: core + x-extraction-priority: 5 + $ref: "#/ActionsTaken" + responding_agencies: + x-extraction: core + x-extraction-priority: 6 + $ref: "#/RespondingAgencies" + units: + x-extraction: core + x-extraction-priority: 4 + type: array + description: Per-unit (apparatus/resource) response records with timestamps + items: + $ref: "#/UnitResponse" + resources_summary: + x-extraction: background + $ref: "#/ResourcesSummary" fire: + x-extraction: gated + x-triggers: + - fire + - fires + - flame + - flames + - smoke + - burn + - burning + - burned + - blaze + - arson + - ignition + - ignited + - ember + - embers + - smoldering $ref: "#/Fire" - wildland: - $ref: "#/Wildland" + explosion: + x-extraction: gated + x-triggers: + - explosion + - explosive + - exploded + - blast + - detonation + - detonated + - bleve + - ruptured + - backdraft + $ref: "#/Explosion" + risk_reduction: + x-extraction: background + $ref: "#/RiskReduction" structure: + x-extraction: gated + x-triggers: + - structure + - building + - house + - home + - apartment + - residence + - roof + - floor + - basement + - attic + - warehouse + - garage + - unit + - storey + - story + - wall + - ceiling $ref: "#/Structure" + wildland: + x-extraction: gated + x-triggers: + - wildland + - wildfire + - brush + - grass + - grassland + - forest + - woods + - vegetation + - acre + - acres + - hectare + - hectares + - timber + - canopy + - crown + - fireline + - containment + $ref: "#/Wildland" + exposures: + x-extraction: gated + x-triggers: + - exposure + - exposures + - spread + - spreading + - adjacent + - neighboring + - neighbouring + - next door + - nearby structure + type: array + description: Properties beyond the origin affected by spread of the incident + items: + $ref: "#/Exposure" casualties: + x-extraction: gated + x-triggers: + - injured + - injury + - injuries + - victim + - victims + - fatality + - fatalities + - deceased + - died + - killed + - dead + - transported + - burns + - inhalation + - casualty + - casualties + - hurt + - unconscious $ref: "#/Casualties" + rescues: + x-extraction: gated + x-triggers: + - rescue + - rescued + - trapped + - extricate + - extrication + - extricated + - pulled out + - carried out + - ladder rescue + - confined space + type: array + description: Rescues and assisted evacuations, with or without injury + items: + $ref: "#/Rescue" + evacuation_displacement: + x-extraction: gated + x-triggers: + - evacuate + - evacuated + - evacuation + - displaced + - displacement + - shelter + - sheltered + - relocated + - red cross + - self-evacuated + $ref: "#/EvacuationDisplacement" ems: + x-extraction: gated + x-triggers: + - ems + - ambulance + - medic + - medics + - paramedic + - patient + - patients + - cpr + - triage + - hospital + - vitals + - oxygen + - defibrillator + - als + - bls + - treated $ref: "#/EMS" hazmat: + x-extraction: gated + x-triggers: + - hazmat + - chemical + - chemicals + - spill + - spilled + - leak + - leaking + - fumes + - placard + - decon + - decontamination + - vapor + - vapour + - propane + - cylinder + - tanker + - corrosive + - toxic $ref: "#/Hazmat" - arson: - $ref: "#/Arson" - responding_agencies: - $ref: "#/RespondingAgencies" - resources_deployed: - $ref: "#/ResourcesDeployed" + emerging_hazards: + x-extraction: gated + x-triggers: + - battery + - batteries + - lithium + - ev + - electric vehicle + - solar + - solar panel + - energy storage + - charger + - charging + - thermal runaway + type: array + description: Battery, EV, solar and other stored-energy hazards (NERIS) + items: + $ref: "#/EmergingHazard" + investigation: + x-extraction: gated + x-triggers: + - investigation + - investigator + - investigated + - cause + - origin + - arson + - fire marshal + - evidence + - police + - suspicious + - incendiary + $ref: "#/Investigation" + persons_involved: + x-extraction: gated + x-triggers: + - owner + - occupant + - occupants + - resident + - residents + - tenant + - witness + - witnesses + - driver + - landlord + - homeowner + type: array + description: Owners, occupants and other parties connected to the incident + items: + $ref: "#/PersonInvolved" + mobile_property: + x-extraction: gated + x-triggers: + - vehicle + - vehicles + - car + - cars + - truck + - van + - suv + - motorcycle + - boat + - vessel + - trailer + - bus + - license plate + - tractor + type: array + description: Vehicles, vessels and other mobile property involved + items: + $ref: "#/MobileProperty" + losses: + x-extraction: gated + x-triggers: + - loss + - losses + - damage + - damages + - destroyed + - cost + - estimated + - insured + - insurance + - value + - worth + - dollars + - total loss + $ref: "#/Losses" weather: + x-extraction: gated + x-triggers: + - weather + - wind + - windy + - gust + - rain + - raining + - snow + - storm + - temperature + - humidity + - dry + - fog + - heat + - freezing $ref: "#/Weather" environmental_impact: + x-extraction: gated + x-triggers: + - runoff + - contamination + - contaminated + - waterway + - river + - creek + - stream + - soil + - pollution + - wildlife + - groundwater + - drain $ref: "#/EnvironmentalImpact" infrastructure_impact: + x-extraction: gated + x-triggers: + - power + - powerline + - electric + - utility + - utilities + - outage + - road closed + - road closure + - water main + - gas line + - bridge + - rail + - traffic + - telecom $ref: "#/InfrastructureImpact" near_miss_and_safety: + x-extraction: gated + x-triggers: + - near miss + - mayday + - close call + - ppe + - scba + - rit + - collapse + - safety officer + - firefighter injured + - accountability $ref: "#/NearMissAndSafety" + situation_status: + x-extraction: core + x-extraction-priority: 7 + $ref: "#/SituationStatus" lessons_learned: + x-extraction: manual $ref: "#/LessonsLearned" follow_up: + x-extraction: manual $ref: "#/FollowUp" periodic_reporting: + x-extraction: background $ref: "#/PeriodicReporting" attachments: + x-extraction: manual $ref: "#/Attachments" + custom_fields: + x-extraction: manual + type: object + description: | + Agency-local fields with no contract home (NFIRS special studies, + IRS local options and similar). Keys are agency-defined strings, + values are scalars. Passed through to mappers untouched. + additionalProperties: true + +# --- Shared value types --- + +Money: + type: object + description: Monetary amount with ISO 4217 currency code + properties: + amount: + type: number + currency: + type: string + description: ISO 4217 code + example: "USD" + +CodeRef: + type: object + description: A code from a named external coding scheme + properties: + scheme: + type: string + description: Coding scheme identifier + enum: [neris, nfirs, uk_irs, airs, ontario_sir, nemsis, nibrs, un_ssirs, local, other] + code: + type: string + label: + type: string + nullable: true + +Quantity: + type: object + description: Value with explicit reported unit, used where the unit itself is data (hazmat) + properties: + value: + type: number + unit: + type: string + description: Unit as reported (e.g. l, kg, gal, lb, m3) + +PresenceStatus: + type: string + enum: [present, absent, undetermined] + +OperationStatus: + type: string + description: Whether a protection system operated when exposed to the incident + enum: [operated, failed_to_operate, fire_too_small_to_activate, not_reached_by_fire, undetermined] + +Coordinates: + type: object + properties: + latitude: + type: number + longitude: + type: number + accuracy_meters: + type: number + nullable: true -# --- Sub-schemas --- +# --- Extraction metadata --- ExtractionMetadata: type: object @@ -85,19 +538,17 @@ ExtractionMetadata: type: number minimum: 0 maximum: 1 - description: Overall confidence score from the LLM extraction (0.0–1.0) + description: Overall confidence score from the LLM extraction (0.0-1.0) completeness: $ref: "#/Completeness" - applicable_forms: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" Completeness: type: object description: | - Tells the system which forms can be fully auto-generated vs which need - manual review. Recalculated server-side after every PATCH /extract. + Extraction-quality summary, recalculated server-side after every + PATCH /extract. Per-template readiness is not stored here; it is + computed live by GET /extract/{extract_id}/readiness against the + registered templates. properties: overall_percent: type: integer @@ -118,18 +569,8 @@ Completeness: items: type: string description: JSON paths of fields that were inferred (not explicitly stated) - forms_fully_generatable: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - forms_needing_review: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - forms_missing_data: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" + +# --- Report metadata --- ReportMetadata: type: object @@ -139,21 +580,39 @@ ReportMetadata: example: "FF-2024-CA-0157" incident_number: type: string + description: Department's own incident number example: "CA-SQF-2024-0421" + x-aliases: [incident no, report number, call number, event number, cad number] + external_ids: + type: array + description: Identifiers for this incident in external systems (CAD event, IRWIN, state registry, partner agency) + items: + type: object + properties: + scheme: + type: string + example: "irwin" + value: + type: string report_date: type: string format: date + x-aliases: [date of report, form date, prepared date] report_time: type: string format: time + x-aliases: [time of report] report_status: $ref: "../schemas/enums.yaml#/ReportStatus" reporting_unit: $ref: "#/ReportingUnit" prepared_by: type: array + description: Member(s) making the report items: $ref: "#/Personnel" + officer_in_charge: + $ref: "#/Personnel" reviewed_by: type: array items: @@ -176,6 +635,7 @@ ReportingUnit: properties: station_name: type: string + x-aliases: [fire station, station] station_id: type: string agency_name: @@ -191,14 +651,21 @@ Personnel: properties: name: type: string + x-pii: true badge_number: type: string + x-pii: true rank: type: string role: type: string + assignment: + type: string + nullable: true contact_number: type: string + nullable: true + x-pii: true signature_captured: type: boolean @@ -207,8 +674,10 @@ Reviewer: properties: name: type: string + x-pii: true badge_number: type: string + x-pii: true rank: type: string role: @@ -219,25 +688,106 @@ Reviewer: approved: type: boolean +# --- Incident core --- + Incident: type: object properties: name: type: string description: Human-readable incident name + x-aliases: [incident name, fire name, incident title] types: type: array items: $ref: "#/IncidentType" + special_modifiers: + type: array + description: Magnitude or class tags qualifying the incident (NERIS special modifiers) + items: + type: string + example: ["mass_casualty", "major_incident"] + false_alarm: + type: object + description: Populated when the final type is a false alarm + properties: + reason: + type: string + enum: [malicious, good_intent, automatic_system_fault, automatic_system_accidental, human_error, undetermined, other] + nullable: true + reason_description: + type: string + nullable: true + special_service_type: + type: string + nullable: true + description: Non-fire service call subtype (lift release, lock-in, flooding, animal assist, co-response) + chimney_fire: + type: boolean + nullable: true + description: Flame confined to a chimney (UK IRS 3.9) + timezone: + type: string + nullable: true + description: IANA timezone of the incident, for local wall-clock rendering + example: "America/Los_Angeles" + # Discovery (UK IRS 5.2-5.4, arson relevance) + discovered_datetime: + type: string + format: date-time + nullable: true + how_discovered: + type: string + nullable: true + description: How the fire or emergency was first discovered + delay_ignition_to_discovery: + type: string + enum: [immediate, under_5_min, 5_to_30_min, over_30_min, undetermined] + nullable: true + delay_discovery_to_call: + type: string + enum: [immediate, under_5_min, 5_to_30_min, over_30_min, undetermined] + nullable: true + # Incident-level timeline (per-unit times live in units[]) start_datetime: type: string format: date-time + description: Estimated ignition or emergency start + x-aliases: [ignition time, time of fire, start time, date of incident, incident date, incident time] alarm_datetime: type: string format: date-time + description: Department alerted / alarm time + x-aliases: [alarm time, time of call, call time, reported time, notification time] first_arrival_datetime: type: string format: date-time + description: First unit on scene + x-aliases: [arrival time, on scene time, time of arrival, attendance time] + command_established_datetime: + type: string + format: date-time + nullable: true + sizeup_complete_datetime: + type: string + format: date-time + nullable: true + water_on_fire_datetime: + type: string + format: date-time + nullable: true + primary_search_begin_datetime: + type: string + format: date-time + nullable: true + primary_search_complete_datetime: + type: string + format: date-time + nullable: true + knocked_down_datetime: + type: string + format: date-time + nullable: true containment_datetime: type: string format: date-time @@ -246,16 +796,75 @@ Incident: type: string format: date-time nullable: true + x-aliases: [under control, control time] + extrication_complete_datetime: + type: string + format: date-time + nullable: true + suppression_complete_datetime: + type: string + format: date-time + nullable: true + loss_stopped_datetime: + type: string + format: date-time + nullable: true + stop_message_datetime: + type: string + format: date-time + nullable: true + description: Stop / situation-under-control message to control room (UK IRS 2.5) cleared_datetime: type: string format: date-time nullable: true + description: Last unit cleared the scene + x-aliases: [last unit cleared, departure time, left scene, return time] + closed_datetime: + type: string + format: date-time + nullable: true + description: Incident administratively closed total_duration_hours: type: number nullable: true + x-aliases: [duration, time on scene, total time] + # Response context + alarm_level: + type: integer + nullable: true + description: Number of alarms / escalation level + x-aliases: [number of alarms, alarms] + shift_or_platoon: + type: string + nullable: true + district: + type: string + nullable: true + description: Response district or box area + people_present: + type: boolean + nullable: true + description: Whether people were present at the location at the time + animals_rescued: + type: integer + nullable: true + animals_deceased: + type: integer + nullable: true + # Narratives narrative: type: string description: Free text summary of the incident + x-aliases: [remarks, comments, description of incident, summary, details] + narrative_impediment: + type: string + nullable: true + description: Obstacles that impacted the response (NERIS) + narrative_outcome: + type: string + nullable: true + description: Final disposition of the incident (NERIS) raw_transcript: type: string description: Original voice or text input verbatim @@ -265,51 +874,163 @@ IncidentType: properties: primary: type: boolean + description: Only one entry may be primary category: $ref: "../schemas/enums.yaml#/IncidentCategory" subcategory: type: string - neris_code: + description: Free-form specific type within the category + x-aliases: [incident type, call type, nature of call, type of incident] + codes: + type: array + description: This type expressed in external coding schemes + items: + $ref: "#/CodeRef" + +# --- Dispatch / call handling --- + +Dispatch: + type: object + description: Call handling data, usually pre-populated from CAD/PSAP when available + properties: + psap_id: + type: string + nullable: true + description: Dispatch center / PSAP identifier + dispatch_center: + type: string + nullable: true + description: Dispatch center name + cad_event_id: + type: string + nullable: true + call_received_datetime: + type: string + format: date-time + nullable: true + description: Call arrived at PSAP or department dispatch center + x-aliases: [call received, time of call, call time] + call_answered_datetime: + type: string + format: date-time + nullable: true + call_created_datetime: + type: string + format: date-time + nullable: true + description: CAD event created + first_unit_dispatched_datetime: + type: string + format: date-time + nullable: true + call_origin: + type: string + enum: [person_landline, person_mobile, person_in_person, automatic_alarm_originator, automatic_alarm_monitoring_center, other_agency, police, ambulance, coastguard, other_fire_service, other, undetermined] + nullable: true + automatic_alarm: + type: boolean + nullable: true + description: Call originated from an automatic alarm system + incident_type_at_dispatch: type: string - description: NERIS incident type code (replaces NFIRS as of Feb 2026) - nfirs_code: + nullable: true + description: Incident type as received by the control room; may differ from final type + determinate_code: type: string - description: Legacy NFIRS incident type code + nullable: true + description: Output code from the dispatch protocol (e.g. EMD/ProQA) + priority_at_call: + type: integer + nullable: true + minimum: 1 + maximum: 5 + dispatcher_comments: + type: array + items: + type: object + properties: + comment: + type: string + timestamp: + type: string + format: date-time + nullable: true + +# --- Location --- Location: type: object properties: + location_type: + type: string + enum: [street_address, intersection, milepost_or_highway, coordinates_only, unaddressable_area, water_body, other] + nullable: true address: type: string nullable: true + description: Full street address as one line + x-aliases: [street address, location, address of incident, premises] + cross_streets: + type: array + description: Nearest cross street(s) + x-aliases: [cross street, intersection, nearest junction] + items: + type: string nearest_landmark: type: string nullable: true - nearest_town: + city: type: string nullable: true + description: City, town or nearest settlement + x-aliases: [town, municipality, village] + district_or_zone: + type: string + nullable: true + description: Administrative district, borough or fire zone + x-aliases: [ward, zone, fire zone] county: type: string nullable: true + x-aliases: [borough, parish] state: type: string nullable: true + description: State, province or region + x-aliases: [province, region] country: type: string nullable: true + description: ISO 3166-1 alpha-2 code preferred + example: "US" postal_code: type: string nullable: true + x-aliases: [zip, zip code, zipcode, pincode, pin code, postcode] + census_area: + type: string + nullable: true + description: Census tract or national statistical area code coordinates: $ref: "#/Coordinates" ignition_point_coordinates: - $ref: "#/CoordinatesBasic" - elevation_range_ft: - type: string + $ref: "#/Coordinates" + grid_reference: + type: object + description: National grid reference where used instead of lat/long + properties: + scheme: + type: string + enum: [usng, mgrs, utm, osgb, other] + value: + type: string + elevation_m: + type: number nullable: true legal_description: type: string nullable: true + description: Township / section / range or equivalent cadastral reference jurisdiction: type: object properties: @@ -321,399 +1042,1782 @@ Location: type: boolean tribal: type: boolean + ownership_at_origin: + type: string + enum: [private, city_or_local, county, state_or_province, federal, tribal, military, foreign, other, undetermined] + nullable: true + description: Ownership of the property at the point of origin + population_density: + type: string + enum: [urban, suburban, rural, wilderness] + nullable: true property_type: type: string nullable: true + x-aliases: [type of property, premises type] property_use: type: string nullable: true - dispatch_center: + description: Use of the property at the time (residential, commercial, school...) + x-aliases: [occupancy, occupancy type, use of property] + property_use_codes: + type: array + description: Property use in external coding schemes (NFIRS 3-digit, NERIS location use) + items: + $ref: "#/CodeRef" + mixed_use: type: string nullable: true + description: Mixed-use classification when the property has multiple uses + over_border: + type: object + description: Incident on another service's ground (UK IRS 1.5-1.7) + properties: + is_over_border: + type: boolean + other_service_name: + type: string + nullable: true + other_service_incident_number: + type: string + nullable: true -Coordinates: - type: object - properties: - latitude: - type: number - longitude: - type: number - accuracy_meters: - type: number +# --- Actions taken --- -CoordinatesBasic: +ActionsTaken: type: object + description: What responders did on scene. Every reporting standard requires this. properties: - latitude: - type: number - longitude: - type: number + actions: + type: array + items: + type: object + properties: + category: + type: string + enum: [fire_suppression, search, rescue, ems_care, extrication, hazmat_mitigation, ventilation, forcible_entry, salvage_overhaul, water_supply, exposure_protection, evacuation, command_control, investigation, public_assist, standby, information_referral, systems_restoration, other] + description: + type: string + nullable: true + codes: + type: array + items: + $ref: "#/CodeRef" + no_action_reason: + type: string + nullable: true + description: Why no action was taken (canceled enroute, no hazard found...) -Fire: +# --- Response: agencies, units, resource totals --- + +RespondingAgencies: type: object properties: - cause_category: + primary_agency: + type: string + all_agencies: + type: array + items: + $ref: "#/RespondingAgency" + mutual_aid_activated: + type: boolean + x-aliases: [mutual aid, aid given or received] + aid_direction: type: string + enum: [given, received, both, none] nullable: true - cause_specific: + aid_type: type: string + enum: [automatic, mutual, other, none] nullable: true - cause_certainty: - $ref: "../schemas/enums.yaml#/CauseCertainty" - arson_suspected: + non_fd_entities: + type: array + description: Non fire department entities that assisted (utility, red cross, public works) + items: + type: string + unified_command: + type: boolean + incident_commander: + type: object + properties: + name: + type: string + x-pii: true + x-aliases: [ic, incident commander, officer in charge, oic] + agency: + type: string + position: + type: string + nullable: true + +RespondingAgency: + type: object + properties: + agency_name: + type: string + agency_type: + type: string + description: fire, ems, police, forestry, military, utility, ngo, other + role: + type: string + personnel_count: + type: integer + nullable: true + incident_number_at_agency: + type: string + nullable: true + description: That agency's own incident number for cross-referencing + +UnitResponse: + type: object + description: One responding unit (apparatus or resource) and its timeline + properties: + unit_id: + type: string + description: Callsign or unit identifier + x-aliases: [callsign, apparatus id, unit number, appliance] + unit_name: + type: string + nullable: true + agency_name: + type: string + nullable: true + apparatus_type: + type: string + enum: [engine_pumper, ladder_aerial, quint, tanker_tender, brush_wildland, arff, dozer_plow, heavy_equipment, aircraft_fixed_wing, helicopter, boat, rescue_unit, usar_unit, hazmat_unit, ambulance_bls, ambulance_als, command_vehicle, support_unit, hand_crew, privately_owned, other] + nullable: true + use: + type: string + enum: [suppression, ems, rescue, hazmat, command, support, other] + nullable: true + personnel_count: + type: integer + nullable: true + personnel: + type: array + items: + type: object + properties: + personnel_id: + type: string + x-pii: true + name: + type: string + x-pii: true + rank: + type: string + nullable: true + role: + type: string + nullable: true + response_mode: + type: string + enum: [emergency_lights_siren, non_emergency, undetermined] + nullable: true + canceled_enroute: + type: boolean + nullable: true + dispatched_datetime: + type: string + format: date-time + nullable: true + enroute_datetime: + type: string + format: date-time + nullable: true + arrived_datetime: + type: string + format: date-time + nullable: true + staged_datetime: + type: string + format: date-time + nullable: true + at_patient_datetime: + type: string + format: date-time + nullable: true + enroute_hospital_datetime: + type: string + format: date-time + nullable: true + arrived_hospital_datetime: + type: string + format: date-time + nullable: true + transfer_of_care_datetime: + type: string + format: date-time + nullable: true + cleared_datetime: + type: string + format: date-time + nullable: true + in_service_datetime: + type: string + format: date-time + nullable: true + description: Back available for calls + turnout_seconds: + type: integer + nullable: true + description: Computed, dispatched to enroute + travel_seconds: + type: integer + nullable: true + description: Computed, enroute to arrived + transport_mode: + type: string + enum: [emergency_lights_siren, non_emergency, undetermined] + nullable: true + hospital_destination: + type: string + nullable: true + actions_taken: + type: array + description: Actions by this unit, same categories as incident actions + items: + type: string + +ResourcesSummary: + type: object + description: Aggregate counts; per-unit detail lives in units[] + properties: + total_personnel: + type: integer + x-aliases: [personnel, manpower, firefighters on scene, staff count] + personnel_breakdown: + type: object + properties: + firefighters: + type: integer + officers: + type: integer + engineers_operators: + type: integer + ems_personnel: + type: integer + incident_command: + type: integer + support_staff: + type: integer + apparatus_counts: + type: object + description: Unit counts by primary use (NFIRS Basic G1) + properties: + suppression: + type: integer + ems: + type: integer + other: + type: integer + crew_types: + type: array + description: Wildland crew types deployed (hand crew type 1/2, engine crew...) + items: + type: string + counts_include_aid_received: + type: boolean + nullable: true + +# --- Fire --- + +Fire: + type: object + properties: + # Cause + cause_category: + type: string + enum: [intentional, unintentional, equipment_failure, act_of_nature, cause_under_investigation, undetermined, other] + nullable: true + x-aliases: [cause, cause of fire, how the fire started, fire cause] + cause_specific: + type: string + nullable: true + cause_codes: + type: array + items: + $ref: "#/CodeRef" + cause_certainty: + $ref: "../schemas/enums.yaml#/CauseCertainty" + arson_suspected: + type: boolean + x-aliases: [suspicious fire, deliberate, malicious] + # Ignition detail + area_of_origin: + type: string + nullable: true + description: Room or area where the fire began + x-aliases: [point of origin, seat of fire, where the fire started, origin] + heat_source: + type: string + nullable: true + description: What provided the heat that started the fire + x-aliases: [source of ignition, ignition source, source of heat] + ignition_power_source: + type: string + nullable: true + description: What powered the ignition source (mains, battery, gas, open flame) + item_first_ignited: + type: string + nullable: true + x-aliases: [first item ignited, what caught fire first] + material_first_ignited: + type: string + nullable: true + x-aliases: [material ignited, first material] + multiple_seats_of_fire: + type: boolean + nullable: true + description: More than one independent point of origin (arson indicator) + human_factors: + type: array + description: Human factors contributing to ignition + items: + type: string + enum: [asleep, impaired_by_alcohol_or_drugs, unattended_person, mentally_disabled, physically_disabled, multiple_persons_involved, age_was_factor, other] + person_involved_age: + type: integer + nullable: true + description: Estimated age of person whose age was a factor + person_involved_sex: + type: string + nullable: true + contributing_factors: + type: array + description: Non-human factors contributing to ignition + items: + type: string + equipment_involved: + type: object + description: Equipment involved in ignition, if any + properties: + involved: + type: boolean + equipment_type: + type: string + nullable: true + brand: + type: string + nullable: true + model: + type: string + nullable: true + serial_number: + type: string + nullable: true + year: + type: integer + nullable: true + power_source: + type: string + nullable: true + portability: + type: string + enum: [portable, stationary] + nullable: true + on_site_materials: + type: array + description: Significant commercial/industrial/agricultural materials on the property + items: + type: string + # Fuel and behavior + fuel_types: + type: array + items: + type: string + fire_spread_directions: + type: array + items: + type: string + rate_of_spread: + $ref: "../schemas/enums.yaml#/RateOfSpread" + rate_of_spread_m_per_min: + type: number + nullable: true + flame_length_m: + type: number + nullable: true + spotting_distance_km: + type: number + nullable: true + unusual_behaviors: + type: array + items: + type: string + rapid_growth_cause: + type: string + nullable: true + description: Cause of any rapid fire growth (UK IRS 8.8) + fire_suppression_factors: + type: array + description: Factors that helped or hindered suppression + items: + type: string + # Suppression operations + suppression_operations: + $ref: "#/SuppressionOperations" + # Investigation need lives in investigation section + +SuppressionOperations: + type: object + properties: + water_supply_type: + type: string + enum: [pressurized_hydrant, rural_water_supply, tanker_shuttle, drafting_static_source, onboard_water_only, none_needed, other, undetermined] + nullable: true + water_used_l: + type: number + nullable: true + x-aliases: [water used, gallons used, litres used] + extinguishing_agents: + type: array + items: + type: string + enum: [water, foam, co2, dry_chemical, wet_chemical, halon_clean_agent, sand_earth, blanket_smothering, other] + suppression_appliances: + type: array + description: Appliances used for suppression (jets, hose reels, monitors, extinguishers) + items: + type: string + equipment_used: + type: array + description: Equipment used at the incident with counts (UK IRS 6.16-6.17) + items: + type: object + properties: + equipment_type: + type: string + count: + type: integer + ba_wearers_count: + type: integer + nullable: true + description: Breathing apparatus wearers + x-aliases: [breathing apparatus, ba count, scba] + firefighting_delay: + type: object + properties: + occurred: + type: boolean + reason: + type: string + nullable: true + public_action_before_arrival: + type: string + nullable: true + description: Main action taken by the public before responders arrived + +# --- Explosion --- + +Explosion: + type: object + description: Explosion or overpressure event, with or without fire (UK IRS 8.10-8.13) + properties: + occurred: + type: boolean + cause: + type: string + nullable: true + stage: + type: string + enum: [before_fire, during_fire, after_fire, no_fire] + nullable: true + containers_involved: + type: array + items: + type: string + +# --- Risk reduction / protection systems --- + +RiskReduction: + type: object + description: Alarms, detectors and suppression systems and how they performed + properties: + smoke_alarm: + type: object + properties: + presence: + $ref: "#/PresenceStatus" + alarm_type: + type: string + enum: [smoke, heat, combination, sprinkler_waterflow, multiple_types, other, undetermined] + nullable: true + power_supply: + type: string + enum: [battery_only, hardwire_only, hardwire_with_battery, plug_in, plug_in_with_battery, mechanical, multiple, other, undetermined] + nullable: true + working: + type: boolean + nullable: true + operation: + $ref: "#/OperationStatus" + effectiveness: + type: string + enum: [alerted_occupants_responded, alerted_occupants_no_response, no_occupants, failed_to_alert, undetermined] + nullable: true + failure_reason: + type: string + enum: [power_failure_or_disconnect, improper_installation, defective, lack_of_maintenance, battery_missing, battery_dead, other, undetermined] + nullable: true + occupant_response: + type: string + nullable: true + fire_alarm: + type: object + description: Building fire alarm system + properties: + presence: + $ref: "#/PresenceStatus" + alarm_type: + type: string + nullable: true + monitored: + type: boolean + nullable: true + operation: + $ref: "#/OperationStatus" + failure_reason: + type: string + nullable: true + other_alarm: + type: object + description: CO, gas, security or other alarm + properties: + presence: + $ref: "#/PresenceStatus" + alarm_type: + type: string + nullable: true + suppression_system: + type: object + description: Automatic extinguishing system + properties: + presence: + $ref: "#/PresenceStatus" + system_type: + type: string + enum: [wet_pipe_sprinkler, dry_pipe_sprinkler, other_sprinkler, dry_chemical, foam, halon_clean_agent, co2, water_mist, other_special_hazard, other, undetermined] + nullable: true + coverage: + type: string + enum: [full, partial, undetermined] + nullable: true + operation: + $ref: "#/OperationStatus" + sprinkler_heads_activated: + type: integer + nullable: true + effective: + type: boolean + nullable: true + failure_reason: + type: string + enum: [system_shut_off, not_enough_agent, agent_did_not_reach_fire, wrong_system_type, fire_outside_protected_area, components_damaged, lack_of_maintenance, manual_intervention, other, undetermined] + nullable: true + cooking_suppression: + type: object + properties: + presence: + $ref: "#/PresenceStatus" + system_type: + type: string + nullable: true + fixed_firefighting_facilities: + type: array + description: Built-in firefighting facilities (risers, hose reels, smoke control, fire lift) + items: + type: object + properties: + facility_type: + type: string + used: + type: boolean + nullable: true + worked: + type: boolean + nullable: true + failure_reason: + type: string + nullable: true + +# --- Structure --- + +Structure: + type: object + properties: + is_structure_involved: + type: boolean + structures_threatened: + type: integer + nullable: true + structures_damaged: + type: integer + nullable: true + structures_destroyed: + type: integer + nullable: true + structures_protected: + type: integer + nullable: true + buildings_involved: + type: integer + nullable: true + description: Number of buildings involved at the origin property + building_status: + type: string + enum: [occupied_in_use, vacant_secured, vacant_unsecured, under_construction, under_renovation, under_demolition, derelict, undetermined] + nullable: true + construction_type: + type: string + nullable: true + x-aliases: [building construction, construction class] + construction_codes: + type: array + items: + $ref: "#/CodeRef" + special_construction_method: + type: string + nullable: true + description: Notable construction method involved (timber frame, sandwich panel, cladding system) + stories_above_grade: + type: integer + nullable: true + x-aliases: [floors, storeys, number of stories, building height] + stories_below_grade: + type: integer + nullable: true + x-aliases: [basement levels, floors below ground] + total_floor_area_m2: + type: number + nullable: true + x-aliases: [floor area, square footage, building area] + main_floor_area_m2: + type: number + nullable: true + residential_units: + type: integer + nullable: true + description: Residential living units in the building of origin + occupancy_at_time: + type: integer + nullable: true + description: Estimated number of people in the building at the time + fire_safety_regulations_apply: + type: boolean + nullable: true + means_of_escape_condition: + type: string + nullable: true + compartmentation_effective: + type: boolean + nullable: true + # Origin and spread within the structure + story_of_origin: + type: integer + nullable: true + description: Negative below grade, 1 is ground floor + room_of_origin: + type: string + nullable: true + room_of_origin_area_m2: + type: number + nullable: true + floor_of_origin_area_m2: + type: number + nullable: true + fire_spread_extent: + type: string + enum: [confined_to_object, confined_to_room, confined_to_floor, confined_to_building, beyond_building, no_flame_damage, undetermined] + nullable: true + x-aliases: [extent of damage, spread of fire, fire spread] + item_contributing_most_to_spread: + type: string + nullable: true + material_contributing_most_to_spread: + type: string + nullable: true + arrival_conditions: + type: string + enum: [no_visible_smoke_or_fire, smoke_showing, fire_showing, fully_involved, collapsed, undetermined] + nullable: true + progressed_beyond_arrival: + type: boolean + nullable: true + description: Fire extended beyond the conditions found on arrival + smoke_damage_only: + type: boolean + nullable: true + description: Heat/smoke damage with no flame damage (UK IRS 8.19) + stories_damaged: + type: object + description: Count of stories by flame damage band (NFIRS-3 J3) + properties: + minor: + type: integer + description: 1-24% flame damage + significant: + type: integer + description: 25-49% flame damage + heavy: + type: integer + description: 50-74% flame damage + extreme: + type: integer + description: 75-100% flame damage + damage_area_on_arrival_m2: + type: number + nullable: true + damage_area_at_stop_m2: + type: number + nullable: true + description: Horizontal area damaged by flame/heat when fire was stopped + +# --- Wildland --- + +Wildland: + type: object + properties: + is_wildland_incident: + type: boolean + discovery_datetime: + type: string + format: date-time + nullable: true + area_type: + type: string + enum: [urban, suburban, rural, wildland_urban_interface, remote_wilderness] + nullable: true + area_burned_ha: + type: number + nullable: true + description: Total area burned in hectares + x-aliases: [acres burned, area burnt, burn area, fire size, hectares burned] + land_ownership_breakdown: + type: object + properties: + federal_ha: + type: number + state_ha: + type: number + private_ha: + type: number + tribal_ha: + type: number + other_ha: + type: number + percent_contained: + type: integer + minimum: 0 + maximum: 100 + x-aliases: [containment, contained percent] + fire_danger_rating: + type: string + enum: [low, moderate, high, very_high, severe, extreme, catastrophic] + nullable: true + description: Fire danger rating in effect at the time + fuel_model: + type: string + nullable: true + description: NFDRS or local fuel model at origin + fuel_moisture_percent: + type: number + nullable: true + complexity_level: + type: string + enum: [type_5, type_4, type_3, type_2, type_1] + nullable: true + description: Incident complexity (type 5 lowest, type 1 highest) + slope_position: + type: string + nullable: true + description: Relative position on slope at origin + aspect: + type: string + nullable: true + person_responsible: + type: object + description: Person who caused the wildland fire, if any (NFIRS-8 L) + properties: + status: + type: string + enum: [identified, unidentified, fire_not_caused_by_person] + age: + type: integer + nullable: true + x-pii: true + sex: + type: string + nullable: true + x-pii: true + activity: + type: string + nullable: true + right_of_way: + type: object + description: Nearby road/rail/power right-of-way (NFIRS-8 M) + properties: + row_type: + type: string + nullable: true + distance_m: + type: number + nullable: true + crops_burned: + type: array + items: + type: string + fire_lines: + type: object + properties: + primary_line_km: + type: number + secondary_line_km: + type: number + dozer_line_km: + type: number + hand_line_km: + type: number + aerial_operations: + type: object + properties: + water_dropped_l: + type: number + retardant_dropped_l: + type: number + total_flight_hours: + type: number + containment_strategies: + type: array + items: + type: string + +# --- Exposures --- + +Exposure: + type: object + description: A property beyond the origin damaged or threatened by spread + properties: + exposure_number: + type: integer + description: 0 is the origin; exposures count up from 1 (NFIRS convention) + exposure_type: + type: string + nullable: true + item_damaged: + type: string + nullable: true + address: + type: string + nullable: true + coordinates: + $ref: "#/Coordinates" + property_use: + type: string + nullable: true + people_present: + type: boolean + nullable: true + damage_rating: + type: string + enum: [none, minor, significant, heavy, destroyed, undetermined] + nullable: true + people_displaced: + type: integer + nullable: true + +# --- Casualties --- + +Casualties: + type: object + description: Injuries and deaths. Uninjured rescues live in rescues[]. + properties: + civilian: + type: array + items: + $ref: "#/CivilianCasualty" + responder: + type: array + items: + $ref: "#/ResponderCasualty" + total_civilian_injuries: + type: integer + x-aliases: [injuries, injured, number injured, victims injured, casualties] + total_civilian_fatalities: + type: integer + x-aliases: [deaths, fatalities, killed, number dead, lives lost, victims died] + total_responder_injuries: + type: integer + x-aliases: [firefighter injuries, personnel injured, service casualties] + total_responder_fatalities: + type: integer + x-aliases: [firefighter deaths, line of duty deaths, lodd] + +CivilianCasualty: + type: object + properties: + name: + type: string + nullable: true + x-pii: true + age: + type: integer + nullable: true + x-pii: true + date_of_birth: + type: string + format: date + nullable: true + x-pii: true + sex: + type: string + nullable: true + x-pii: true + race_ethnicity: + type: string + nullable: true + x-pii: true + description: Only where the target jurisdiction collects it (US, UK) + affiliation: + type: string + enum: [civilian, ems_non_fd, police, other_responder, undetermined] + nullable: true + injury_datetime: + type: string + format: date-time + nullable: true + injury_type: + type: string + description: Nature of the injury (burns, smoke inhalation, trauma) + primary_symptom: + type: string + nullable: true + primary_body_part: + type: string + nullable: true + severity: + $ref: "../schemas/enums.yaml#/InjurySeverity" + cause: + type: string + nullable: true + human_factors: + type: array + items: + type: string + enum: [asleep, unconscious, impaired_by_alcohol, impaired_by_drugs, mentally_disabled, physically_disabled, physically_restrained, unattended_person, other] + contributing_factors: + type: array + items: + type: string + activity_when_injured: + type: string + enum: [escaping, rescue_attempt, fire_control, returning_before_control, returning_after_control, sleeping, unable_to_act, irrational_act, other, undetermined] + nullable: true + location_at_ignition: + type: string + enum: [in_area_of_origin, in_building_not_in_area, outside_building, not_on_property, undetermined] + nullable: true + story_at_start: + type: integer + nullable: true + story_where_injured: + type: integer + nullable: true + location_where_found: + type: string + nullable: true + cause_of_failure_to_escape: + type: string + nullable: true + description: Why the person could not escape (Canada NFID casualty file) + disposition: + type: string + nullable: true + transported: + type: boolean + hospital: + type: string + nullable: true + fatal_circumstances: + type: string + nullable: true + death_certificate_reconciled: + type: boolean + nullable: true + +ResponderCasualty: + type: object + properties: + personnel_id: + type: string + x-pii: true + name: + type: string + nullable: true + x-pii: true + age: + type: integer + nullable: true + x-pii: true + sex: + type: string + nullable: true + x-pii: true + agency: + type: string + role: + type: string + rank: + type: string + nullable: true + career_or_volunteer: + type: string + enum: [career, volunteer, undetermined] + nullable: true + years_of_service: + type: number + nullable: true + usual_assignment: + type: string + nullable: true + physical_condition_prior: + type: string + enum: [rested, fatigued, ill_or_injured, other, undetermined] + nullable: true + prior_responses_24h: + type: integer + nullable: true + injury_datetime: + type: string + format: date-time + nullable: true + injury_type: + type: string + primary_symptom: + type: string + nullable: true + primary_body_part: + type: string + nullable: true + severity: + $ref: "../schemas/enums.yaml#/InjurySeverity" + cause: + type: string + nullable: true + contributing_factor: + type: string + nullable: true + object_involved: + type: string + nullable: true + activity_at_injury: + type: string + nullable: true + where_occurred: + type: string + enum: [enroute_to_scene, at_scene_inside, at_scene_outside, enroute_to_facility, at_facility, returning, at_station, other, undetermined] + nullable: true + story_where_injured: + type: integer + nullable: true + protective_equipment_failure: + type: object + properties: + failed: + type: boolean + item: + type: string + nullable: true + problem: + type: string + nullable: true + duty_status: + type: string + enum: [on_duty, off_duty_responding, off_duty, undetermined] + nullable: true + treatment: + type: string + nullable: true + taken_to: + type: string + enum: [hospital, doctors_office, morgue, residence, station, not_transported, other] + nullable: true + transported: type: boolean - material_first_ignited: + hospital: type: string nullable: true - fuel_types: + hospitalized_overnight: + type: boolean + nullable: true + return_to_duty_date: + type: string + format: date + nullable: true + osha_recordable: + type: boolean + exposure_only: + type: boolean + nullable: true + description: Chemical/biological exposure without immediate symptoms + +# --- Rescues --- + +Rescue: + type: object + description: One person rescued, assisted or self-evacuated (NERIS rescue modules) + properties: + person_type: + type: string + enum: [civilian, firefighter, other_responder] + rescue_type: + type: string + enum: [rescue, assist, self_evacuation, body_recovery, no_rescue_needed] + nullable: true + presence_known_beforehand: + type: boolean + nullable: true + age: + type: integer + nullable: true + x-pii: true + sex: + type: string + nullable: true + x-pii: true + primary_mode: + type: string + nullable: true + description: Primary rescue mode (interior search, ladder, water, rope, extrication) + actions: type: array items: type: string - fire_spread_directions: + impediments: type: array items: type: string - rate_of_spread: - $ref: "../schemas/enums.yaml#/RateOfSpread" - flame_lengths_ft: + room_type: type: string nullable: true - spotting_distance_miles: - type: number + elevation: + type: string nullable: true - unusual_behaviors: - type: array - items: - type: string - detector_present: - type: boolean + description: Elevation at which the person was found (below grade, ground, upper story, roof) + removal_path: + type: string nullable: true - detector_operated: - type: boolean + description: Route used to remove the person (internal stairs, window, aerial) + relative_time_to_suppression: + type: string + enum: [before_suppression, during_suppression, after_suppression, undetermined] nullable: true - suppression_system_present: + gas_isolation: type: boolean nullable: true - suppression_system_operated: + description: Space was isolated from heat/toxic gas flow + mayday: + type: object + description: Firefighter emergencies only + properties: + called: + type: boolean + relative_time: + type: string + nullable: true + rit_activated: + type: boolean + nullable: true + resulting_casualty: type: boolean nullable: true - estimated_damage_usd: - type: number - nullable: true - contents_loss_usd: - type: number - nullable: true + description: True if this person also appears in casualties -Wildland: +# --- Evacuation and displacement --- + +EvacuationDisplacement: type: object properties: - is_wildland_incident: + evacuation_occurred: type: boolean - total_acres_burned: - type: number nullable: true - land_ownership_breakdown: - type: object - properties: - federal_acres: - type: number - state_acres: - type: number - private_acres: - type: number - tribal_acres: - type: number - percent_contained: + evacuation_status: + type: string + enum: [none, planned, in_progress, completed, repopulation_in_progress, shelter_in_place] + nullable: true + people_evacuated_without_assistance: type: integer - minimum: 0 - maximum: 100 - fire_lines: - type: object - properties: - primary_line_miles: - type: number - secondary_line_miles: - type: number - dozer_line_miles: - type: number - hand_line_miles: - type: number - aerial_operations: - type: object - properties: - water_drops_gallons: - type: integer - retardant_drops_gallons: - type: integer - total_flight_hours: - type: number - containment_strategies: - type: array - items: - type: string - -Structure: - type: object - properties: - is_structure_involved: - type: boolean - structures_threatened: + nullable: true + people_evacuated_with_assistance: type: integer nullable: true - structures_damaged: + people_assisted_by_fd: type: integer nullable: true - structures_destroyed: + total_people_evacuated: type: integer nullable: true - structures_protected: + x-aliases: [evacuees, people evacuated, number evacuated] + buildings_evacuated: type: integer nullable: true - construction_type: + evacuation_delay_reason: type: string nullable: true - stories: + evacuation_completion_minutes: type: integer nullable: true - area_sqft: - type: number + people_sheltering_in_place: + type: integer nullable: true - occupancy_at_time: + people_in_temporary_shelters: + type: integer + nullable: true + people_trapped: type: integer nullable: true + people_missing: + type: integer + nullable: true + people_displaced: + type: integer + nullable: true + description: People who cannot return to the property + x-aliases: [displaced, made homeless, rendered homeless] + displacement_causes: + type: array + items: + type: string -Casualties: +# --- EMS --- + +EMS: type: object properties: - civilian: + ems_response_required: + type: boolean + patients: type: array items: - $ref: "#/CivilianCasualty" - responder: + $ref: "#/EMSPatient" + total_patients: + type: integer + x-aliases: [patients, number of patients, casualties treated] + ems_agency_responded: + type: string + nullable: true + nemsis_report_required: + type: boolean + nemsis_report_ids: type: array + description: Linked ePCR ids (NEMSIS eRecord.01); the full ePCR stays in the EMS system items: - $ref: "#/ResponderCasualty" - total_civilian_injuries: - type: integer - total_civilian_fatalities: - type: integer - total_responder_injuries: - type: integer - total_responder_fatalities: - type: integer + type: string -CivilianCasualty: +EMSPatient: type: object + description: Summary-level patient record; the ePCR is the clinical record properties: - age: + patient_ref_id: + type: string + nemsis_report_id: + type: string + nullable: true + age_approx: type: integer nullable: true - sex: + x-pii: true + date_of_birth: type: string + format: date nullable: true - injury_type: + x-pii: true + sex: type: string - severity: - $ref: "../schemas/enums.yaml#/InjurySeverity" - cause: + nullable: true + x-pii: true + chief_complaint: type: string nullable: true - location_at_time: + provider_impression: type: string nullable: true - transported: - type: boolean - hospital: + description: Provider's primary impression/assessment + injury_intent: type: string + enum: [accidental, self_inflicted, inflicted_by_other, undetermined] nullable: true - -ResponderCasualty: - type: object - properties: - personnel_id: + body_sites: + type: array + description: Injured body sites with injury type per site + items: + type: object + properties: + site: + type: string + injury_type: + type: string + procedures: + type: array + description: Procedures performed on scene (CPR, oxygen, splinting, defibrillation) + items: + type: string + cardiac_arrest: + type: object + properties: + occurred: + type: boolean + pre_arrival: + type: boolean + nullable: true + witnessed: + type: boolean + nullable: true + bystander_cpr: + type: boolean + nullable: true + initial_rhythm: + type: string + nullable: true + safety_equipment_used: + type: array + description: Safety equipment used by the patient (seat belt, airbag, helmet) + items: + type: string + highest_care_level_on_scene: type: string - agency: + enum: [first_responder, emt_basic, emt_intermediate, paramedic, physician, other] + nullable: true + patient_status: type: string - role: + enum: [improved, unchanged, worsened] + nullable: true + at_patient_datetime: type: string - injury_type: + format: date-time + nullable: true + transfer_of_care_datetime: type: string - severity: - $ref: "../schemas/enums.yaml#/InjurySeverity" - treatment: + format: date-time + nullable: true + disposition: type: string + enum: [treated_and_transported_by_fd, transported_by_other_agency, treated_no_transport, refused_care, dead_at_scene, transferred_care, other] nullable: true transported: type: boolean - hospital: - type: string - nullable: true - return_to_duty_date: + hospital_destination: type: string - format: date nullable: true - osha_recordable: - type: boolean - nfirs_5_required: - type: boolean -EMS: +# --- Hazmat --- + +Hazmat: type: object properties: - ems_response_required: + involved: type: boolean - patients: + materials: type: array items: - $ref: "#/EMSPatient" - total_patients: - type: integer - ems_agency_responded: + $ref: "#/HazmatMaterial" + ignition_or_release_first: + type: string + enum: [ignition_first, release_first, no_fire, undetermined] + nullable: true + release_cause: + type: string + enum: [intentional, unintentional, container_failure, act_of_nature, cause_under_investigation, undetermined] + nullable: true + release_factors: + type: array + items: + type: string + mitigation_factors: + type: array + description: Factors or impediments that affected mitigation + items: + type: string + actions_taken: + type: array + description: Hazmat-specific actions (identification, containment, decontamination, neutralization) + items: + type: string + equipment_involved_in_release: + type: object + properties: + involved: + type: boolean + equipment_type: + type: string + nullable: true + brand: + type: string + nullable: true + model: + type: string + nullable: true + year: + type: integer + nullable: true + area_affected: + $ref: "#/Quantity" + area_evacuated: + $ref: "#/Quantity" + epa_reportable_quantity_exceeded: + type: boolean + disposition: + type: string + enum: [completed_by_fire_service, completed_with_fire_service_present, released_to_local_agency, released_to_state_agency, released_to_federal_agency, released_to_private_contractor, released_to_owner, undetermined] + nullable: true + description: Who the cleanup/scene was released to. Evacuee counts live in evacuation_displacement. + +HazmatMaterial: + type: object + properties: + name: + type: string + x-aliases: [chemical, substance, hazardous material, chemical name] + un_number: + type: string + nullable: true + x-aliases: [un number, un id] + dot_hazard_class: + type: string + nullable: true + description: DOT/UN hazard class and division, e.g. "3" or "2.1" + cas_number: + type: string + nullable: true + physical_state: + type: string + enum: [solid, liquid, gas, undetermined] + nullable: true + container_type: + type: string + nullable: true + container_capacity: + $ref: "#/Quantity" + released: + type: boolean + nullable: true + amount_released: + $ref: "#/Quantity" + released_into: type: string + enum: [air, water, soil, contained_on_site, sewer_drain, other, undetermined] nullable: true - nemsis_report_required: + released_from_story: + type: integer + nullable: true + released_inside_structure: type: boolean - nemsis_report_ids: - type: array - items: - type: string + nullable: true -EMSPatient: +# --- Emerging hazards --- + +EmergingHazard: type: object + description: Stored-energy and similar emerging hazards (NERIS emerging_hazard module) properties: - patient_ref_id: + category: type: string - age_approx: - type: integer - nullable: true - sex: + enum: [battery_energy_storage, electric_vehicle, micromobility_device, consumer_electronics, photovoltaic_system, power_generation, csst_gas_tubing, other] + subtype: type: string nullable: true - chief_complaint: + source_or_target: type: string + enum: [ignition_source, target_only, both, undetermined] nullable: true - disposition: + suppression_approach: type: string nullable: true - transported: + reignition_occurred: type: boolean - date_of_birth: - type: string - format: date nullable: true - nemsis_data_captured: + ev_crash_involved: type: boolean + nullable: true + description: Electric vehicle was involved in a crash + lightning_suspected: + type: boolean + nullable: true + description: CSST cases, lightning as suspected cause + notes: + type: string + nullable: true -Hazmat: +# --- Investigation (includes arson) --- + +Investigation: type: object properties: - involved: + investigation_needed: type: boolean - materials: + description: Incident commander's assessment that formal investigation is required + investigation_types: type: array + description: Types of investigation completed (origin_and_cause, arson, insurance, forensic) items: - type: object - properties: - name: - type: string - un_number: - type: string - nullable: true - quantity: - type: string - nullable: true - epa_reportable_quantity_exceeded: + type: string + investigation_ongoing: type: boolean - spill_size_gallons: - type: number nullable: true - -Arson: - type: object - properties: - suspected: - type: boolean - confirmed: - type: boolean + case_status: + type: string + enum: [open, closed_with_arrest, closed_exceptional, closed, inactive] + nullable: true + agency_referred_to: + type: object + properties: + name: + type: string + case_number: + type: string + nullable: true law_enforcement_notified: type: boolean - investigation_required: + evidence_collected: type: boolean - investigation_agency: + laboratory_used: type: string nullable: true - evidence_collected: - type: boolean nibrs_report_required: type: boolean + arson: + $ref: "#/Arson" notes: type: string nullable: true -RespondingAgencies: +Arson: type: object properties: - primary_agency: + suspected: + type: boolean + confirmed: + type: boolean + motivation_factors: + type: array + description: Suspected motivations (fraud, intimidation, concealment, thrill, protest) + items: + type: string + group_involvement: type: string - all_agencies: + nullable: true + entry_method: + type: string + nullable: true + extent_of_involvement_on_arrival: + type: string + nullable: true + incendiary_device: + type: object + properties: + container: + type: string + nullable: true + ignition_delay_mechanism: + type: string + nullable: true + fuel: + type: string + nullable: true + material_availability: + type: string + enum: [transported_to_scene, available_at_scene, undetermined] + nullable: true + initial_observations: type: array + description: Scene observations (forced entry, doors locked, security system state) items: - $ref: "#/RespondingAgency" - mutual_aid_activated: - type: boolean - mutual_aid_agencies: + type: string + other_indicators: type: array + description: Contextual indicators (vacancy, for sale, insurance change, financial problems) items: type: string - unified_command: - type: boolean + juvenile_firesetter: + type: object + properties: + involved: + type: boolean + subjects: + type: array + items: + type: object + properties: + age: + type: integer + nullable: true + x-pii: true + sex: + type: string + nullable: true + x-pii: true + family_type: + type: string + nullable: true + x-pii: true + risk_factors: + type: array + items: + type: string + disposition: + type: string + nullable: true -RespondingAgency: +# --- Persons involved --- + +PersonInvolved: type: object + description: Owner, occupant or other party connected to the incident (not casualties) properties: - agency_name: + role: type: string - agency_type: + enum: [owner, occupant, tenant, reporting_party, responsible_party, witness, business_representative, insurance_holder, other] + name: type: string - role: + nullable: true + x-pii: true + x-aliases: [owner name, occupant name, person name] + business_name: type: string - personnel_count: - type: integer + nullable: true + address: + type: string + nullable: true + x-pii: true + same_address_as_incident: + type: boolean + nullable: true + phone: + type: string + nullable: true + x-pii: true + email: + type: string + nullable: true + x-pii: true + insurance: + type: object + properties: + insured: + type: boolean + nullable: true + company: + type: string + nullable: true + x-aliases: [insurance company, insurer] + policy_number: + type: string + nullable: true + x-pii: true -ResourcesDeployed: +# --- Mobile property / vehicles --- + +MobileProperty: type: object properties: - total_personnel: + involvement: + type: string + enum: [ignition_source_and_burned, ignition_source_not_burned, burned_not_ignition_source, collision, hazmat_release, rescued_from, threatened_only, other] + nullable: true + property_type: + type: string + enum: [passenger_car, motorcycle, bus, heavy_goods_vehicle, agricultural_vehicle, construction_vehicle, recreational_vehicle, train_rail, boat_vessel, aircraft, trailer, mobile_home, other] + nullable: true + make: + type: string + nullable: true + x-aliases: [vehicle make, manufacturer] + model: + type: string + nullable: true + x-aliases: [vehicle model] + year: type: integer - personnel_breakdown: + nullable: true + fuel_type: + type: string + enum: [petrol_gasoline, diesel, electric, hybrid, hydrogen, cng_lpg, other, undetermined] + nullable: true + license_plate: + type: string + nullable: true + x-pii: true + x-aliases: [registration, number plate, reg number, plate number] + license_region: + type: string + nullable: true + description: Registering state/province/country + vin: + type: string + nullable: true + x-pii: true + x-aliases: [chassis number, vehicle identification number] + dot_icc_number: + type: string + nullable: true + reported_stolen: + type: boolean + nullable: true + appeared_abandoned: + type: boolean + nullable: true + occupants: + type: integer + nullable: true + extrication: type: object + description: Extrication from this vehicle (UK IRS RTC block) properties: - firefighters: - type: integer - crew_supervisors: - type: integer - engineers: - type: integer - incident_command: - type: integer - support_staff: + performed: + type: boolean + method: + type: string + nullable: true + vehicle_position: + type: string + nullable: true + time_taken_minutes: type: integer - apparatus: - type: array - items: - $ref: "#/Apparatus" - crew_types: - type: array - items: - type: string + nullable: true + +# --- Losses --- -Apparatus: +Losses: type: object + description: Monetary values. Currency follows the reporting agency. properties: - type: - type: string - count: - type: integer + no_loss: + type: boolean + nullable: true + property_loss: + allOf: + - $ref: "#/Money" + x-aliases: [property damage, damage estimate, estimated loss, money lost, loss value, damage cost] + contents_loss: + allOf: + - $ref: "#/Money" + x-aliases: [contents damage, contents value lost] + pre_incident_property_value: + $ref: "#/Money" + pre_incident_contents_value: + $ref: "#/Money" + property_saved: + allOf: + - $ref: "#/Money" + x-aliases: [value saved, save value] + other_costs: + $ref: "#/Money" + estimate_method: + type: string + enum: [rough_estimate, owner_estimate, insurance_assessment, investigator_assessment, official_valuation, other] + nullable: true + +# --- Weather --- Weather: type: object @@ -721,7 +2825,11 @@ Weather: on_arrival: $ref: "#/WeatherReading" worst_conditions: - $ref: "#/WeatherReadingExtended" + $ref: "#/WeatherReading" + weather_type: + type: string + enum: [clear, cloudy, rain, snow_ice, fog, high_winds, thunderstorm_lightning, extreme_heat, extreme_cold, other] + nullable: true factors_influencing_fire: type: array items: @@ -733,38 +2841,30 @@ WeatherReading: datetime: type: string format: date-time - temperature_f: + temperature_c: type: number + x-aliases: [temperature, temp] relative_humidity_percent: type: number - wind_speed_mph: + x-aliases: [humidity, rh] + wind_speed_kph: + type: number + x-aliases: [wind, wind speed] + wind_gusts_kph: type: number + nullable: true wind_direction: type: string haines_index: type: integer nullable: true -WeatherReadingExtended: - type: object - properties: - datetime: - type: string - format: date-time - temperature_f: - type: number - relative_humidity_percent: - type: number - wind_speed_mph: - type: number - wind_gusts_mph: - type: number - nullable: true +# --- Environmental impact --- EnvironmentalImpact: type: object properties: - wildlife_habitat_affected_acres: + habitat_affected_ha: type: number nullable: true watershed_impact: @@ -784,6 +2884,8 @@ EnvironmentalImpact: type: boolean nullable: true +# --- Infrastructure impact --- + InfrastructureImpact: type: object properties: @@ -795,7 +2897,7 @@ InfrastructureImpact: InfrastructureItem: type: object properties: - type: + infrastructure_type: type: string unit: type: string @@ -804,6 +2906,8 @@ InfrastructureItem: severity: type: string +# --- Near miss and safety --- + NearMissAndSafety: type: object properties: @@ -829,11 +2933,125 @@ NearMissAndSafety: nullable: true safety_breaches: type: integer + maydays_count: + type: integer + nullable: true + attacks_on_personnel: + type: object + description: Attacks on responders travelling to, at, or from the incident (UK IRS 3.10-3.13) + properties: + occurred: + type: boolean + attack_type: + type: string + enum: [verbal_abuse, physical_no_weapon, weapon, objects_thrown, vehicle_used, other] + nullable: true + serious_injuries: + type: integer + nullable: true + slight_injuries: + type: integer + nullable: true weather_related_risks: type: array items: type: string +# --- Situation status (ICS-209 layer) --- + +SituationStatus: + type: object + description: | + Evolving large-incident status for sitreps (ICS-209 shape). Person counts + live in casualties and evacuation_displacement; structure counts live in + structure. Mappers compose the ICS-209 matrices from those sections. + properties: + report_version: + type: string + enum: [initial, update, final] + nullable: true + report_number: + type: integer + nullable: true + period_from: + type: string + format: date-time + nullable: true + period_to: + type: string + format: date-time + nullable: true + complexity_level: + type: string + enum: [type_5, type_4, type_3, type_2, type_1] + nullable: true + imt_type: + type: string + nullable: true + description: Incident management organization (single resource, type 3 IMT, unified command) + significant_events: + type: string + nullable: true + primary_hazards: + type: string + nullable: true + description: Primary materials or hazards involved + threat_management: + type: array + description: Active protective actions + items: + type: string + enum: [no_likely_threat, potential_future_threat, mass_notifications_in_progress, mass_notifications_completed, no_evacuations_imminent, planning_for_evacuation, planning_for_shelter_in_place, evacuations_in_progress, shelter_in_place_in_progress, repopulation_in_progress, area_restriction_in_effect, other] + projected_activity: + type: object + description: Projected incident activity by timeframe + properties: + next_12_hours: + type: string + nullable: true + next_24_hours: + type: string + nullable: true + next_48_hours: + type: string + nullable: true + next_72_hours: + type: string + nullable: true + beyond_72_hours: + type: string + nullable: true + strategic_objectives: + type: string + nullable: true + threat_summary: + type: string + nullable: true + critical_resource_needs: + type: array + items: + type: string + planned_actions: + type: string + nullable: true + projected_final_size_ha: + type: number + nullable: true + anticipated_completion_date: + type: string + format: date + nullable: true + demobilization_start_date: + type: string + format: date + nullable: true + costs_to_date: + $ref: "#/Money" + projected_final_cost: + $ref: "#/Money" + +# --- Lessons learned --- + LessonsLearned: type: object properties: @@ -850,6 +3068,8 @@ LessonsLearned: items: type: string +# --- Follow up --- + FollowUp: type: object properties: @@ -866,9 +3086,9 @@ FollowUp: rehabilitation: type: object properties: - erosion_control_acres: + erosion_control_ha: type: number - reseeding_acres: + reseeding_ha: type: number hazard_tree_removal_required: type: boolean @@ -876,8 +3096,8 @@ FollowUp: type: string format: date nullable: true - investigation_ongoing: - type: boolean + +# --- Periodic reporting --- PeriodicReporting: type: object @@ -901,6 +3121,8 @@ PeriodicReporting: format: date-time nullable: true +# --- Attachments --- + Attachments: type: object properties: diff --git a/contracts/schemas/incident-record.yaml b/contracts/schemas/incident-record.yaml index 686e7ce8..b2920acf 100644 --- a/contracts/schemas/incident-record.yaml +++ b/contracts/schemas/incident-record.yaml @@ -1,7 +1,22 @@ # Incident management schemas +# +# IncidentRecord is the Postgres row that OWNS the JSONB incident contract, +# the single store of incident data. A draft incident row is created +# automatically when extraction completes; review corrections (PATCH +# /extract) write into this row's document, and form generation reads from +# it by incident_id. Nothing else (extractions included) keeps a copy. +# The analytics block holds stats promoted out of the JSONB so long-period +# queries (monthly/annual counts, response-time trends, loss totals) run on +# indexed columns without digging into the document. Every analytics field is +# nullable and recomputed server-side from the contract whenever the document +# changes; clients never write them directly. CreateIncidentRequest: type: object + description: Finalizes the draft incident that was auto-created when the + extraction completed, assigning the department's incident number and tags. + It does not create a second row; the extract_id resolves to the existing + draft. required: - extract_id properties: @@ -54,10 +69,22 @@ IncidentRecord: incident_type: type: string nullable: true - incident_date: + description: Primary incident subcategory as free text + incident_category: + allOf: + - $ref: "../schemas/enums.yaml#/IncidentCategory" + nullable: true + description: Primary incident category, promoted for grouping + incident_datetime: type: string - format: date + format: date-time nullable: true + description: When the incident happened, promoted for date-range queries + and sorting. Derived from the contract on every document change, using + incident.alarm_datetime, falling back to incident.start_datetime, + then dispatch.call_received_datetime. + analytics: + $ref: "#/IncidentAnalytics" forms_generated: type: array items: @@ -88,6 +115,70 @@ IncidentRecord: format: date-time nullable: true +IncidentAnalytics: + type: object + description: | + Read-only stats promoted from the incident contract into queryable + columns. Recomputed server-side on every contract change. + properties: + city: + type: string + nullable: true + state: + type: string + nullable: true + description: State, province or region + country: + type: string + nullable: true + description: ISO 3166-1 alpha-2 + civilian_injuries: + type: integer + nullable: true + civilian_fatalities: + type: integer + nullable: true + responder_injuries: + type: integer + nullable: true + responder_fatalities: + type: integer + nullable: true + people_rescued: + type: integer + nullable: true + people_evacuated: + type: integer + nullable: true + structures_destroyed: + type: integer + nullable: true + area_burned_ha: + type: number + nullable: true + total_loss_amount: + type: number + nullable: true + description: property_loss + contents_loss from the contract + total_loss_currency: + type: string + nullable: true + description: ISO 4217 + call_to_arrival_seconds: + type: integer + nullable: true + description: Call received (or alarm) to first arrival + turnout_seconds_first_unit: + type: integer + nullable: true + travel_seconds_first_unit: + type: integer + nullable: true + on_scene_duration_seconds: + type: integer + nullable: true + description: First arrival to last unit cleared + IncidentRecordFull: description: Full incident record with linked extraction and forms allOf: @@ -137,9 +228,19 @@ IncidentListResponse: incident_type: type: string nullable: true - incident_date: + incident_category: + allOf: + - $ref: "../schemas/enums.yaml#/IncidentCategory" + nullable: true + incident_datetime: + type: string + format: date-time + nullable: true + city: + type: string + nullable: true + country: type: string - format: date nullable: true forms_count: type: integer diff --git a/contracts/schemas/system.yaml b/contracts/schemas/system.yaml index 8c256932..74d5b3c0 100644 --- a/contracts/schemas/system.yaml +++ b/contracts/schemas/system.yaml @@ -22,7 +22,7 @@ HealthStatus: properties: database: $ref: "#/ComponentHealth" - ollama: + llm: $ref: "#/ComponentHealth" whisper: $ref: "#/ComponentHealth" @@ -46,43 +46,43 @@ ComponentHealth: detail: type: string nullable: true - model_loaded: - type: string - nullable: true - description: Currently loaded model (ollama component) disk_free_gb: type: number nullable: true description: Free disk space (storage component) - ollama_version: + provider: + type: string + nullable: true + description: Configured LLM provider (llm component) + enum: + - ollama + - openai + - gemini + - anthropic + - custom + model: type: string nullable: true - description: Ollama server version (ollama component) + description: Model this deployment sends prompts to (llm component) + external: + type: boolean + nullable: true + description: > + True when prompts leave this machine, which is the case for every + hosted provider (llm component) + probed: + type: boolean + nullable: true + description: > + Whether the status came from a live call. Hosted providers are not + probed, because a call on every health check spends quota to answer a + question the configuration already answers (llm component) models_available: type: array nullable: true - description: All models pulled and available on the Ollama server (ollama component) + description: Models the provider will serve (llm component) items: - type: object - properties: - name: - type: string - size_gb: - type: number - quantization: - type: string - nullable: true - loaded: - type: boolean - current_load: - type: object - nullable: true - description: Current processing load (ollama component) - properties: - active_requests: - type: integer - queued_requests: - type: integer + type: string SchemaVersion: type: object diff --git a/contracts/schemas/template-record.yaml b/contracts/schemas/template-record.yaml new file mode 100644 index 00000000..0ce81112 --- /dev/null +++ b/contracts/schemas/template-record.yaml @@ -0,0 +1,429 @@ +# Template & Configuration schemas + +TemplateSummary: + type: object + properties: + template_id: + type: string + format: uuid + form_type: + type: string + description: Unique form type identifier (built-in or custom jurisdiction) + display_name: + type: string + jurisdiction: + type: string + nullable: true + description: Jurisdiction code (e.g. "US-Federal", "US-CA", "US-GA") + agency_type: + type: string + nullable: true + version: + type: string + last_updated: + type: string + format: date + field_count: + type: integer + status: + type: string + enum: + - active + - legacy + - draft + +CreateTemplateRequest: + type: object + required: + - form_type + - display_name + - fields + properties: + form_type: + type: string + description: Unique form type identifier + display_name: + type: string + jurisdiction: + type: string + nullable: true + description: Jurisdiction code. Optional - the visual editor may register a + template before jurisdiction is assigned. + agency_type: + type: string + nullable: true + fields: + type: array + items: + $ref: "#/TemplateField" + source_standard: + type: string + nullable: true + description: Reference to the source standard (e.g. "NERIS v2.0", "NFIRS 5.0") + pdf_template_ref: + type: string + nullable: true + description: Reference to the PDF template file the layout coordinates apply to + +Template: + allOf: + - type: object + properties: + template_id: + type: string + format: uuid + version: + type: string + last_updated: + type: string + format: date + field_count: + type: integer + status: + type: string + enum: [active, legacy, draft] + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + - $ref: "#/CreateTemplateRequest" + +TemplateField: + type: object + description: | + One box on the form. The `source` decides how it gets its value: + - source=schema: `incident_mapping` is required, value is a contract lookup. + - source=static: `static_text` is required, stamped as-is on every form. + - source=manual: no mapping. User types the value on the review screen; + it is stored under custom_fields."{form_type}.{field_name}" in the + contract and filled from there. + - source=open: no mapping. `description` is required and is the + instruction the LLM uses to extract the value during the extraction + layer. The value is stored under custom_fields."{form_type}.{field_name}" + and reviewed like any other extracted value. + required: + - field_name + - field_type + - source + - required + properties: + field_name: + type: string + description: Field identifier within this template + field_type: + type: string + enum: + - string + - integer + - number + - boolean + - date + - datetime + - time + - enum + - text + - array + description: Data type of the field + source: + $ref: "../schemas/enums.yaml#/FieldSource" + required: + type: boolean + description: + type: string + nullable: true + description: | + Help text for the field. For source=open this is required and doubles + as the extraction instruction sent to the LLM, so write it as a plain + statement of what to pull from the narrative (e.g. "Name of the + insurance company covering the property"). + max_length: + type: integer + nullable: true + min_value: + type: number + nullable: true + max_value: + type: number + nullable: true + allowed_values: + type: array + items: + type: string + nullable: true + description: Valid values for enum fields + incident_mapping: + type: string + nullable: true + description: | + JSON path in the FireForm incident contract this field pulls its value + from (e.g. "fire.cause_category"). Required when source=schema, null + for every other source. + static_text: + type: string + nullable: true + description: | + Fixed text drawn into the field. Required when source=static, null for + every other source. + default_value: + nullable: true + description: Default value if the mapped incident field is null + unit: + type: string + nullable: true + description: | + Target unit for the stamped value when the form wants something other + than the incident contract's SI unit (e.g. "acres" for a field mapped + to wildland.area_burned_ha, "fahrenheit" for temperature_c). The + mapper converts during generation. Null means stamp the value as + stored. + layout: + nullable: true + description: Visual placement of the field on the PDF. Null for fields with + no fixed position. + allOf: + - $ref: "#/TemplateFieldLayout" + +TemplateFieldLayout: + type: object + description: | + Visual placement of a field on the PDF page. Coordinates are in PDF points + with the origin at the bottom-left of the page, so the field box runs from + start = (x, y) to end = (x + width, y + height). The editor is responsible + for converting rendered-canvas pixels to PDF points before saving. + required: + - page + - x + - y + - width + - height + properties: + page: + type: integer + description: Zero-based page index + x: + type: number + description: Lower-left X of the box, in PDF points + y: + type: number + description: Lower-left Y of the box, in PDF points (origin bottom-left) + width: + type: number + height: + type: number + font: + type: string + default: Helvetica + font_size: + type: number + default: 10 + color: + type: string + default: "#000000" + description: Hex color, e.g. "#000000" + align: + type: string + enum: + - left + - center + - right + default: left + +TemplateDraft: + type: object + description: | + Result of uploading a PDF for template authoring. Carries everything the + visual editor needs: the stored PDF reference, page geometry, and the + fields auto-detected by commonforms with mapping suggestions. Each + detected field is already a full TemplateField draft, so the editor can + let the user adjust boxes, change sources, accept or override suggested + mappings, add or delete fields, and then send the exact same array in + POST /api/v1/templates. Fields the detector could not interpret still + come back with a layout box and empty mapping; coordinates alone are a + valid starting point. + required: + - upload_id + - status + - pdf_template_ref + - page_count + - pages + properties: + upload_id: + type: string + format: uuid + status: + type: string + enum: + - processing + - completed + - failed + description: Field detection status. The PDF itself is stored as soon + as the upload returns; only detection is asynchronous. + pdf_template_ref: + type: string + description: Opaque reference to the stored PDF, passed back in the + template body as pdf_template_ref + original_filename: + type: string + page_count: + type: integer + pages: + type: array + description: Per-page geometry in PDF points, index 0 = first page + items: + type: object + required: + - page + - width + - height + properties: + page: + type: integer + width: + type: number + height: + type: number + detected_fields: + type: array + description: Present when status is completed. Empty when the detector + found nothing; the user then draws boxes from scratch. + items: + $ref: "#/DraftField" + detection_error: + type: string + nullable: true + description: Present when status is failed. The upload is still usable; + the editor falls back to manual box drawing. + retry_after_seconds: + type: integer + description: Polling hint, present while status is processing + +DraftField: + type: object + description: | + One auto-detected field plus ranked mapping suggestions. + + Detected labels are messy ("Incident No.:", stray colons, no text at + all), so the suggester normalizes the label before scoring: lowercase, + strip punctuation, collapse whitespace, expand common form abbreviations + ("no" → "number", "dt" → "date", "addr" → "address", "tel" → "phone", + "dob" → "date of birth"). Scoring then runs against the schema catalog + (names, x-aliases and descriptions, same index and scorer as + GET /api/v1/schema/fields). + + Two thresholds apply. Below the confidence floor, `suggestions` is empty: + an empty list with a good search box beats a wrong guess, because users + trust pre-filled values too much. Above the higher auto-apply threshold, + the top suggestion is pre-applied as the field's incident_mapping and + source is set to schema; in between, suggestions are listed but nothing + is pre-applied. + required: + - field + properties: + field: + $ref: "#/TemplateField" + detected_label: + type: string + nullable: true + description: Raw label text found near the box on the PDF, kept verbatim + (not normalized) so the editor can show what the suggestion was based on + suggestions: + type: array + description: Ranked incident-contract mapping suggestions for this box, + best first. Empty when nothing scored above the confidence floor. + items: + $ref: "#/MappingSuggestion" + +MappingSuggestion: + type: object + required: + - path + - score + properties: + path: + type: string + description: Incident-contract JSON path (e.g. "location.postal_code") + label: + type: string + description: Human-friendly name of the schema field + field_type: + type: string + description: Data type of the schema field + section: + type: string + description: Top-level contract section the path belongs to + description: + type: string + description: The schema field's own description from the contract + score: + type: number + description: Match confidence 0-1 from the fuzzy matcher (no LLM) + +SchemaFieldEntry: + type: object + description: One searchable entry in the incident-contract field catalog. + required: + - path + - field_type + - section + properties: + path: + type: string + description: Dotted JSON path into the incident contract + label: + type: string + description: Human-friendly name derived from the field name + field_type: + type: string + section: + type: string + description: Top-level contract section (fire, location, casualties, ...) + description: + type: string + nullable: true + enum_values: + type: array + items: + type: string + nullable: true + pii: + type: boolean + description: True when the contract marks this field x-pii + aliases: + type: array + items: + type: string + description: Alternate names the search index also matches (zip for + postal_code, victim for casualty, and similar). Sourced from the + field's x-aliases entry in incident-contract.yaml - the contract is + the single place aliases are defined; the catalog builder reads them + from the schema file at startup, nothing re-declares them in code. + +SchemaFieldSearchResponse: + type: object + required: + - total + - fields + properties: + query: + type: string + nullable: true + description: Echo of the q parameter, null when listing the full catalog + total: + type: integer + schema_version: + type: string + description: Incident-contract version the catalog was built from + fields: + type: array + description: Ranked matches when q is given, full catalog otherwise + items: + allOf: + - $ref: "#/SchemaFieldEntry" + - type: object + properties: + score: + type: number + nullable: true + description: Present only for search results diff --git a/contracts/schemas/template.yaml b/contracts/schemas/template.yaml deleted file mode 100644 index f9ce9214..00000000 --- a/contracts/schemas/template.yaml +++ /dev/null @@ -1,203 +0,0 @@ -# Template & Configuration schemas - -TemplateSummary: - type: object - properties: - template_id: - type: string - format: uuid - form_type: - type: string - description: Unique form type identifier (built-in or custom jurisdiction) - display_name: - type: string - jurisdiction: - type: string - nullable: true - description: Jurisdiction code (e.g. "US-Federal", "US-CA", "US-GA") - agency_type: - type: string - nullable: true - version: - type: string - last_updated: - type: string - format: date - field_count: - type: integer - status: - type: string - enum: - - active - - legacy - - draft - -CreateTemplateRequest: - type: object - required: - - form_type - - display_name - - fields - properties: - form_type: - type: string - description: Unique form type identifier - display_name: - type: string - jurisdiction: - type: string - nullable: true - description: Jurisdiction code. Optional — the visual editor may register a - template before jurisdiction is assigned. - agency_type: - type: string - nullable: true - fields: - type: array - items: - $ref: "#/TemplateField" - source_standard: - type: string - nullable: true - description: Reference to the source standard (e.g. "NERIS v2.0", "NFIRS 5.0") - pdf_template_ref: - type: string - nullable: true - description: Reference to the PDF template file the layout coordinates apply to - -Template: - allOf: - - type: object - properties: - template_id: - type: string - format: uuid - version: - type: string - last_updated: - type: string - format: date - field_count: - type: integer - status: - type: string - enum: [active, legacy, draft] - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - - $ref: "#/CreateTemplateRequest" - -TemplateField: - type: object - required: - - field_name - - field_type - - required - properties: - field_name: - type: string - description: Field identifier within this template - field_type: - type: string - enum: - - string - - integer - - number - - boolean - - date - - datetime - - time - - enum - - text - - array - description: Data type of the field - required: - type: boolean - description: - type: string - nullable: true - max_length: - type: integer - nullable: true - min_value: - type: number - nullable: true - max_value: - type: number - nullable: true - allowed_values: - type: array - items: - type: string - nullable: true - description: Valid values for enum fields - incident_mapping: - type: string - nullable: true - description: | - JSON path in the FireForm incident schema this field pulls its value from - (e.g. "fire.cause_category"). Null for a static field. Exactly one of - incident_mapping or static_text is expected. - static_text: - type: string - nullable: true - description: | - Fixed text drawn into the field instead of a mapped value. Null for a - data-mapped field. Mutually exclusive with incident_mapping. - default_value: - nullable: true - description: Default value if the mapped incident field is null - layout: - nullable: true - description: Visual placement of the field on the PDF. Null for fields with - no fixed position. - allOf: - - $ref: "#/TemplateFieldLayout" - -TemplateFieldLayout: - type: object - description: | - Visual placement of a field on the PDF page. Coordinates are in PDF points - with the origin at the bottom-left of the page, so the field box runs from - start = (x, y) to end = (x + width, y + height). The editor is responsible - for converting rendered-canvas pixels to PDF points before saving. - required: - - page - - x - - y - - width - - height - properties: - page: - type: integer - description: Zero-based page index - x: - type: number - description: Lower-left X of the box, in PDF points - y: - type: number - description: Lower-left Y of the box, in PDF points (origin bottom-left) - width: - type: number - height: - type: number - font: - type: string - default: Helvetica - font_size: - type: number - default: 10 - color: - type: string - default: "#000000" - description: Hex color, e.g. "#000000" - align: - type: string - enum: - - left - - center - - right - default: left diff --git a/data/forms/generated/.gitkeep b/data/forms/generated/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/.env.example b/docker/.env.example index d5ccca58..3bb20f60 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -12,7 +12,86 @@ DATABASE_URL=postgresql://fireform:fireform@localhost:5432/fireform # Override to point at an external Ollama instance (e.g. a GPU server). OLLAMA_HOST=http://ollama:11434 OLLAMA_MODEL=qwen2.5:1.5b -OLLAMA_TIMEOUT=300 +OLLAMA_TIMEOUT=600 +# How many chunk prompts the extraction worker sends at once. Keep it in step +# with the Ollama server's own parallelism; going wider only queues requests. +# LLM_MAX_PARALLEL overrides this for any provider. +OLLAMA_NUM_PARALLEL=4 +# Ceiling on tokens per section answer. Keep it in step with OLLAMA_TIMEOUT: a +# section that runs past the timeout is lost, while one that hits this ceiling +# keeps every field it finished before the cut. A small model on CPU manages +# about three tokens a second. +OLLAMA_MAX_TOKENS=1200 + +# --- LLM provider --------------------------------------------------------- +# Which backend answers prompts. One provider for the whole deployment. +# ollama the local models above, and the default +# openai needs OPENAI_API_KEY +# gemini needs GEMINI_API_KEY +# anthropic needs ANTHROPIC_API_KEY +# custom any endpoint that serves the OpenAI chat completions API +# (vLLM, LM Studio, llama.cpp, Groq, OpenRouter, a company proxy). +# Needs LLM_BASE_URL. +LLM_PROVIDER=ollama + +# Model name. Blank falls back to OLLAMA_MODEL, and only for Ollama. Every +# other provider must name one, because a vendor model name pinned in our +# source goes stale. +LLM_MODEL= + +# Endpoint override. Required by LLM_PROVIDER=custom, for example +# http://localhost:8001/v1. Ollama falls back to OLLAMA_HOST when this is blank. +LLM_BASE_URL= + +# One key per provider, so several can sit here and LLM_PROVIDER alone decides +# which is used. LLM_API_KEY belongs to the custom endpoint and can stay empty +# for an endpoint that needs no auth. +OPENAI_API_KEY= +GEMINI_API_KEY= +ANTHROPIC_API_KEY= +LLM_API_KEY= + +# Sending narratives to a hosted provider means names, addresses and medical +# detail leave this machine. A cloud provider will not start until this is +# switched on deliberately. +LLM_ALLOW_EXTERNAL=false + +# Per call limits. Blank falls back to the OLLAMA_ values above. +LLM_TIMEOUT=600 +LLM_MAX_TOKENS=1200 +# How many prompts the extraction worker sends at once, whichever provider is +# in use. Blank falls back to OLLAMA_NUM_PARALLEL. +LLM_MAX_PARALLEL=4 + +# Rate limits. A 429 is the provider asking us to wait, so wait: ten tries, ten +# seconds apart. A Retry-After header asking for longer is honoured up to the +# ceiling. Anything still limited after that is a quota problem no retry fixes. +LLM_RATE_LIMIT_RETRIES=10 +LLM_RATE_LIMIT_WAIT_SECONDS=10 +LLM_RATE_LIMIT_MAX_WAIT_SECONDS=60 +LLM_RESPECT_RETRY_AFTER=true + +# A 5xx is usually a blip. Retried a couple of times, quickly. +LLM_SERVER_RETRIES=2 +LLM_SERVER_RETRY_WAIT_SECONDS=2 + +# Extra HTTP headers as a JSON object, for a gateway that wants its own header +# instead of a bearer token. Empty in every normal setup. +# LLM_EXTRA_HEADERS={"X-My-Header": "value"} +LLM_EXTRA_HEADERS= + +# --- Extraction ----------------------------------------------------------- +# Extra tries after a chunk's first answer fails validation. +EXTRACTION_CHUNK_RETRIES=1 +# Let the same input be extracted more than once. Development only: it makes +# testing a narrative repeatedly possible without re-uploading it. Each run +# still gets its own extraction and its own draft incident. +EXTRACTION_ALLOW_RERUN=false +# Deployment context the extractor falls back on when the narrative is silent. +# A request can override any of these through the extraction body's "defaults". +FIREFORM_DEFAULT_COUNTRY=US +FIREFORM_DEFAULT_TIMEZONE=UTC +FIREFORM_DEFAULT_CURRENCY=USD # --- Whisper -------------------------------------------------------------- # Whisper runs in Docker with port mapped to host. App runs on host. diff --git a/docker/README.md b/docker/README.md index e238c640..4856c66d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -19,6 +19,15 @@ docker/ See `.env.example` for the full list with descriptions. +Both compose files load the env file into the app and worker containers with an +`env_file` block, so a setting added to `.env.dev` or `.env.prod` reaches the +running code without being listed a second time. The `environment` block below +it still wins, and it is there for the handful of values a container needs +different from the host, such as service names and data directories. + +`--env-file` on the compose command line is a separate thing. It only fills in +`${}` placeholders inside the compose file and puts nothing into the container. + ## Volumes `docker compose down` never removes volumes. Use `docker compose down -v` only to intentionally wipe all data. diff --git a/docker/dev/compose.yml b/docker/dev/compose.yml index 820711d1..a8bd27db 100644 --- a/docker/dev/compose.yml +++ b/docker/dev/compose.yml @@ -23,6 +23,10 @@ services: ollama: image: ollama/ollama:latest container_name: fireform-ollama + environment: + # The server side of the extraction worker's parallelism. Both read the + # same variable so they cannot drift apart. + - OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL:-4} ports: - "127.0.0.1:11434:11434" volumes: @@ -96,14 +100,21 @@ services: - fireform_uploads:/data/uploads ports: - "${APP_PORT:-8000}:8000" + # Everything in the env file reaches the container, so a new setting works + # without being listed twice. The block below still wins where a container + # needs a different value than the host (service names, data dirs). + env_file: + - ../.env.dev environment: - PYTHONUNBUFFERED=1 - CUDA_VISIBLE_DEVICES= - PYTHONPATH=/app - DATABASE_URL=postgresql://fireform:fireform@postgres:5432/fireform - OLLAMA_HOST=${OLLAMA_HOST:-http://ollama:11434} - - OLLAMA_TIMEOUT=${OLLAMA_TIMEOUT:-300} + - OLLAMA_TIMEOUT=${OLLAMA_TIMEOUT:-600} - OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:1.5b} + - OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL:-4} + - OLLAMA_MAX_TOKENS=${OLLAMA_MAX_TOKENS:-1200} - WHISPER_HOST=${WHISPER_HOST:-http://whisper:9000} - FIREFORM_DATA_DIR=/data/uploads - FIREFORM_TEMPLATE_DIR=/data/uploads @@ -130,13 +141,17 @@ services: volumes: - ../..:/app - fireform_uploads:/data/uploads + env_file: + - ../.env.dev environment: - PYTHONUNBUFFERED=1 - PYTHONPATH=/app - DATABASE_URL=postgresql://fireform:fireform@postgres:5432/fireform - OLLAMA_HOST=${OLLAMA_HOST:-http://ollama:11434} - - OLLAMA_TIMEOUT=${OLLAMA_TIMEOUT:-300} + - OLLAMA_TIMEOUT=${OLLAMA_TIMEOUT:-600} - OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:1.5b} + - OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL:-4} + - OLLAMA_MAX_TOKENS=${OLLAMA_MAX_TOKENS:-1200} - FIREFORM_DATA_DIR=/data/uploads - FIREFORM_TEMPLATE_DIR=/data/uploads - CELERY_BROKER_URL=redis://redis:6379/0 diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index ed65a283..44efeeb4 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -31,6 +31,9 @@ COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/pytho COPY --from=builder /usr/local/bin /usr/local/bin COPY app/ ./app/ +# The extraction worker reads the incident contract at startup for its chunk +# tiers and triggers, so the schemas ship with the image. +COPY contracts/schemas/ ./contracts/schemas/ COPY requirements.txt . COPY docker/entrypoint.sh /entrypoint.sh diff --git a/docker/prod/compose.yml b/docker/prod/compose.yml index 9ac8f39f..a94dc7dd 100644 --- a/docker/prod/compose.yml +++ b/docker/prod/compose.yml @@ -56,6 +56,11 @@ services: - fireform_uploads:/data/uploads ports: - "${APP_PORT}:8000" + # Everything in the env file reaches the container, so a new setting works + # without being listed twice. The block below still wins where a container + # needs a different value than the host. + env_file: + - ../.env.prod environment: - PYTHONUNBUFFERED=1 - CUDA_VISIBLE_DEVICES= diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..092631f0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Tools used at development time only, not needed to run the server. +# datamodel-code-generator turns the incident contract into Pydantic models +# (see scripts/generate_contract_models.py, run via `make generate-contract-models`). +datamodel-code-generator==0.25.9 +ruff==0.16.1 diff --git a/requirements.txt b/requirements.txt index 0a800f01..c09489c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,10 @@ psycopg2-binary pytest httpx numpy<2 -ollama +openai pypdf +reportlab +Pillow python-multipart celery[redis] redis diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..c04f5335 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +target-version = "py311" +line-length = 100 + +[lint] +# Pin the rule set explicitly. Ruff's defaults changed in 0.16, so relying on +# them means a version bump silently changes what CI enforces. +select = ["E4", "E7", "E9", "F"] diff --git a/scripts/generate_contract_models.py b/scripts/generate_contract_models.py new file mode 100644 index 00000000..e90ee2ce --- /dev/null +++ b/scripts/generate_contract_models.py @@ -0,0 +1,236 @@ +"""Generate Pydantic models for the incident contract. + +The incident contract (contracts/schemas/incident-contract.yaml) is the single +source of truth for every downstream form. This script turns it into typed +Pydantic v2 models so the extraction worker, validation, and correction paths +get typed access without anyone hand-writing (and then drifting) the models. + +What it does: + +1. The contract file is a flat map of named schemas, not a JSON Schema on its + own, so we wrap every entry under `$defs` with a root `$ref` to + IncidentContract and rewrite the internal `#/X` refs to `#/$defs/X`. +2. datamodel-code-generator turns that into Pydantic v2 models. +3. Enums the contract shares through contracts/schemas/enums.yaml already exist + in app/api/schemas/enums.py. We drop the generated copies and import the + existing ones instead, so there is one definition per enum. If a shared enum + has drifted from enums.py, generation fails loudly and tells you to sync it. + +Run it with `make generate-contract-models` whenever the contract changes. +""" + +from __future__ import annotations + +import ast +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONTRACT = REPO_ROOT / "contracts" / "schemas" / "incident-contract.yaml" +ENUMS_YAML = REPO_ROOT / "contracts" / "schemas" / "enums.yaml" +ENUMS_PY = REPO_ROOT / "app" / "api" / "schemas" / "enums.py" +OUTPUT = REPO_ROOT / "app" / "api" / "schemas" / "incident_contract.py" +ENUMS_IMPORT = "app.api.schemas.enums" +ROOT_MODEL = "IncidentContract" + +HEADER = """# This file is generated from contracts/schemas/incident-contract.yaml. +# DO NOT EDIT BY HAND. Run `make generate-contract-models` to regenerate. +""" + + +def shared_enum_names() -> set[str]: + """Enum names the contract pulls in from enums.yaml (the deliberate shared + set, so a field's inline enum that happens to share a name is left alone).""" + text = CONTRACT.read_text() + names = set() + for line in text.splitlines(): + marker = "../schemas/enums.yaml#/" + if marker in line: + names.add(line.split(marker, 1)[1].strip().strip('"').strip("'")) + return names + + +def enum_members_from_py() -> dict[str, list[str]]: + """Read app/api/schemas/enums.py without importing it (no side effects).""" + tree = ast.parse(ENUMS_PY.read_text()) + out: dict[str, list[str]] = {} + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + if not any(isinstance(b, ast.Name) and b.id == "Enum" for b in node.bases): + continue + out[node.name] = [ + stmt.value.value + for stmt in node.body + if isinstance(stmt, ast.Assign) and isinstance(stmt.value, ast.Constant) + ] + return out + + +def enum_members_from_yaml() -> dict[str, list[str]]: + data = yaml.safe_load(ENUMS_YAML.read_text()) + return { + name: spec["enum"] + for name, spec in data.items() + if isinstance(spec, dict) and "enum" in spec + } + + +def build_wrapped(dest: Path) -> None: + """Wrap the flat contract in a JSON Schema envelope codegen can consume.""" + doc = yaml.safe_load(CONTRACT.read_text()) + + def rewrite(obj): + if isinstance(obj, dict): + return { + k: ( + "#/$defs/" + v[2:] + if k == "$ref" and isinstance(v, str) and v.startswith("#/") + else rewrite(v) + ) + for k, v in obj.items() + } + if isinstance(obj, list): + return [rewrite(i) for i in obj] + return obj + + wrapped = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": ROOT_MODEL, + "$ref": f"#/$defs/{ROOT_MODEL}", + "$defs": rewrite(doc), + } + dest.write_text(yaml.safe_dump(wrapped, sort_keys=False)) + + +def run_codegen(wrapped: Path, out: Path) -> None: + subprocess.run( + [ + "datamodel-codegen", + "--input", str(wrapped), + "--input-file-type", "jsonschema", + "--output", str(out), + "--output-model-type", "pydantic_v2.BaseModel", + "--force-optional", + "--use-schema-description", + "--use-field-description", + "--use-annotated", + "--field-constraints", + "--use-standard-collections", + "--use-double-quotes", + "--disable-timestamp", + "--target-python-version", "3.11", + ], + check=True, + cwd=REPO_ROOT, + ) + + +def strip_shared_enums(source: str, shared: set[str]) -> str: + """Remove generated copies of the shared enums and import them instead. + + A shared enum is only removed when enums.py defines it with the exact same + members; a mismatch means enums.py drifted from the contract, so we stop and + say so rather than silently dropping members. + """ + py_enums = enum_members_from_py() + yaml_enums = enum_members_from_yaml() + + tree = ast.parse(source) + drop_ranges: list[tuple[int, int]] = [] + reused: list[str] = [] + last_import_line = 0 + + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = max(last_import_line, node.end_lineno) + continue + if not isinstance(node, ast.ClassDef) or node.name not in shared: + continue + is_enum = any(isinstance(b, ast.Name) and b.id == "Enum" for b in node.bases) + if not is_enum: + continue + expected = yaml_enums.get(node.name) + if node.name not in py_enums or py_enums[node.name] != expected: + sys.exit( + f"Shared enum {node.name} in enums.py does not match the " + f"contract. Sync app/api/schemas/enums.py to enums.yaml " + f"({expected}) and re-run." + ) + drop_ranges.append((node.lineno, node.end_lineno)) + reused.append(node.name) + + lines = source.splitlines() + drop = {n for start, end in drop_ranges for n in range(start, end + 1)} + kept = [line for i, line in enumerate(lines, start=1) if i not in drop] + + if reused: + names = ", ".join(sorted(reused)) + import_line = f"from {ENUMS_IMPORT} import {names}" + insert_at = sum( + 1 for i, _ in enumerate(lines, start=1) if i <= last_import_line and i not in drop + ) + kept.insert(insert_at, import_line) + + return "\n".join(kept) + "\n" + + +# Types a contract field can share a name with. A field called `date` binds +# `date = None` in its class body, which shadows the imported type and leaves +# pydantic resolving the annotation to NoneType, so the field silently rejects +# every value. Importing these under a distinct name removes the collision. +SHADOWABLE_TYPES = {"date": "date_type", "time": "time_type"} + + +def alias_shadowed_types(source: str) -> str: + """Import date and time under names no contract field can shadow.""" + imported = ", ".join(f"{name} as {alias}" for name, alias in SHADOWABLE_TYPES.items()) + out = source.replace("from datetime import date, time", f"from datetime import {imported}", 1) + for name, alias in SHADOWABLE_TYPES.items(): + out = out.replace(f"Optional[{name}]", f"Optional[{alias}]") + return out + + +def main() -> None: + shared = shared_enum_names() + with tempfile.TemporaryDirectory() as tmp: + # Written next to enums.yaml so the "../schemas/enums.yaml" ref resolves. + wrapped = CONTRACT.parent / ".contract.wrapped.yaml" + raw = Path(tmp) / "models.py" + try: + build_wrapped(wrapped) + run_codegen(wrapped, raw) + finally: + wrapped.unlink(missing_ok=True) + body = alias_shadowed_types(strip_shared_enums(raw.read_text(), shared)) + + # Drop codegen's own header; it names the temporary wrapped file. + body = "\n".join( + line + for line in body.splitlines() + if not line.startswith(("# generated by datamodel-codegen", "# filename:")) + ).lstrip("\n") + + OUTPUT.write_text(HEADER + "\n" + body + "\n") + _ruff("check", "--fix", str(OUTPUT)) + _ruff("format", str(OUTPUT)) + print(f"Wrote {OUTPUT.relative_to(REPO_ROOT)} (reused shared enums from enums.py).") + + +def _ruff(*args: str) -> None: + """Tidy the output if ruff is available; skip quietly if it is not.""" + ruff = shutil.which("ruff") + cmd = [ruff, *args] if ruff else [sys.executable, "-m", "ruff", *args] + try: + subprocess.run(cmd, cwd=REPO_ROOT, check=False) + except FileNotFoundError: + pass + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index 62fae518..b1b20a0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ (Controller → LLM / commonforms) so tests run fast without Docker or Ollama. """ -import io from unittest.mock import patch, MagicMock import pytest @@ -14,7 +13,18 @@ from app.main import app from app.api.deps import get_db -from app.models import Template, FormSubmission, Job, Input, Extraction, Incident, Form, Report # noqa: F401 — registers tables +from app.models import ( # noqa: F401 — importing these registers their tables + Extraction, + Form, + FormSubmission, + FormTemplate, + Incident, + Input, + Job, + Report, + Template, + TemplateUpload, +) # --------------------------------------------------------------------------- # In-memory database @@ -79,29 +89,30 @@ def pdf_bytes(): return _MINIMAL_PDF -@pytest.fixture -def pdf_upload(pdf_bytes): - """A tuple suitable for httpx/TestClient file upload.""" - return ("file", ("test_form.pdf", io.BytesIO(pdf_bytes), "application/pdf")) - - # --------------------------------------------------------------------------- # Controller mock — patches the heavy dependencies at the route level # --------------------------------------------------------------------------- @pytest.fixture def mock_controller(): - """Patch Controller so create_template / fill_form don't touch the FS or LLM.""" - with patch("app.api.routes.templates.Controller") as tpl_cls, \ - patch("app.api.routes.forms.Controller") as form_cls: - tpl_instance = MagicMock() - tpl_instance.create_template.return_value = "src/inputs/test_template.pdf" - tpl_cls.return_value = tpl_instance - + """Patch the forms Controller so fill_form doesn't touch the FS or LLM.""" + with patch("app.api.routes.forms.Controller") as form_cls: form_instance = MagicMock() form_instance.fill_form.return_value = "src/outputs/filled_output.pdf" form_cls.return_value = form_instance - yield { - "template_ctrl": tpl_instance, - "form_ctrl": form_instance, - } + yield {"form_ctrl": form_instance} + + +@pytest.fixture +def seed_template(): + """Insert a legacy Template row directly (the /templates/create endpoint was + removed in the contract migration). Returns a factory -> template id.""" + def _make(name: str = "T", pdf_path: str = "src/inputs/t.pdf", fields: dict | None = None) -> int: + with Session(_engine) as session: + tpl = Template(name=name, pdf_path=pdf_path, fields=fields if fields is not None else {"name": "string"}) + session.add(tpl) + session.commit() + session.refresh(tpl) + return tpl.id + + return _make diff --git a/tests/test_api.py b/tests/test_api.py index 32104ae7..595486f6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -84,104 +84,21 @@ def test_list_templates_ordering(self, db): class TestTemplateEndpoints: def test_list_templates_empty(self, client): + """Contract registry list is empty until a template is registered.""" resp = client.get(f"{API_PREFIX}/templates") assert resp.status_code == 200 assert resp.json() == [] - def test_create_template(self, client, mock_controller): - payload = { - "name": "Fire Report", - "pdf_path": "src/inputs/fire_report.pdf", - "fields": { - "Name": "string", - "Date": "string", - "Location": "string", - }, - } - resp = client.post(f"{API_PREFIX}/templates/create", json=payload) - assert resp.status_code == 200 - - data = resp.json() - assert data["id"] is not None - assert data["name"] == "Fire Report" - assert data["fields"]["Location"] == "string" - # Plain create just persists the row; commonforms only runs via - # the separate /make-fillable endpoint. - mock_controller["template_ctrl"].create_template.assert_not_called() - - def test_create_then_list(self, client, mock_controller): - """Creating a template should make it appear in the list.""" - client.post(f"{API_PREFIX}/templates/create", json={ - "name": "T1", - "pdf_path": "a.pdf", - "fields": {"f": "string"}, - }) - resp = client.get(f"{API_PREFIX}/templates") - assert resp.status_code == 200 - assert len(resp.json()) == 1 - assert resp.json()[0]["name"] == "T1" - - def test_upload_pdf(self, client, pdf_upload, tmp_path, monkeypatch): - """Upload a valid PDF file.""" - # Point the upload directory inside tmp_path (which is inside the project - # for the path-safety check — we monkeypatch the check). - monkeypatch.setattr( - "app.api.routes.templates.PROJECT_ROOT", - tmp_path, - ) - resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf_upload], - data={"directory": str(tmp_path)}, - ) - assert resp.status_code == 200 - data = resp.json() - assert data["filename"] == "test_form.pdf" - assert data["pdf_path"].endswith(".pdf") - - def test_upload_non_pdf_rejected(self, client): - import io - bad_file = ("file", ("notes.txt", io.BytesIO(b"hello"), "text/plain")) - resp = client.post(f"{API_PREFIX}/templates/upload", files=[bad_file]) - assert resp.status_code == 400 - assert "PDF" in resp.json()["detail"] - - def test_preview_missing_file(self, client): - resp = client.get(f"{API_PREFIX}/templates/preview", params={"path": "src/inputs/nonexistent.pdf"}) - assert resp.status_code == 404 - - def test_directory_traversal_blocked(self, client): - import io - pdf = ("file", ("evil.pdf", io.BytesIO(b"%PDF-1.4"), "application/pdf")) - resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf], - data={"directory": "/etc"}, - ) - assert resp.status_code == 400 - assert "inside the project" in resp.json()["detail"] - # ═══════════════════════════════════════════════════════════════════════════ -# Form fill endpoints +# Form fill endpoints (legacy pipeline — templates seeded directly in the DB +# since the /templates/create endpoint was removed in the contract migration) # ═══════════════════════════════════════════════════════════════════════════ class TestFormEndpoints: - def _seed_template(self, client, mock_controller): - """Helper: create a template and return its ID.""" - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Employee Form", - "pdf_path": "src/inputs/employee.pdf", - "fields": { - "Employee's name": "string", - "Employee's email": "string", - }, - }) - return resp.json()["id"] - - def test_fill_form_success(self, client, mock_controller): - tpl_id = self._seed_template(client, mock_controller) + def test_fill_form_success(self, client, mock_controller, seed_template): + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, @@ -202,8 +119,8 @@ def test_fill_form_missing_template(self, client, mock_controller): }) assert resp.status_code == 404 - def test_fill_form_template_file_not_found(self, client, mock_controller): - tpl_id = self._seed_template(client, mock_controller) + def test_fill_form_template_file_not_found(self, client, mock_controller, seed_template): + tpl_id = seed_template() mock_controller["form_ctrl"].fill_form.side_effect = FileNotFoundError("PDF template not found") resp = client.post(f"{API_PREFIX}/forms/fill", json={ @@ -241,7 +158,7 @@ def fake_post(url, params=None, files=None, timeout=None): captured["files"] = files return fake_response - monkeypatch.setattr("app.api.routes.forms.requests.post", fake_post) + monkeypatch.setattr("app.services.whisper.requests.post", fake_post) audio = ("audio", ("recording.wav", io.BytesIO(b"RIFFfake"), "audio/wav")) resp = client.post(f"{API_PREFIX}/forms/transcribe", files=[audio]) @@ -253,39 +170,40 @@ def fake_post(url, params=None, files=None, timeout=None): assert captured["params"]["output"] == "json" def test_list_models(self, client, monkeypatch): - ""f"{API_PREFIX}/forms/models lists Ollama models and always includes the default.""" - from unittest.mock import MagicMock + ""f"{API_PREFIX}/forms/models lists what the provider serves, default marked.""" + from app.services.llm.models import ModelInfo - fake_response = MagicMock() - fake_response.json.return_value = {"models": [{"name": "qwen2.5:3b"}, {"name": "mistral:latest"}]} - fake_response.raise_for_status.return_value = None - monkeypatch.setattr("app.api.routes.forms.requests.get", lambda *a, **k: fake_response) - monkeypatch.setenv("OLLAMA_MODEL", "qwen2.5:1.5b") + monkeypatch.setattr( + "app.api.routes.forms.llm.list_models", + lambda: [ + ModelInfo(name="qwen2.5:1.5b", default=True), + ModelInfo(name="qwen2.5:3b"), + ModelInfo(name="mistral:latest"), + ], + ) resp = client.get(f"{API_PREFIX}/forms/models") assert resp.status_code == 200 body = resp.json() assert body["default"] == "qwen2.5:1.5b" - assert "qwen2.5:1.5b" in body["models"] # default injected even if not pulled - assert "qwen2.5:3b" in body["models"] + assert body["models"] == ["qwen2.5:1.5b", "qwen2.5:3b", "mistral:latest"] - def test_list_models_ollama_down(self, client, monkeypatch): - """If Ollama is unreachable, still return the default alone.""" - import requests - - def boom(*a, **k): - raise requests.exceptions.ConnectionError("down") + def test_list_models_provider_down(self, client, monkeypatch): + """A provider that will not list them still yields the configured model.""" + from app.services.llm.models import ModelInfo - monkeypatch.setattr("app.api.routes.forms.requests.get", boom) - monkeypatch.setenv("OLLAMA_MODEL", "qwen2.5:1.5b") + monkeypatch.setattr( + "app.api.routes.forms.llm.list_models", + lambda: [ModelInfo(name="qwen2.5:1.5b", default=True)], + ) resp = client.get(f"{API_PREFIX}/forms/models") assert resp.status_code == 200 assert resp.json()["models"] == ["qwen2.5:1.5b"] - def test_fill_form_passes_model_override(self, client, mock_controller): + def test_fill_form_passes_model_override(self, client, mock_controller, seed_template): """A `model` in the request reaches Controller.fill_form but isn't persisted.""" - tpl_id = self._seed_template(client, mock_controller) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, "input_text": "John Doe", @@ -303,7 +221,7 @@ def test_transcribe_service_unavailable(self, client, monkeypatch): def fake_post(*args, **kwargs): raise requests.exceptions.ConnectionError("no service") - monkeypatch.setattr("app.api.routes.forms.requests.post", fake_post) + monkeypatch.setattr("app.services.whisper.requests.post", fake_post) audio = ("audio", ("recording.wav", io.BytesIO(b"data"), "audio/wav")) resp = client.post(f"{API_PREFIX}/forms/transcribe", files=[audio]) @@ -316,45 +234,27 @@ def fake_post(*args, **kwargs): class TestE2EPipeline: """ - Full pipeline: upload PDF → create template → fill form → verify DB state. - This is the critical path that the product depends on. + Legacy fill pipeline: seed template → fill form → verify DB state. + Template registration via API was removed in the contract migration, so the + template is seeded directly in the DB. """ - def test_full_flow(self, client, mock_controller, pdf_upload, tmp_path, monkeypatch, db): - # -- Step 1: Upload a PDF -- - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - upload_resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf_upload], - data={"directory": str(tmp_path)}, - ) - assert upload_resp.status_code == 200 - uploaded_path = upload_resp.json()["pdf_path"] - assert uploaded_path.endswith(".pdf") - - # -- Step 2: Create a template from the uploaded PDF -- - create_resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Incident Report", - "pdf_path": uploaded_path, - "fields": { + def test_full_flow(self, client, mock_controller, seed_template, db): + # -- Step 1: Seed a template -- + template_id = seed_template( + name="Incident Report", + pdf_path="src/inputs/incident.pdf", + fields={ "Officer name": "string", "Badge number": "string", "Incident date": "string", "Location": "string", "Description": "string", }, - }) - assert create_resp.status_code == 200 - template_id = create_resp.json()["id"] + ) assert template_id is not None - # -- Step 3: Verify template appears in list -- - list_resp = client.get(f"{API_PREFIX}/templates") - assert list_resp.status_code == 200 - templates = list_resp.json() - assert any(t["id"] == template_id for t in templates) - - # -- Step 4: Fill the form -- + # -- Step 2: Fill the form -- fill_resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": template_id, "input_text": ( diff --git a/tests/test_deletion.py b/tests/test_deletion.py index 0d036dcb..cb03084b 100644 --- a/tests/test_deletion.py +++ b/tests/test_deletion.py @@ -1,5 +1,5 @@ -"""Tests for DELETE /api/v1/templates/{id}, DELETE /api/v1/forms/{id}, -POST /api/v1/forms/purge, and API-key access control. +"""Tests for DELETE /api/v1/forms/{id}, POST /api/v1/forms/purge, and API-key +access control. """ from datetime import datetime, timedelta, timezone @@ -13,14 +13,14 @@ # Helpers # --------------------------------------------------------------------------- -def _seed_template(client, name="T1", pdf_path="src/inputs/t.pdf"): - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": name, - "pdf_path": pdf_path, - "fields": {"name": "string"}, - }) - assert resp.status_code == 200, resp.json() - return resp.json()["id"] +def _seed_template(db, name="T1", pdf_path="src/inputs/t.pdf"): + """Insert a legacy Template row directly — the /templates/create endpoint was + removed in the contract migration.""" + tpl = Template(name=name, pdf_path=pdf_path, fields={"name": "string"}) + db.add(tpl) + db.commit() + db.refresh(tpl) + return tpl.id def _seed_submission(db, template_id, output_pdf_path="src/outputs/out.pdf"): @@ -35,64 +35,6 @@ def _seed_submission(db, template_id, output_pdf_path="src/outputs/out.pdf"): return sub.id -# =========================================================================== -# DELETE /api/v1/templates/{template_id} -# =========================================================================== - -class TestDeleteTemplate: - - def test_delete_template_no_key_required_when_unconfigured(self, client): - """No API key needed when FIREFORM_API_KEY is empty (default).""" - tpl_id = _seed_template(client) - resp = client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "success" - - def test_delete_template_removes_from_db(self, client, db): - tpl_id = _seed_template(client) - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert db.get(Template, tpl_id) is None - - def test_delete_template_not_found(self, client): - resp = client.delete(f"{API_PREFIX}/templates/99999") - assert resp.status_code == 404 - - def test_delete_template_cascades_submissions(self, client, db): - tpl_id = _seed_template(client) - sub_id = _seed_submission(db, tpl_id) - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - - assert db.get(FormSubmission, sub_id) is None - assert db.get(Template, tpl_id) is None - - def test_delete_template_deletes_pdf_file(self, client, tmp_path, monkeypatch): - """Verify the template PDF file is removed from disk on delete.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - pdf_file = tmp_path / "myform.pdf" - pdf_file.write_bytes(b"%PDF-1.4 fake") - - relative_path = "myform.pdf" - tpl_id = _seed_template(client, pdf_path=relative_path) - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert not pdf_file.exists() - - def test_delete_template_deletes_submission_output_pdfs(self, client, db, tmp_path, monkeypatch): - """Output PDFs of related submissions should be wiped on template deletion.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - - out_pdf = tmp_path / "filled.pdf" - out_pdf.write_bytes(b"%PDF-1.4 filled") - - tpl_id = _seed_template(client, pdf_path="tpl.pdf") - _seed_submission(db, tpl_id, output_pdf_path="filled.pdf") - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert not out_pdf.exists() - - # =========================================================================== # DELETE /api/v1/forms/{submission_id} # =========================================================================== @@ -100,14 +42,14 @@ def test_delete_template_deletes_submission_output_pdfs(self, client, db, tmp_pa class TestDeleteSubmission: def test_delete_submission_no_key_when_unconfigured(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete(f"{API_PREFIX}/forms/{sub_id}") assert resp.status_code == 200 assert resp.json()["status"] == "success" def test_delete_submission_removes_from_db(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) client.delete(f"{API_PREFIX}/forms/{sub_id}") assert db.get(FormSubmission, sub_id) is None @@ -121,7 +63,7 @@ def test_delete_submission_removes_output_pdf(self, client, db, tmp_path, monkey out_pdf = tmp_path / "filled_out.pdf" out_pdf.write_bytes(b"%PDF-1.4") - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id, output_pdf_path="filled_out.pdf") client.delete(f"{API_PREFIX}/forms/{sub_id}") @@ -148,7 +90,7 @@ def _seed_old_submission(self, db, tpl_id, days_old=40): return sub.id def test_purge_removes_old_submissions(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) old_id = self._seed_old_submission(db, tpl_id, days_old=40) new_id = _seed_submission(db, tpl_id) # recent @@ -160,7 +102,7 @@ def test_purge_removes_old_submissions(self, client, db): assert db.get(FormSubmission, new_id) is not None def test_purge_nothing_to_remove(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) _seed_submission(db, tpl_id) # recent resp = client.post(f"{API_PREFIX}/forms/purge?days=30") assert resp.status_code == 200 @@ -171,7 +113,7 @@ def test_purge_removes_output_pdf_file(self, client, db, tmp_path, monkeypatch): out_pdf = tmp_path / "old_filled.pdf" out_pdf.write_bytes(b"%PDF-1.4") - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) old_ts = datetime.now(timezone.utc) - timedelta(days=50) sub = FormSubmission( template_id=tpl_id, @@ -197,43 +139,14 @@ class TestApiKeyAccessControl: def _set_api_key(self, monkeypatch): monkeypatch.setattr("app.api.deps.FIREFORM_API_KEY", "secret-test-key") - def test_delete_template_requires_key_when_set(self, client): - tpl_id = _seed_template(client) - resp = client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert resp.status_code == 401 - - def test_delete_template_with_valid_x_api_key(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"X-API-Key": "secret-test-key"}, - ) - assert resp.status_code == 200 - - def test_delete_template_with_bearer_token(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"Authorization": "Bearer secret-test-key"}, - ) - assert resp.status_code == 200 - - def test_delete_template_wrong_key_rejected(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"X-API-Key": "wrong-key"}, - ) - assert resp.status_code == 401 - def test_delete_submission_requires_key_when_set(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete(f"{API_PREFIX}/forms/{sub_id}") assert resp.status_code == 401 def test_delete_submission_with_valid_key(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete( f"{API_PREFIX}/forms/{sub_id}", diff --git a/tests/test_field_catalog.py b/tests/test_field_catalog.py new file mode 100644 index 00000000..33ff2388 --- /dev/null +++ b/tests/test_field_catalog.py @@ -0,0 +1,129 @@ +"""Tests for the incident-contract field catalog and GET /schema/fields. + +The catalog is built from contracts/schemas/incident-contract.yaml, so these +assert on fields that have been in the contract since it was written rather +than on exact counts, which move with every schema change. +""" + +from app.core.config import API_PREFIX +from app.services import field_catalog + +FIELDS_URL = f"{API_PREFIX}/schema/fields" + + +def _by_path(path): + return next((e for e in field_catalog.catalog() if e.path == path), None) + + +# --------------------------------------------------------------------------- +# Building the catalog +# --------------------------------------------------------------------------- +def test_catalog_flattens_nested_objects(): + entry = _by_path("location.postal_code") + assert entry is not None + assert entry.section == "location" + assert entry.field_type == "string" + assert entry.label == "Postal code" + + +def test_catalog_marks_array_hops(): + paths = [e.path for e in field_catalog.catalog()] + assert any(p.startswith("persons_involved[].") for p in paths) + + +def test_catalog_reads_aliases_from_the_contract(): + entry = _by_path("location.postal_code") + assert "zip" in entry.aliases + + +def test_catalog_reads_the_pii_flag(): + pii_paths = [e.path for e in field_catalog.catalog() if e.pii] + assert pii_paths, "the contract marks several fields x-pii" + + +def test_catalog_resolves_enum_references(): + with_enums = [e for e in field_catalog.catalog() if e.enum_values] + assert with_enums, "enum-typed fields should carry their values" + + +def test_schema_version_comes_from_the_contract(): + assert field_catalog.schema_version() + + +# --------------------------------------------------------------------------- +# Matching +# --------------------------------------------------------------------------- +def test_exact_name_wins(): + top = field_catalog.search("postal_code", limit=3)[0] + assert top[0].path == "location.postal_code" + assert top[1] == 1.0 + + +def test_alias_finds_the_field(): + top = field_catalog.search("zip", limit=3)[0] + assert top[0].path == "location.postal_code" + + +def test_an_exact_name_outranks_an_alias(): + entry = _by_path("location.postal_code") + # score_entry takes an already normalized query, the way search does. + name_score = field_catalog.score_entry(entry, field_catalog.normalize("postal_code")) + alias_score = field_catalog.score_entry(entry, "zip") + assert name_score > alias_score + + +def test_nonsense_query_returns_nothing(): + assert field_catalog.search("qwertyuiop asdf", limit=5) == [] + + +def test_search_respects_the_limit(): + assert len(field_catalog.search("date", limit=3)) <= 3 + + +def test_listing_returns_the_whole_catalog(): + # The editor caches the catalog and filters locally, so a bare listing + # must not be truncated by the search limit. + assert len(field_catalog.search(limit=5)) == len(field_catalog.catalog()) + + +def test_section_filter(): + hits = field_catalog.search(section="location") + assert hits + assert {entry.section for entry, _ in hits} == {"location"} + + +def test_label_normalization_expands_form_shorthand(): + assert field_catalog.normalize_label("Incident No.:") == "incident number" + assert field_catalog.normalize_label("Dt of Loss") == "date of loss" + + +# --------------------------------------------------------------------------- +# GET /schema/fields +# --------------------------------------------------------------------------- +def test_endpoint_searches(client): + resp = client.get(FIELDS_URL, params={"q": "zip", "limit": 5}) + assert resp.status_code == 200 + body = resp.json() + assert body["query"] == "zip" + assert body["schema_version"] + assert body["total"] == len(body["fields"]) + first = body["fields"][0] + assert first["path"] == "location.postal_code" + assert first["score"] > 0 + assert "zip" in first["aliases"] + + +def test_endpoint_lists_without_a_query(client): + body = client.get(FIELDS_URL).json() + assert body["query"] is None + assert body["total"] > 100 + assert body["fields"][0]["score"] is None + + +def test_endpoint_filters_by_section(client): + body = client.get(FIELDS_URL, params={"section": "location"}).json() + assert {f["section"] for f in body["fields"]} == {"location"} + + +def test_endpoint_rejects_an_oversized_limit(client): + assert client.get(FIELDS_URL, params={"q": "date", "limit": 500}).status_code == 422 diff --git a/tests/test_incident_contract_models.py b/tests/test_incident_contract_models.py new file mode 100644 index 00000000..ce59fbd2 --- /dev/null +++ b/tests/test_incident_contract_models.py @@ -0,0 +1,49 @@ +"""Round-trip tests for the generated incident-contract models. + +These guard the generator's contract: every field is optional, the shared +enums are the ones from app/api/schemas/enums.py (not regenerated copies), and +a contract dict survives validate -> dump -> validate unchanged. +""" + +from app.api.schemas import enums +from app.api.schemas.incident_contract import IncidentContract, IncidentType + +SAMPLE = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": { + "name": "Warehouse fire", + "types": [{"primary": True, "category": "natural_disaster"}], + }, + "casualties": {"civilian": [{"severity": "life_threatening"}]}, +} + + +def test_empty_contract_validates(): + """Absent field means unknown, so an empty document is valid.""" + model = IncidentContract.model_validate({}) + assert model.model_dump(exclude_none=True) == {} + + +def test_sample_round_trips(): + model = IncidentContract.model_validate(SAMPLE) + dumped = model.model_dump(exclude_none=True, mode="json") + + # exclude_none keeps only what was actually filled. + assert dumped == SAMPLE + assert "dispatch" not in dumped + + # Re-validating the dump gives an equal model. + assert IncidentContract.model_validate(dumped) == model + + +def test_shared_enums_are_reused(): + """The models import the existing enums instead of defining new copies.""" + assert IncidentType.model_fields["category"].annotation.__args__[0] is enums.IncidentCategory + + +def test_contract_synced_enum_members(): + """Members added to the contract are reachable through the reused enum.""" + model = IncidentContract.model_validate(SAMPLE) + assert model.incident.types[0].category is enums.IncidentCategory.natural_disaster + assert model.casualties.civilian[0].severity is enums.InjurySeverity.life_threatening diff --git a/tests/test_jobs.py b/tests/test_jobs.py index c79e4a42..d1c5504b 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -6,21 +6,13 @@ class TestJobEndpoints: - def _seed_template(self, client): - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Test Template", - "pdf_path": "test.pdf", - "fields": {"name": "string"}, - }) - return resp.json()["id"] - @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_single(self, mock_task, client): + def test_submit_async_single(self, mock_task, client, seed_template): mock_result = MagicMock() mock_result.id = "celery-task-id-1" mock_task.delay.return_value = mock_result - tpl_id = self._seed_template(client) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "John Doe firefighter", @@ -34,14 +26,14 @@ def test_submit_async_single(self, mock_task, client): mock_task.delay.assert_called_once_with(tpl_id, "John Doe firefighter", None) @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_batch(self, mock_task, client): + def test_submit_async_batch(self, mock_task, client, seed_template): mock_task.delay.side_effect = [ MagicMock(id="task-1"), MagicMock(id="task-2"), ] - t1 = self._seed_template(client) - t2 = self._seed_template(client) + t1 = seed_template() + t2 = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [t1, t2], "input_text": "batch input", @@ -62,10 +54,10 @@ def test_submit_async_missing_template(self, mock_task, client): mock_task.delay.assert_not_called() @patch("app.api.routes.jobs.fill_form_task") - def test_get_job_status(self, mock_task, client): + def test_get_job_status(self, mock_task, client, seed_template): mock_task.delay.return_value = MagicMock(id="celery-abc") - tpl_id = self._seed_template(client) + tpl_id = seed_template() submit_resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "test input", @@ -85,10 +77,10 @@ def test_get_job_not_found(self, client): assert resp.status_code == 404 @patch("app.api.routes.jobs.fill_form_task") - def test_submit_with_model_override(self, mock_task, client): + def test_submit_with_model_override(self, mock_task, client, seed_template): mock_task.delay.return_value = MagicMock(id="celery-xyz") - tpl_id = self._seed_template(client) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "test", diff --git a/tests/test_llm_service.py b/tests/test_llm_service.py new file mode 100644 index 00000000..532b9bbd --- /dev/null +++ b/tests/test_llm_service.py @@ -0,0 +1,604 @@ +"""Tests for the LLM module. + +Nothing here touches a network. The provider table is resolved against a +stand-in config object, and the OpenAI client is replaced with a fake that +records what it was asked for and hands back whatever the test lined up. +""" + +from types import SimpleNamespace + +import httpx +import openai +import pytest + +from app.services.llm import client as llm_client +from app.services.llm import providers +from app.services.llm.errors import ( + LLMAuthError, + LLMConfigError, + LLMRateLimitError, + LLMResponseError, + LLMTimeoutError, + LLMUnavailableError, +) +from app.services.llm.gate import RateLimitGate +from app.services.llm.parsing import close_truncated, extract_json_object + +DEFAULTS = { + "LLM_PROVIDER": "ollama", + "LLM_MODEL": "", + "LLM_BASE_URL": "", + "LLM_API_KEY": "", + "OPENAI_API_KEY": "", + "GEMINI_API_KEY": "", + "ANTHROPIC_API_KEY": "", + "LLM_EXTRA_HEADERS": "", + "LLM_ALLOW_EXTERNAL": False, + "LLM_TIMEOUT": 600, + "LLM_MAX_TOKENS": 1200, + "LLM_RATE_LIMIT_RETRIES": 10, + "LLM_RATE_LIMIT_WAIT_SECONDS": 10.0, + "LLM_RATE_LIMIT_MAX_WAIT_SECONDS": 60.0, + "LLM_RESPECT_RETRY_AFTER": True, + "LLM_SERVER_RETRIES": 2, + "LLM_SERVER_RETRY_WAIT_SECONDS": 2.0, + "OLLAMA_HOST": "http://ollama:11434", + "OLLAMA_MODEL": "qwen2.5:1.5b", +} + + +# --------------------------------------------------------------------------- +# Fixtures and fakes +# --------------------------------------------------------------------------- + + +@pytest.fixture +def configure(monkeypatch): + """Point the module at a stand-in config, and clear its caches after.""" + + def _configure(**overrides): + cfg = SimpleNamespace(**{**DEFAULTS, **overrides}) + monkeypatch.setattr(providers, "app_config", cfg) + llm_client.reset() + return cfg + + yield _configure + llm_client.reset() + + +@pytest.fixture +def no_sleep(monkeypatch): + """Record the waits instead of serving them.""" + waits: list[float] = [] + monkeypatch.setattr(llm_client.time, "sleep", waits.append) + return waits + + +def answer(text: str): + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=text))]) + + +class FakeCompletions: + def __init__(self, results): + self.results = list(results) + self.calls: list[dict] = [] + + def create(self, **kwargs): + self.calls.append(kwargs) + result = self.results.pop(0) if len(self.results) > 1 else self.results[0] + if isinstance(result, Exception): + raise result + return result + + +def install(monkeypatch, results, models=None): + """Replace the SDK client with a fake, and hand back the completions stub.""" + completions = FakeCompletions(results) + + def list_models(): + if isinstance(models, Exception): + raise models + return [SimpleNamespace(id=name) for name in (models or [])] + + fake = SimpleNamespace( + chat=SimpleNamespace(completions=completions), + models=SimpleNamespace(list=list_models), + ) + monkeypatch.setattr(llm_client, "_client", fake) + return completions + + +def http_error(kind, status, headers=None, message=None): + request = httpx.Request("POST", "http://provider/v1/chat/completions") + response = httpx.Response(status, headers=headers or {}, request=request) + return kind(message or f"{status} from the provider", response=response, body=None) + + +# --------------------------------------------------------------------------- +# Provider table and configuration +# --------------------------------------------------------------------------- + + +class TestResolve: + def test_ollama_is_the_default_and_needs_nothing(self, configure): + settings = providers.resolve(configure()) + assert settings.provider == "ollama" + assert settings.model == "qwen2.5:1.5b" + assert settings.base_url == "http://ollama:11434/v1" + assert settings.external is False + assert settings.json_mode is True + + def test_unknown_provider_names_the_valid_ones(self, configure): + with pytest.raises(LLMConfigError) as exc: + providers.resolve(configure(LLM_PROVIDER="llamafile")) + assert "llamafile" in str(exc.value) + assert "ollama" in str(exc.value) + + def test_hosted_provider_needs_its_key(self, configure): + with pytest.raises(LLMConfigError, match="OPENAI_API_KEY"): + providers.resolve( + configure( + LLM_PROVIDER="openai", + LLM_MODEL="gpt-4o-mini", + LLM_ALLOW_EXTERNAL=True, + ) + ) + + def test_hosted_provider_needs_a_model_named(self, configure): + with pytest.raises(LLMConfigError, match="LLM_MODEL"): + providers.resolve( + configure( + LLM_PROVIDER="gemini", + GEMINI_API_KEY="k", + LLM_ALLOW_EXTERNAL=True, + ) + ) + + def test_hosted_provider_blocked_without_the_external_flag(self, configure): + with pytest.raises(LLMConfigError, match="LLM_ALLOW_EXTERNAL"): + providers.resolve( + configure(LLM_PROVIDER="openai", LLM_MODEL="gpt-4o-mini", OPENAI_API_KEY="k") + ) + + def test_hosted_provider_allowed_once_the_flag_is_set(self, configure): + settings = providers.resolve( + configure( + LLM_PROVIDER="openai", + LLM_MODEL="gpt-4o-mini", + OPENAI_API_KEY="k", + LLM_ALLOW_EXTERNAL=True, + ) + ) + assert settings.external is True + assert settings.base_url is None # the SDK's own default + + def test_anthropic_has_no_json_mode_so_it_prefills(self, configure): + settings = providers.resolve( + configure( + LLM_PROVIDER="anthropic", + LLM_MODEL="claude-sonnet-4-6", + ANTHROPIC_API_KEY="k", + LLM_ALLOW_EXTERNAL=True, + ) + ) + assert settings.json_mode is False + assert settings.json_prefill is True + + def test_custom_endpoint_needs_a_base_url(self, configure): + with pytest.raises(LLMConfigError, match="LLM_BASE_URL"): + providers.resolve(configure(LLM_PROVIDER="custom", LLM_MODEL="mistral")) + + def test_custom_endpoint_on_this_machine_is_not_external(self, configure): + settings = providers.resolve( + configure( + LLM_PROVIDER="custom", + LLM_MODEL="mistral", + LLM_BASE_URL="http://localhost:8001/v1", + ) + ) + assert settings.external is False + assert settings.api_key == providers._NO_KEY_PLACEHOLDER + + @pytest.mark.parametrize( + "url", ["http://192.168.1.9:8001/v1", "http://10.0.0.4/v1", "http://vllm:8000/v1"] + ) + def test_custom_endpoint_on_the_local_network_is_not_external(self, configure, url): + settings = providers.resolve( + configure(LLM_PROVIDER="custom", LLM_MODEL="m", LLM_BASE_URL=url) + ) + assert settings.external is False + + def test_custom_endpoint_off_site_still_needs_the_flag(self, configure): + with pytest.raises(LLMConfigError, match="LLM_ALLOW_EXTERNAL"): + providers.resolve( + configure( + LLM_PROVIDER="custom", + LLM_MODEL="mixtral", + LLM_BASE_URL="https://openrouter.ai/api/v1", + ) + ) + + def test_base_url_override_wins_for_a_named_provider(self, configure): + settings = providers.resolve( + configure(LLM_PROVIDER="ollama", LLM_BASE_URL="http://gpu-box:11434/v1") + ) + assert settings.base_url == "http://gpu-box:11434/v1" + + def test_extra_headers_must_be_a_json_object(self, configure): + with pytest.raises(LLMConfigError, match="LLM_EXTRA_HEADERS"): + providers.resolve(configure(LLM_EXTRA_HEADERS="not json")) + + def test_extra_headers_are_parsed(self, configure): + settings = providers.resolve(configure(LLM_EXTRA_HEADERS='{"X-Tenant": "fire-dept"}')) + assert settings.extra_headers == {"X-Tenant": "fire-dept"} + + def test_nonsense_limits_are_refused(self, configure): + with pytest.raises(LLMConfigError, match="greater than zero"): + providers.resolve(configure(LLM_MAX_TOKENS=0)) + + +# --------------------------------------------------------------------------- +# Building the request +# --------------------------------------------------------------------------- + + +class TestRequestShape: + def test_json_mode_is_asked_for_where_it_works(self, configure, monkeypatch): + configure() + calls = install(monkeypatch, [answer('{"a": 1}')]) + llm_client.generate_json("find things") + + sent = calls.calls[0] + assert sent["response_format"] == {"type": "json_object"} + assert sent["temperature"] == 0 + assert sent["max_tokens"] == 1200 + assert sent["model"] == "qwen2.5:1.5b" + assert sent["messages"] == [{"role": "user", "content": "find things"}] + + def test_anthropic_gets_a_prefill_instead_of_json_mode(self, configure, monkeypatch): + configure( + LLM_PROVIDER="anthropic", + LLM_MODEL="claude-sonnet-4-6", + ANTHROPIC_API_KEY="k", + LLM_ALLOW_EXTERNAL=True, + ) + calls = install(monkeypatch, [answer('"a": 1}')]) + result = llm_client.generate_json("find things") + + sent = calls.calls[0] + assert "response_format" not in sent + assert sent["messages"][-1] == {"role": "assistant", "content": "{"} + # The opening brace the model continued from is put back before parsing. + assert result == {"a": 1} + + def test_a_model_named_by_the_caller_wins(self, configure, monkeypatch): + configure() + calls = install(monkeypatch, [answer("hello")]) + llm_client.generate("hi", model="llama3.2") + assert calls.calls[0]["model"] == "llama3.2" + + def test_a_refused_parameter_is_dropped_and_the_call_retried(self, configure, monkeypatch): + configure() + refusal = http_error( + openai.BadRequestError, + 400, + message="Unsupported parameter: 'temperature' is not supported with this model", + ) + calls = install(monkeypatch, [refusal, answer("fine")]) + + assert llm_client.generate("hi") == "fine" + assert "temperature" not in calls.calls[1] + + +# --------------------------------------------------------------------------- +# Rate limits +# --------------------------------------------------------------------------- + + +class TestRateLimits: + def test_a_429_is_retried_ten_times_then_gives_up(self, configure, monkeypatch, no_sleep): + configure() + limited = http_error(openai.RateLimitError, 429) + calls = install(monkeypatch, [limited]) + + with pytest.raises(LLMRateLimitError) as exc: + llm_client.generate("hi") + + assert len(calls.calls) == 11 # the first try plus ten retries + assert no_sleep == [10.0] * 10 + assert exc.value.retry_after_seconds == 10.0 + + def test_it_recovers_when_the_limit_lifts(self, configure, monkeypatch, no_sleep): + configure() + limited = http_error(openai.RateLimitError, 429) + calls = install(monkeypatch, [limited, limited, answer("done")]) + + assert llm_client.generate("hi") == "done" + assert len(calls.calls) == 3 + assert no_sleep == [10.0, 10.0] + + def test_a_longer_retry_after_is_honoured(self, configure, monkeypatch, no_sleep): + configure() + limited = http_error(openai.RateLimitError, 429, headers={"retry-after": "25"}) + install(monkeypatch, [limited, answer("done")]) + + llm_client.generate("hi") + assert no_sleep == [25.0] + + def test_an_unreasonable_retry_after_is_capped(self, configure, monkeypatch, no_sleep): + configure() + limited = http_error(openai.RateLimitError, 429, headers={"retry-after": "3600"}) + install(monkeypatch, [limited, answer("done")]) + + llm_client.generate("hi") + assert no_sleep == [60.0] + + def test_a_shorter_retry_after_does_not_shorten_the_wait( + self, configure, monkeypatch, no_sleep + ): + configure() + limited = http_error(openai.RateLimitError, 429, headers={"retry-after": "1"}) + install(monkeypatch, [limited, answer("done")]) + + llm_client.generate("hi") + assert no_sleep == [10.0] + + def test_overloaded_is_treated_like_a_rate_limit(self, configure, monkeypatch, no_sleep): + configure() + overloaded = http_error(openai.APIStatusError, 529) + install(monkeypatch, [overloaded, answer("done")]) + + assert llm_client.generate("hi") == "done" + assert no_sleep == [10.0] + + +class TestGate: + def test_the_gate_stops_the_rest_of_the_batch(self, configure, monkeypatch, no_sleep): + configure() + limited = http_error(openai.RateLimitError, 429) + calls = install(monkeypatch, [limited]) + gate = RateLimitGate() + + with pytest.raises(LLMRateLimitError): + llm_client.generate("first", gate=gate) + assert gate.tripped + first_round = len(calls.calls) + + with pytest.raises(LLMRateLimitError, match="rate limiting this run"): + llm_client.generate("second", gate=gate) + + # The second prompt did not wait, and did not call the provider again. + assert len(calls.calls) == first_round + assert no_sleep == [10.0] * 10 + + def test_an_untripped_gate_changes_nothing(self, configure, monkeypatch): + configure() + install(monkeypatch, [answer("fine")]) + assert llm_client.generate("hi", gate=RateLimitGate()) == "fine" + + +# --------------------------------------------------------------------------- +# Other failures +# --------------------------------------------------------------------------- + + +class TestFailures: + def test_a_rejected_key_is_not_retried(self, configure, monkeypatch, no_sleep): + configure() + calls = install(monkeypatch, [http_error(openai.AuthenticationError, 401)]) + + with pytest.raises(LLMAuthError, match="rejected the API key"): + llm_client.generate("hi") + assert len(calls.calls) == 1 + assert no_sleep == [] + + def test_an_unreachable_provider_says_so(self, configure, monkeypatch): + configure() + request = httpx.Request("POST", "http://ollama:11434/v1/chat/completions") + install(monkeypatch, [openai.APIConnectionError(request=request)]) + + with pytest.raises(LLMUnavailableError, match="could not reach"): + llm_client.generate("hi") + + def test_a_timeout_is_reported_as_one(self, configure, monkeypatch): + configure() + request = httpx.Request("POST", "http://ollama:11434/v1/chat/completions") + install(monkeypatch, [openai.APITimeoutError(request=request)]) + + with pytest.raises(LLMTimeoutError, match="600s"): + llm_client.generate("hi") + + def test_a_server_error_is_retried_twice_then_reported( + self, configure, monkeypatch, no_sleep + ): + configure() + calls = install(monkeypatch, [http_error(openai.APIStatusError, 502)]) + + with pytest.raises(LLMUnavailableError, match="502"): + llm_client.generate("hi") + assert len(calls.calls) == 3 # the first try plus two retries + assert no_sleep == [2.0, 2.0] + + def test_an_empty_answer_is_a_response_error(self, configure, monkeypatch): + configure() + install(monkeypatch, [SimpleNamespace(choices=[])]) + + with pytest.raises(LLMResponseError, match="no answer"): + llm_client.generate("hi") + + +# --------------------------------------------------------------------------- +# Parsing what came back +# --------------------------------------------------------------------------- + + +class TestParsing: + def test_a_fenced_answer_still_parses(self): + assert extract_json_object('```json\n{"a": 1}\n```') == {"a": 1} + + def test_prose_around_the_object_is_ignored(self): + assert extract_json_object('Sure, here you go: {"a": 1} Hope that helps.') == {"a": 1} + + def test_an_answer_cut_off_keeps_the_complete_fields(self): + parsed = extract_json_object('{"a": 1, "b": 2, "c": "half a val') + assert parsed == {"a": 1, "b": 2} + + def test_an_answer_with_no_object_is_rejected(self): + with pytest.raises(LLMResponseError, match="no JSON object"): + extract_json_object("I could not find anything.") + + def test_a_list_is_not_an_object(self): + with pytest.raises(LLMResponseError, match="expected a JSON object"): + extract_json_object("[1, 2, 3]") + + +class TestTruncationRepair: + """The repair pass is only reached once normal parsing has failed. + + What matters is that it never invents a value, only ever drops the field it + could not see the end of. These pin that. + """ + + @pytest.mark.parametrize( + "cut, expected", + [ + ('{"a": 1, "b": 2, "c": "half a val', {"a": 1, "b": 2}), + ('{"a": 1, "bcd', {"a": 1}), + ('{"a": 1,', {"a": 1}), + ('{"a": 1, "b": 12', {"a": 1}), + ('{"a": {"b": 1}, "c": tru', {"a": {"b": 1}}), + ('{"a": [1, 2', {"a": [1]}), + ('{"a": [{"b": 1}, {"c": ', {"a": [{"b": 1}]}), + ('{"a": "x", "b": "y', {"a": "x"}), + ], + ) + def test_complete_fields_survive_and_the_cut_one_is_dropped(self, cut, expected): + assert extract_json_object(cut) == expected + + def test_a_comma_inside_a_value_is_not_a_field_boundary(self): + parsed = extract_json_object('{"address": "400 Oak Street, Apt 2", "time": "14:0') + assert parsed == {"address": "400 Oak Street, Apt 2"} + + def test_a_brace_inside_a_value_is_not_structure(self): + parsed = extract_json_object('{"note": "he said {this}", "next": "cut') + assert parsed == {"note": "he said {this}"} + + def test_an_escaped_quote_does_not_end_the_string(self): + parsed = extract_json_object('{"note": "he said \\"go\\", loudly", "next": "cut') + assert parsed == {"note": 'he said "go", loudly'} + + def test_a_real_answer_that_hit_the_token_ceiling(self): + """Verbatim shape of a qwen2.5 answer that ran out of tokens.""" + truncated = ( + '{\n "incident": {\n "name": "Oak Street fire",\n' + ' "alarm_datetime": "2026-04-18T21:14:00-07:00",\n' + ' "cleared_datetime": "2026-04-18T21:19' + ) + parsed = extract_json_object(truncated) + assert parsed["incident"]["name"] == "Oak Street fire" + assert parsed["incident"]["alarm_datetime"] == "2026-04-18T21:14:00-07:00" + assert "cleared_datetime" not in parsed["incident"] + + def test_a_cut_off_list_keeps_its_complete_entries(self): + truncated = '{"units": [{"unit_id": "E12"}, {"unit_id": "E13"}, {"unit_id": "E1' + assert extract_json_object(truncated) == { + "units": [{"unit_id": "E12"}, {"unit_id": "E13"}] + } + + def test_a_cut_before_any_complete_field_gives_up(self): + assert close_truncated('{"a": 1') is None + with pytest.raises(LLMResponseError): + extract_json_object('{"a": 1') + + def test_nothing_usable_gives_up_rather_than_guessing(self): + assert close_truncated("no json here") is None + + def test_a_complete_answer_never_reaches_the_repair(self): + assert extract_json_object('{"a": 1, "b": [2, 3]}') == {"a": 1, "b": [2, 3]} + + +# --------------------------------------------------------------------------- +# Inspecting the provider +# --------------------------------------------------------------------------- + + +class TestInspection: + def test_models_are_listed_with_the_configured_one_marked(self, configure, monkeypatch): + configure() + install(monkeypatch, [answer("x")], models=["qwen2.5:1.5b", "llama3.2"]) + + models = llm_client.list_models() + assert [m.name for m in models] == ["qwen2.5:1.5b", "llama3.2"] + assert [m.default for m in models] == [True, False] + + def test_a_provider_that_will_not_list_still_reports_the_configured_model( + self, configure, monkeypatch + ): + configure() + install(monkeypatch, [answer("x")], models=RuntimeError("no permission")) + + models = llm_client.list_models() + assert [m.name for m in models] == ["qwen2.5:1.5b"] + assert models[0].default is True + + def test_a_local_provider_is_probed(self, configure, monkeypatch): + configure() + install(monkeypatch, [answer("x")], models=["qwen2.5:1.5b"]) + + report = llm_client.health() + assert report.status == "healthy" + assert report.probed is True + assert report.external is False + + def test_a_local_provider_that_is_down_is_unhealthy(self, configure, monkeypatch): + configure() + install(monkeypatch, [answer("x")], models=RuntimeError("connection refused")) + + report = llm_client.health() + assert report.status == "unhealthy" + assert "connection refused" in report.detail + + def test_a_hosted_provider_is_not_probed(self, configure, monkeypatch): + configure( + LLM_PROVIDER="openai", + LLM_MODEL="gpt-4o-mini", + OPENAI_API_KEY="k", + LLM_ALLOW_EXTERNAL=True, + ) + install(monkeypatch, [answer("x")], models=RuntimeError("should not be called")) + + report = llm_client.health() + assert report.probed is False + assert report.external is True + assert report.status == "healthy" + + def test_health_reports_a_broken_configuration_instead_of_raising(self, configure): + configure(LLM_PROVIDER="nope") + report = llm_client.health() + assert report.status == "unhealthy" + assert "nope" in report.detail + + def test_check_config_returns_the_resolved_settings(self, configure): + configure() + assert llm_client.check_config().model == "qwen2.5:1.5b" + + +class TestStartupGuards: + """Both processes that serve extractions have to refuse a bad configuration.""" + + def test_the_worker_guard_exits_rather_than_raising(self, configure): + """Celery swallows anything deriving from Exception in a signal handler, + logs it and carries on, which would leave a worker running that cannot + serve a single extraction. SystemExit is the only thing that lands. + """ + from app.core.celery import _check_llm_config + + configure(LLM_PROVIDER="nonsense") + with pytest.raises(SystemExit): + _check_llm_config() + + def test_the_worker_guard_passes_a_good_configuration(self, configure): + from app.core.celery import _check_llm_config + + configure() + assert _check_llm_config() is None diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 6c404b80..9b6e203f 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -41,6 +41,7 @@ def test_upgrade_head(alembic_cfg, alembic_engine): assert "incidents" in tables assert "forms" in tables assert "reports" in tables + assert "form_templates" in tables assert "alembic_version" in tables @@ -173,7 +174,7 @@ def test_extractions_columns(alembic_cfg, alembic_engine): "completed_at", "model_used", "processing_time_seconds", - "incident_contract", + "partial_result", "corrections", "error_type", "error_detail", @@ -204,9 +205,28 @@ def test_incidents_columns(alembic_cfg, alembic_engine): "status", "incident_name", "incident_type", - "incident_date", "tags", "notes", + "incident_contract", + "incident_category", + "incident_datetime", + "city", + "state", + "country", + "civilian_injuries", + "civilian_fatalities", + "responder_injuries", + "responder_fatalities", + "people_rescued", + "people_evacuated", + "structures_destroyed", + "area_burned_ha", + "total_loss_amount", + "total_loss_currency", + "call_to_arrival_seconds", + "turnout_seconds_first_unit", + "travel_seconds_first_unit", + "on_scene_duration_seconds", "created_at", "updated_at", "deleted_at", @@ -232,8 +252,9 @@ def test_forms_columns(alembic_cfg, alembic_engine): "form_id", "form_type", "status", - "extract_id", + "template_id", "incident_id", + "batch_id", "job_id", "completed_at", "pdf_ready", @@ -251,9 +272,10 @@ def test_forms_fk(alembic_cfg, alembic_engine): inspector = inspect(alembic_engine) fks = {fk["referred_table"]: fk for fk in inspector.get_foreign_keys("forms")} - # extract_id → extractions and incident_id → incidents; job_id has NO FK constraint - assert "extractions" in fks - assert fks["extractions"]["referred_columns"] == ["extract_id"] + # template_id → form_templates and incident_id → incidents; job_id and + # batch_id have NO FK constraint (batch_id is a grouping key, no Batch table) + assert "form_templates" in fks + assert fks["form_templates"]["referred_columns"] == ["template_id"] assert "incidents" in fks assert fks["incidents"]["referred_columns"] == ["incident_id"] assert len(fks) == 2 @@ -293,9 +315,9 @@ def test_reports_no_fk(alembic_cfg, alembic_engine): def test_downgrade_002(alembic_cfg, alembic_engine): - """Downgrade by one step removes only the 002 tables, leaving 001 tables intact.""" + """Downgrade to 001 removes the 002, 003 and 004 tables, leaving 001 intact.""" command.upgrade(alembic_cfg, "head") - command.downgrade(alembic_cfg, "-1") + command.downgrade(alembic_cfg, "001") inspector = inspect(alembic_engine) tables = inspector.get_table_names() @@ -304,6 +326,209 @@ def test_downgrade_002(alembic_cfg, alembic_engine): assert "incidents" not in tables assert "forms" not in tables assert "reports" not in tables + assert "form_templates" not in tables + assert "template_uploads" not in tables assert "template" in tables assert "formsubmission" in tables assert "job" in tables + + +# --------------------------------------------------------------------------- +# 004 — form_templates registry and template_uploads drafts +# --------------------------------------------------------------------------- + +def test_form_templates_columns(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("form_templates")} + assert columns == { + "template_id", + "form_type", + "display_name", + "jurisdiction", + "agency_type", + "fields", + "source_standard", + "pdf_template_ref", + "version", + "status", + "created_at", + "updated_at", + } + + +def test_form_templates_unique_form_type(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + indexes = {ix["name"]: ix for ix in inspector.get_indexes("form_templates")} + assert indexes["ix_form_templates_form_type"]["unique"] + + +def test_form_templates_no_fk(alembic_cfg, alembic_engine): + """form_templates is a standalone registry — no FK to legacy template/incidents.""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + assert inspector.get_foreign_keys("form_templates") == [] + + +def test_template_uploads_columns(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("template_uploads")} + assert columns == { + "upload_id", + "status", + "pdf_path", + "pdf_template_ref", + "original_filename", + "page_count", + "pages", + "detected_fields", + "detection_error", + "job_id", + "created_at", + "updated_at", + } + + +def test_template_uploads_no_fk(alembic_cfg, alembic_engine): + """Uploads are drafts, not templates, so nothing points at them yet.""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + assert inspector.get_foreign_keys("template_uploads") == [] + + +def test_downgrade_004(alembic_cfg, alembic_engine): + """Downgrade to 003 removes both 004 tables, leaving 003 intact. + + Targets the "003" revision explicitly rather than "-1": 005 now sits on + top of head, so a relative one-step downgrade only undoes 005, not 004. + """ + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "003") + + inspector = inspect(alembic_engine) + tables = inspector.get_table_names() + assert "form_templates" not in tables + assert "template_uploads" not in tables + assert "inputs" in tables + assert "forms" in tables + assert "reports" in tables + + +# --------------------------------------------------------------------------- +# 005 — forms: template_id/incident_id/batch_id, drop extract_id +# --------------------------------------------------------------------------- + +def test_forms_template_id_not_nullable(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"]: c for c in inspector.get_columns("forms")} + assert not columns["template_id"]["nullable"] + + +def test_forms_incident_id_not_nullable(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"]: c for c in inspector.get_columns("forms")} + assert not columns["incident_id"]["nullable"] + + +def test_forms_batch_id_nullable_no_fk(alembic_cfg, alembic_engine): + """batch_id is a plain grouping key — nullable, no FK (no Batch table).""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"]: c for c in inspector.get_columns("forms")} + assert columns["batch_id"]["nullable"] + referred_tables = {fk["referred_table"] for fk in inspector.get_foreign_keys("forms")} + assert "batches" not in referred_tables + + +def test_forms_extract_id_removed(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("forms")} + assert "extract_id" not in columns + + +def test_downgrade_005(alembic_cfg, alembic_engine): + """Downgrade to 004 restores 004's forms shape. + + Targets "004" explicitly rather than "-1": 006 now sits on top of head, so + a relative one-step downgrade only undoes 006, not 005. + """ + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "004") + + inspector = inspect(alembic_engine) + columns = {c["name"]: c for c in inspector.get_columns("forms")} + assert "extract_id" in columns + assert "template_id" not in columns + assert "batch_id" not in columns + assert columns["incident_id"]["nullable"] + + fks = {fk["referred_table"] for fk in inspector.get_foreign_keys("forms")} + assert "extractions" in fks + assert "form_templates" not in fks + + # form_templates itself is untouched — only introduced by 004, not by 005 + assert "form_templates" in inspector.get_table_names() + + +def test_round_trip_005(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "004") + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("forms")} + assert "template_id" in columns + assert "batch_id" in columns + assert "extract_id" not in columns + + +def test_incidents_indexes(alembic_cfg, alembic_engine): + """006 adds the two indexes GET /incidents and the number check rely on.""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + indexes = {ix["name"]: ix for ix in inspector.get_indexes("incidents")} + + assert indexes["ix_incidents_live_datetime"]["column_names"] == [ + "deleted_at", + "incident_datetime", + ] + assert indexes["ix_incidents_number_live"]["unique"] + + +def test_downgrade_006(alembic_cfg, alembic_engine): + """Downgrade to 005 drops the indexes and puts incident_date back.""" + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "005") + + inspector = inspect(alembic_engine) + assert "incident_date" in {c["name"] for c in inspector.get_columns("incidents")} + assert "ix_incidents_live_datetime" not in { + ix["name"] for ix in inspector.get_indexes("incidents") + } + + +def test_round_trip_006(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "005") + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + assert "incident_date" not in {c["name"] for c in inspector.get_columns("incidents")} + assert "ix_incidents_number_live" in { + ix["name"] for ix in inspector.get_indexes("incidents") + } diff --git a/tests/test_model.py b/tests/test_model.py deleted file mode 100644 index 6de314f9..00000000 --- a/tests/test_model.py +++ /dev/null @@ -1,12 +0,0 @@ -# test_ollama.py -import os - -import ollama - -try: - response = ollama.chat(model=os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b"), messages=[ - {'role': 'user', 'content': 'Say hello in Spanish'} - ]) - print("Success! Response:", response['message']['content']) -except Exception as e: - print("Error:", e) diff --git a/tests/test_repositories_promote.py b/tests/test_repositories_promote.py new file mode 100644 index 00000000..57381bd3 --- /dev/null +++ b/tests/test_repositories_promote.py @@ -0,0 +1,267 @@ +"""Tests for the extraction/incident repositories and the analytics recompute. + +Repositories run against the shared in-memory SQLite engine from conftest.py. +`promote` is a pure function and is tested directly over an empty and a full +contract. +""" + +from app.db import repositories as repo +from app.models import Input, Extraction +from app.api.schemas.enums import InputType, ExtractionStatus, ReportStatus +from app.services.incidents import promote, PROMOTED_COLUMNS + + +# --------------------------------------------------------------------------- +# Repository paths +# --------------------------------------------------------------------------- + +def _make_input(db): + return repo.create_input(db, Input(input_type=InputType.text, transcript="x")) + + +def _make_extraction(db, input_id): + return repo.create_extraction(db, Extraction(input_id=input_id)) + + +class TestExtractionRepository: + + def test_create_and_get_extraction(self, db): + inp = _make_input(db) + created = _make_extraction(db, inp.input_id) + assert created.extract_id is not None + + fetched = repo.get_extraction(db, created.extract_id) + assert fetched is not None + assert fetched.extract_id == created.extract_id + assert fetched.input_id == inp.input_id + assert fetched.status == ExtractionStatus.processing + + def test_get_extraction_unknown_returns_none(self, db): + from uuid import uuid4 + assert repo.get_extraction(db, uuid4()) is None + + def test_update_extraction_persists_changes(self, db): + inp = _make_input(db) + extraction = _make_extraction(db, inp.input_id) + + extraction.status = ExtractionStatus.completed + extraction.model_used = "llama3:8b" + repo.update_extraction(db, extraction) + + reloaded = repo.get_extraction(db, extraction.extract_id) + assert reloaded.status == ExtractionStatus.completed + assert reloaded.model_used == "llama3:8b" + + +class TestIncidentRepository: + + def test_create_draft_incident_links_to_extraction(self, db): + inp = _make_input(db) + extraction = _make_extraction(db, inp.input_id) + + incident = repo.create_draft_incident(db, extraction.extract_id) + assert incident.incident_id is not None + assert incident.extract_id == extraction.extract_id + assert incident.status == ReportStatus.draft + + def test_get_incident_and_by_extract(self, db): + inp = _make_input(db) + extraction = _make_extraction(db, inp.input_id) + incident = repo.create_draft_incident(db, extraction.extract_id) + + assert repo.get_incident(db, incident.incident_id).incident_id == incident.incident_id + by_extract = repo.get_incident_by_extract(db, extraction.extract_id) + assert by_extract.incident_id == incident.incident_id + + def test_get_incident_unknown_returns_none(self, db): + from uuid import uuid4 + assert repo.get_incident(db, uuid4()) is None + + def test_update_incident_persists_promoted_columns(self, db): + inp = _make_input(db) + extraction = _make_extraction(db, inp.input_id) + incident = repo.create_draft_incident(db, extraction.extract_id) + + incident.incident_number = "CA-SQF-2024-0421" + incident.city = "Springfield" + incident.civilian_injuries = 3 + repo.update_incident(db, incident) + + reloaded = repo.get_incident(db, incident.incident_id) + assert reloaded.incident_number == "CA-SQF-2024-0421" + assert reloaded.city == "Springfield" + assert reloaded.civilian_injuries == 3 + + +# --------------------------------------------------------------------------- +# promote +# --------------------------------------------------------------------------- + +class TestPromoteEmptyContract: + + def test_empty_dict_yields_all_none(self): + result = promote({}) + assert set(result) == set(PROMOTED_COLUMNS) + assert all(value is None for value in result.values()) + + def test_none_contract_yields_all_none(self): + assert all(value is None for value in promote(None).values()) + + def test_partial_contract_only_fills_present_fields(self): + result = promote({"location": {"city": "Reno", "state": "NV"}}) + assert result["city"] == "Reno" + assert result["state"] == "NV" + assert result["country"] is None + assert result["incident_datetime"] is None + + +FULL_CONTRACT = { + "incident": { + "name": "Bear Creek Wildfire", + "types": [ + {"primary": False, "category": "ems", "subcategory": "medical_assist"}, + {"primary": True, "category": "fire", "subcategory": "wildland_fire"}, + ], + "alarm_datetime": "2024-07-10T13:50:00-07:00", + "start_datetime": "2024-07-10T13:40:00-07:00", + "first_arrival_datetime": "2024-07-10T13:56:00-07:00", + "cleared_datetime": "2024-07-10T15:56:00-07:00", + }, + "dispatch": {"call_received_datetime": "2024-07-10T13:52:00-07:00"}, + "location": {"city": "Reno", "state": "NV", "country": "US"}, + "casualties": { + "total_civilian_injuries": 2, + "total_civilian_fatalities": 1, + "total_responder_injuries": 3, + "total_responder_fatalities": 0, + }, + "rescues": [{"person_type": "civilian"}, {"person_type": "civilian"}], + "evacuation_displacement": {"total_people_evacuated": 40}, + "structure": {"structures_destroyed": 2}, + "wildland": {"area_burned_ha": 12.5}, + "losses": { + "property_loss": {"amount": 100000, "currency": "USD"}, + "contents_loss": {"amount": 25000, "currency": "USD"}, + }, + "units": [ + { + "unit_id": "E2", + "arrived_datetime": "2024-07-10T13:58:00-07:00", + "dispatched_datetime": "2024-07-10T13:52:00-07:00", + "enroute_datetime": "2024-07-10T13:53:00-07:00", + }, + { + "unit_id": "E1", + "arrived_datetime": "2024-07-10T13:56:00-07:00", + "turnout_seconds": 60, + "travel_seconds": 180, + }, + ], +} + + +class TestPromoteFullContract: + + def setup_method(self): + self.result = promote(FULL_CONTRACT) + + def test_category_from_primary_type(self): + assert self.result["incident_category"] == "fire" + + def test_name_promoted(self): + assert self.result["incident_name"] == "Bear Creek Wildfire" + + def test_type_is_the_primary_subcategory(self): + # Both come off the same entry, so they can never describe two + # different types: the non-primary ems/medical_assist pair is ignored. + assert self.result["incident_type"] == "wildland_fire" + + def test_incident_datetime_prefers_alarm(self): + # Alarm wins over start and dispatch call-received. + assert self.result["incident_datetime"].isoformat() == "2024-07-10T13:50:00-07:00" + + def test_location_promoted(self): + assert self.result["city"] == "Reno" + assert self.result["state"] == "NV" + assert self.result["country"] == "US" + + def test_casualty_counts_from_totals(self): + assert self.result["civilian_injuries"] == 2 + assert self.result["civilian_fatalities"] == 1 + assert self.result["responder_injuries"] == 3 + assert self.result["responder_fatalities"] == 0 + + def test_people_rescued_is_array_length(self): + assert self.result["people_rescued"] == 2 + + def test_people_evacuated_from_total(self): + assert self.result["people_evacuated"] == 40 + + def test_structures_and_area(self): + assert self.result["structures_destroyed"] == 2 + assert self.result["area_burned_ha"] == 12.5 + + def test_total_loss_is_property_plus_contents(self): + assert self.result["total_loss_amount"] == 125000 + assert self.result["total_loss_currency"] == "USD" + + def test_call_to_arrival_uses_call_received(self): + # 13:52 call received -> 13:56 first arrival = 4 minutes. + assert self.result["call_to_arrival_seconds"] == 240 + + def test_on_scene_duration(self): + # 13:56 first arrival -> 15:56 cleared = 2 hours. + assert self.result["on_scene_duration_seconds"] == 7200 + + def test_first_unit_is_earliest_arrival_with_precomputed_timing(self): + # E1 arrived first (13:56) and carries precomputed turnout/travel. + assert self.result["turnout_seconds_first_unit"] == 60 + assert self.result["travel_seconds_first_unit"] == 180 + + +class TestPromoteFallbacks: + + def test_incident_datetime_falls_back_to_start_then_call_received(self): + start_only = promote({"incident": {"start_datetime": "2024-07-10T13:40:00-07:00"}}) + assert start_only["incident_datetime"].isoformat() == "2024-07-10T13:40:00-07:00" + + dispatch_only = promote({"dispatch": {"call_received_datetime": "2024-07-10T13:52:00-07:00"}}) + assert dispatch_only["incident_datetime"].isoformat() == "2024-07-10T13:52:00-07:00" + + def test_call_to_arrival_falls_back_to_alarm_when_no_call_received(self): + result = promote({ + "incident": { + "alarm_datetime": "2024-07-10T13:50:00-07:00", + "first_arrival_datetime": "2024-07-10T13:56:00-07:00", + }, + }) + assert result["call_to_arrival_seconds"] == 360 + + def test_first_unit_turnout_travel_computed_from_timestamps(self): + result = promote({ + "units": [{ + "unit_id": "E7", + "arrived_datetime": "2024-07-10T13:58:00-07:00", + "dispatched_datetime": "2024-07-10T13:52:00-07:00", + "enroute_datetime": "2024-07-10T13:53:00-07:00", + }], + }) + assert result["turnout_seconds_first_unit"] == 60 # 13:52 -> 13:53 + assert result["travel_seconds_first_unit"] == 300 # 13:53 -> 13:58 + + def test_negative_interval_is_none(self): + result = promote({ + "incident": { + "first_arrival_datetime": "2024-07-10T14:00:00-07:00", + "cleared_datetime": "2024-07-10T13:00:00-07:00", + }, + }) + assert result["on_scene_duration_seconds"] is None + + def test_single_loss_side_still_totals(self): + result = promote({"losses": {"property_loss": {"amount": 5000, "currency": "GBP"}}}) + assert result["total_loss_amount"] == 5000 + assert result["total_loss_currency"] == "GBP" + + def test_unparseable_datetime_is_none(self): + assert promote({"incident": {"alarm_datetime": "not a date"}})["incident_datetime"] is None diff --git a/tests/test_template_detection.py b/tests/test_template_detection.py new file mode 100644 index 00000000..01d13d5b --- /dev/null +++ b/tests/test_template_detection.py @@ -0,0 +1,360 @@ +"""Tests for template field detection (app/services/template_detection.py). + +commonforms is mocked throughout. What is exercised here is everything around +it: reading widget rectangles into layout boxes, naming fields, picking the +label next to a box, scoring mapping suggestions, and writing the draft back. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pypdf import PdfReader, PdfWriter +from pypdf.generic import ( + ArrayObject, + DictionaryObject, + FloatObject, + NameObject, + TextStringObject, +) +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, FieldSource +from app.api.schemas.templates import MappingSuggestion, TemplateFieldLayout +from app.db.repositories import create_job, create_template_upload, get_job_by_uuid +from app.models import Job, TemplateUpload +from app.services import template_detection as detection + + +# --------------------------------------------------------------------------- +# Fixtures: a small PDF carrying real form widgets +# --------------------------------------------------------------------------- +def _widget(name, rect): + return DictionaryObject( + { + NameObject("/Type"): NameObject("/Annot"), + NameObject("/Subtype"): NameObject("/Widget"), + NameObject("/FT"): NameObject("/Tx"), + NameObject("/Rect"): ArrayObject([FloatObject(v) for v in rect]), + NameObject("/T"): TextStringObject(name), + } + ) + + +@pytest.fixture +def widget_pdf(tmp_path): + """A one-page PDF with two text widgets, the lower one listed first.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + page = writer.pages[0] + annotations = ArrayObject() + for name, rect in ( + ("fire_cause", (188.33, 560.0, 388.33, 578.0)), + ("Incident No.", (188.33, 621.33, 315.66, 650.0)), + ): + annotations.append(writer._add_object(_widget(name, rect))) + page[NameObject("/Annots")] = annotations + + path = tmp_path / "form.pdf" + with path.open("wb") as handle: + writer.write(handle) + return path + + +@pytest.fixture +def flat_pdf(tmp_path): + """A page with no widgets at all, the case commonforms exists for.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + path = tmp_path / "flat.pdf" + with path.open("wb") as handle: + writer.write(handle) + return path + + +# --------------------------------------------------------------------------- +# Geometry +# --------------------------------------------------------------------------- +def test_read_pages_returns_points(widget_pdf): + pages = detection.read_pages(widget_pdf) + assert [p.model_dump() for p in pages] == [{"page": 0, "width": 612.0, "height": 792.0}] + + +def test_widget_rects_become_layout_boxes(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + layout = drafts[0].field.layout + assert layout.page == 0 + assert layout.x == pytest.approx(188.33) + assert layout.y == pytest.approx(621.33) + assert layout.width == pytest.approx(127.33, abs=0.01) + assert layout.height == pytest.approx(28.67, abs=0.01) + + +def test_boxes_come_back_in_reading_order(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + # The higher box on the page comes first even though it is second in the + # annotation array. + assert [d.field.layout.y for d in drafts] == sorted( + [d.field.layout.y for d in drafts], reverse=True + ) + + +# --------------------------------------------------------------------------- +# Field naming +# --------------------------------------------------------------------------- +def test_widget_names_are_turned_into_slugs(widget_pdf): + names = [d.field.field_name for d in detection.build_draft_fields(widget_pdf)] + assert names == ["incident_no", "fire_cause"] + + +def test_repeated_names_are_made_unique(): + used = set() + assert detection._field_name("Date", used, 1) == "date" + assert detection._field_name("Date", used, 2) == "date_2" + assert detection._field_name("Date", used, 3) == "date_3" + + +def test_an_unnamed_box_falls_back_to_its_position(): + assert detection._field_name(None, set(), 7) == "field_7" + + +# --------------------------------------------------------------------------- +# Labels +# --------------------------------------------------------------------------- +def _layout(**over): + base = {"page": 0, "x": 200.0, "y": 600.0, "width": 120.0, "height": 18.0} + base.update(over) + return TemplateFieldLayout(**base) + + +def test_label_to_the_left_is_preferred(): + texts = [("Incident No.", 120.0, 604.0), ("Section B", 200.0, 622.0)] + assert detection._nearest_label(_layout(), texts) == "Incident No." + + +def test_label_above_is_used_when_nothing_sits_to_the_left(): + texts = [("Incident No.", 200.0, 622.0)] + assert detection._nearest_label(_layout(), texts) == "Incident No." + + +def test_far_away_text_is_not_a_label(): + texts = [("Unrelated", 5.0, 604.0)] + assert detection._nearest_label(_layout(), texts) is None + + +def test_no_text_means_no_label(widget_pdf): + # The fixture PDF has widgets but no page text. + assert all(d.detected_label is None for d in detection.build_draft_fields(widget_pdf)) + + +# --------------------------------------------------------------------------- +# Mapping suggestions +# --------------------------------------------------------------------------- +def test_a_clear_label_gets_suggestions(): + suggestions = detection.suggest_mappings("Incident No.") + assert suggestions + assert suggestions[0].path == "report_metadata.incident_number" + assert suggestions[0].section == "report_metadata" + + +def test_a_meaningless_label_gets_nothing(): + assert detection.suggest_mappings("qwertyuiop") == [] + + +def test_a_missing_label_gets_nothing(): + assert detection.suggest_mappings(None) == [] + + +def test_a_confident_suggestion_is_pre_applied(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + incident = drafts[0].field + # The widget is named "Incident No.", which scores high enough to apply. + assert incident.source == FieldSource.schema + assert incident.incident_mapping == "report_metadata.incident_number" + + +def test_a_weak_suggestion_is_only_offered(monkeypatch): + from app.api.schemas.templates import TemplateField + from app.api.schemas.enums import TemplateFieldType + + field = TemplateField( + field_name="box_1", + field_type=TemplateFieldType.string, + source=FieldSource.manual, + required=False, + ) + weak = [MappingSuggestion(path="location.postal_code", score=0.6)] + applied = detection._apply_suggestion(field, weak) + assert applied.source == FieldSource.manual + assert applied.incident_mapping is None + + +def test_an_applied_enum_mapping_brings_its_values(): + from app.api.schemas.templates import TemplateField + from app.api.schemas.enums import TemplateFieldType + + entry = next(e for e in detection.field_catalog.catalog() if e.enum_values) + field = TemplateField( + field_name="box_1", + field_type=TemplateFieldType.string, + source=FieldSource.manual, + required=False, + ) + applied = detection._apply_suggestion( + field, [MappingSuggestion(path=entry.path, score=0.99)] + ) + assert applied.field_type == TemplateFieldType.enum + assert applied.allowed_values == list(entry.enum_values) + + +# --------------------------------------------------------------------------- +# Running commonforms +# --------------------------------------------------------------------------- +def test_detection_skips_commonforms_when_widgets_exist(widget_pdf): + with patch("app.services.controller.Controller") as controller: + drafts = detection.detect_fields(widget_pdf) + controller.assert_not_called() + assert len(drafts) == 2 + + +def test_a_flat_pdf_goes_through_commonforms(flat_pdf, widget_pdf): + instance = MagicMock() + instance.prepare_fillable.return_value = str(widget_pdf) + with patch("app.services.controller.Controller", return_value=instance): + drafts = detection.detect_fields(flat_pdf) + assert len(drafts) == 2 + + +def test_a_one_page_pdf_is_padded_before_commonforms(flat_pdf, widget_pdf): + """commonforms cannot read a one-page document, so it never sees one.""" + seen = {} + + def capture(path): + seen["path"] = path + seen["pages"] = len(PdfReader(path).pages) + return str(widget_pdf) + + instance = MagicMock() + instance.prepare_fillable.side_effect = capture + with patch("app.services.controller.Controller", return_value=instance): + detection.detect_fields(flat_pdf) + + assert seen["path"] != str(flat_pdf) + assert seen["pages"] == 2 + # The padded copy and the fillable it produced are scratch, not artefacts. + assert not Path(seen["path"]).exists() + assert not widget_pdf.exists() + assert flat_pdf.exists() + + +def test_a_multi_page_pdf_is_passed_through_as_is(tmp_path, widget_pdf): + writer = PdfWriter() + writer.add_blank_page(612, 792) + writer.add_blank_page(612, 792) + flat = tmp_path / "two_pages.pdf" + with flat.open("wb") as handle: + writer.write(handle) + + instance = MagicMock() + instance.prepare_fillable.return_value = str(widget_pdf) + with patch("app.services.controller.Controller", return_value=instance): + detection.detect_fields(flat) + instance.prepare_fillable.assert_called_once_with(str(flat)) + + +def test_boxes_found_on_the_padding_are_dropped(flat_pdf, tmp_path): + """A box detected on the blank page belongs to no page of the real PDF.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + writer.add_blank_page(612, 792) + for page_ix, name in ((0, "real_box"), (1, "padding_box")): + page = writer.pages[page_ix] + annots = ArrayObject() + annots.append(writer._add_object(_widget(name, (100.0, 100.0, 200.0, 118.0)))) + page[NameObject("/Annots")] = annots + fillable = tmp_path / "detected.pdf" + with fillable.open("wb") as handle: + writer.write(handle) + + instance = MagicMock() + instance.prepare_fillable.return_value = str(fillable) + with patch("app.services.controller.Controller", return_value=instance): + drafts = detection.detect_fields(flat_pdf) + + assert [d.field.layout.page for d in drafts] == [0] + + +# --------------------------------------------------------------------------- +# The background run +# --------------------------------------------------------------------------- +def _seed_upload(session, pdf_path): + upload = TemplateUpload( + pdf_path=str(pdf_path), + pdf_template_ref="templates/uploads/x.pdf", + page_count=1, + pages=[{"page": 0, "width": 612.0, "height": 792.0}], + ) + return create_template_upload(session, upload) + + +def _seed_job(session): + return create_job( + session, Job(celery_task_id="t", job_type="template_field_detection", status="queued") + ) + + +def test_run_detection_writes_the_draft(test_engine, widget_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, widget_pdf) + job = _seed_job(session) + + result = detection.run_detection(session, upload.upload_id, job.job_id) + + assert result["status"] == "completed" + assert result["detected_fields"] == 2 + + session.refresh(upload) + assert upload.status == DetectionStatus.completed + assert len(upload.detected_fields) == 2 + assert upload.detected_fields[0]["field"]["layout"]["page"] == 0 + + finished = get_job_by_uuid(session, job.job_id) + assert finished.status == "completed" + assert finished.progress_percent == 100 + assert finished.result_url == f"/api/v1/templates/pdf/{upload.upload_id}" + + +def test_run_detection_records_a_failure_without_losing_the_upload(test_engine, flat_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, flat_pdf) + job = _seed_job(session) + + with patch.object(detection, "detect_fields", side_effect=RuntimeError("model gone")): + result = detection.run_detection(session, upload.upload_id, job.job_id) + + assert result["status"] == "failed" + session.refresh(upload) + assert upload.status == DetectionStatus.failed + assert upload.detection_error == "model gone" + # Geometry survives, so the editor can still be used by hand. + assert upload.page_count == 1 + + failed_job = get_job_by_uuid(session, job.job_id) + assert failed_job.status == "failed" + assert failed_job.error["error_code"] == "DETECTION_FAILED" + + +def test_run_detection_on_a_vanished_upload(test_engine): + from uuid import uuid4 + + with Session(test_engine) as session: + job = _seed_job(session) + result = detection.run_detection(session, uuid4(), job.job_id) + assert result["status"] == "failed" + assert get_job_by_uuid(session, job.job_id).error["error_code"] == "UPLOAD_NOT_FOUND" + + +def test_run_detection_without_a_job(test_engine, widget_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, widget_pdf) + assert detection.run_detection(session, upload.upload_id)["status"] == "completed" diff --git a/tests/test_templates_pdf.py b/tests/test_templates_pdf.py new file mode 100644 index 00000000..b6eedc20 --- /dev/null +++ b/tests/test_templates_pdf.py @@ -0,0 +1,250 @@ +"""Tests for the template PDF authoring flow. + +Covers POST /templates/pdf, GET /templates/pdf/{upload_id} and +GET /templates/{template_id}/pdf. commonforms never runs here: detection is +dispatched to Celery, and these check the parts around it. +""" + +import io +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus +from app.core.config import API_PREFIX +from app.models import FormTemplate, TemplateUpload + +TEMPLATES_URL = f"{API_PREFIX}/templates" + + +@pytest.fixture +def upload_dir(tmp_path, monkeypatch): + """Send stored PDFs to a temp directory instead of the real data dir.""" + target = tmp_path / "templates" / "uploads" + monkeypatch.setattr("app.services.form_templates.TEMPLATE_UPLOAD_DIR", target) + monkeypatch.setattr("app.services.form_templates.DATA_DIR", tmp_path) + return target + + +@pytest.fixture +def no_celery(): + """Stand in for the detection task so nothing is dispatched to a broker.""" + with patch("app.services.form_templates.detect_template_fields_task") as task: + task.delay.return_value = MagicMock(id="celery-task-1") + yield task + + +def _files(pdf_bytes, name="texas_sfm.pdf"): + return {"pdf_file": (name, io.BytesIO(pdf_bytes), "application/pdf")} + + +def _upload(client, pdf_bytes, detect=True, name="texas_sfm.pdf"): + return client.post( + f"{TEMPLATES_URL}/pdf", + files=_files(pdf_bytes, name), + data={"detect_fields": str(detect).lower()}, + ) + + +# --------------------------------------------------------------------------- +# POST /templates/pdf +# --------------------------------------------------------------------------- +def test_upload_returns_202_with_geometry_and_poll_url(client, pdf_bytes, upload_dir, no_celery): + resp = _upload(client, pdf_bytes) + assert resp.status_code == 202, resp.json() + body = resp.json() + + assert body["status"] == "processing" + assert body["page_count"] == 1 + assert body["pages"] == [{"page": 0, "width": 612.0, "height": 792.0}] + assert body["original_filename"] == "texas_sfm.pdf" + assert body["pdf_template_ref"].endswith(".pdf") + assert body["poll_url"] == f"/api/v1/templates/pdf/{body['upload_id']}" + assert body["job_id"] + assert body["retry_after_seconds"] == 5 + # Detection has not run, so there is no field list yet. + assert body["detected_fields"] is None + + +def test_upload_stores_the_pdf_on_disk(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes).json() + stored = upload_dir / f"{body['upload_id']}.pdf" + assert stored.read_bytes() == pdf_bytes + + +def test_upload_dispatches_detection(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes).json() + no_celery.delay.assert_called_once_with(body["upload_id"], body["job_id"]) + + +def test_upload_without_detection_completes_immediately(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes, detect=False).json() + assert body["status"] == "completed" + assert body["detected_fields"] == [] + assert body["job_id"] is None + assert body["retry_after_seconds"] is None + no_celery.delay.assert_not_called() + + +def test_upload_rejects_a_non_pdf(client, upload_dir, no_celery): + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"just some text", "notes.pdf")) + assert resp.status_code == 415 + assert resp.json()["error_code"] == "UNSUPPORTED_FORMAT" + + +def test_upload_rejects_an_empty_file(client, upload_dir, no_celery): + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"", "empty.pdf")) + assert resp.status_code == 400 + assert resp.json()["error_code"] == "MISSING_FILE" + + +def test_upload_rejects_an_oversized_pdf(client, upload_dir, no_celery, monkeypatch): + monkeypatch.setattr("app.api.routes.form_templates.MAX_TEMPLATE_PDF_BYTES", 10) + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"%PDF-1.4 padded out here")) + assert resp.status_code == 413 + assert resp.json()["error_code"] == "FILE_TOO_LARGE" + + +def test_upload_without_a_file_is_a_422(client, upload_dir, no_celery): + assert client.post(f"{TEMPLATES_URL}/pdf").status_code == 422 + + +def test_a_pdf_that_cannot_be_read_is_a_415(client, upload_dir, no_celery): + # Right magic bytes, nothing behind them. + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"%PDF-1.4 truncated")) + assert resp.status_code == 415 + assert resp.json()["error_code"] == "INVALID_PDF" + assert list(upload_dir.glob("*.pdf")) == [] + + +# --------------------------------------------------------------------------- +# GET /templates/pdf/{upload_id} +# --------------------------------------------------------------------------- +def test_draft_poll_returns_the_stored_state(client, pdf_bytes, upload_dir, no_celery): + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + resp = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["upload_id"] == upload_id + assert body["status"] == "processing" + assert body["retry_after_seconds"] == 5 + + +def test_draft_poll_after_detection(client, pdf_bytes, upload_dir, no_celery, test_engine): + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + + with Session(test_engine) as session: + upload = session.get(TemplateUpload, __import__("uuid").UUID(upload_id)) + upload.status = DetectionStatus.completed + upload.detected_fields = [ + { + "field": { + "field_name": "incident_number", + "field_type": "string", + "source": "schema", + "required": False, + "incident_mapping": "report_metadata.incident_number", + "layout": {"page": 0, "x": 10, "y": 20, "width": 100, "height": 18}, + }, + "detected_label": "Incident No.", + "suggestions": [ + {"path": "report_metadata.incident_number", "score": 0.93}, + ], + } + ] + session.add(upload) + session.commit() + + body = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}").json() + assert body["status"] == "completed" + assert body["retry_after_seconds"] is None + field = body["detected_fields"][0] + assert field["detected_label"] == "Incident No." + assert field["field"]["incident_mapping"] == "report_metadata.incident_number" + assert field["suggestions"][0]["score"] == 0.93 + + +def test_draft_poll_reports_a_failed_detection(client, pdf_bytes, upload_dir, no_celery, test_engine): + from uuid import UUID + + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + with Session(test_engine) as session: + upload = session.get(TemplateUpload, UUID(upload_id)) + upload.status = DetectionStatus.failed + upload.detection_error = "model download failed" + session.add(upload) + session.commit() + + body = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}").json() + assert body["status"] == "failed" + assert body["detection_error"] == "model download failed" + # The upload is still usable, geometry and all. + assert body["page_count"] == 1 + + +def test_draft_poll_unknown_upload_is_404(client): + resp = client.get(f"{TEMPLATES_URL}/pdf/{uuid4()}") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "UPLOAD_NOT_FOUND" + + +def test_pdf_path_does_not_shadow_a_template_id(client, pdf_bytes, upload_dir, no_celery): + """The literal /pdf route has to win over /{template_id}.""" + assert _upload(client, pdf_bytes).status_code == 202 + + +# --------------------------------------------------------------------------- +# GET /templates/{template_id}/pdf +# --------------------------------------------------------------------------- +def _seed_template(session, pdf_template_ref): + template = FormTemplate( + form_type="state_texas", + display_name="Texas SFM", + fields=[], + pdf_template_ref=pdf_template_ref, + ) + session.add(template) + session.commit() + session.refresh(template) + return template.template_id + + +def test_download_source_pdf(client, pdf_bytes, upload_dir, no_celery, test_engine, tmp_path): + upload = _upload(client, pdf_bytes).json() + with Session(test_engine) as session: + template_id = _seed_template(session, upload["pdf_template_ref"]) + + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/pdf" + assert resp.content == pdf_bytes + + +def test_download_without_a_source_pdf_is_404(client, upload_dir, test_engine): + with Session(test_engine) as session: + template_id = _seed_template(session, None) + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_PDF_NOT_FOUND" + + +def test_download_missing_file_is_404(client, upload_dir, test_engine): + with Session(test_engine) as session: + template_id = _seed_template(session, "templates/uploads/gone.pdf") + assert client.get(f"{TEMPLATES_URL}/{template_id}/pdf").status_code == 404 + + +def test_download_cannot_escape_the_data_directory(client, upload_dir, test_engine, tmp_path): + outside = tmp_path.parent / "secret.pdf" + outside.write_bytes(b"%PDF-1.4 secret") + with Session(test_engine) as session: + template_id = _seed_template(session, "../secret.pdf") + + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 404 + + +def test_download_unknown_template_is_404(client): + assert client.get(f"{TEMPLATES_URL}/{uuid4()}/pdf").status_code == 404 diff --git a/tests/test_templates_v1.py b/tests/test_templates_v1.py new file mode 100644 index 00000000..50467a6b --- /dev/null +++ b/tests/test_templates_v1.py @@ -0,0 +1,379 @@ +"""Tests for the contract Layer 6 template registry (app/api/routes/form_templates.py). + +Covers list / create / get / replace / fields against the in-memory DB. +""" + +from app.core.config import API_PREFIX + +TEMPLATES_URL = f"{API_PREFIX}/templates" + + +def _layout(**over) -> dict: + base = {"page": 0, "x": 188.33, "y": 621.33, "width": 127.33, "height": 28.67} + base.update(over) + return base + + +def _payload(form_type: str = "state_texas") -> dict: + return { + "form_type": form_type, + "display_name": "Texas State Fire Marshal Incident Report", + "jurisdiction": "US-TX", + "agency_type": "fire_department", + "fields": [ + { + "field_name": "incident_number", + "field_type": "string", + "source": "schema", + "required": True, + "max_length": 20, + "description": "State-assigned incident number", + "incident_mapping": "report_metadata.incident_number", + "layout": _layout(font="Helvetica", font_size=10, color="#000000", align="left"), + }, + { + "field_name": "fire_cause", + "field_type": "enum", + "source": "schema", + "required": False, + "allowed_values": ["accidental", "natural", "intentional", "undetermined"], + "incident_mapping": "fire.cause_category", + "layout": _layout(y=560.0), + }, + ], + "source_standard": "Texas SFM 2026", + } + + +def _create(client, **overrides): + body = _payload(**overrides) + return client.post(TEMPLATES_URL, json=body) + + +def test_list_empty(client): + resp = client.get(TEMPLATES_URL) + assert resp.status_code == 200 + assert resp.json() == [] + + +def test_create_returns_201_with_server_fields(client): + resp = _create(client) + assert resp.status_code == 201 + body = resp.json() + + assert body["form_type"] == "state_texas" + assert body["display_name"] == "Texas State Fire Marshal Incident Report" + assert body["field_count"] == 2 + assert body["status"] == "active" + assert body["version"] == "1.0" + assert body["template_id"] + assert body["last_updated"] + assert body["created_at"] + assert len(body["fields"]) == 2 + + +def test_create_duplicate_form_type_returns_409(client): + assert _create(client).status_code == 201 + dup = _create(client) + assert dup.status_code == 409 + assert dup.json()["error_code"] == "TEMPLATE_EXISTS" + + +def test_create_then_list(client): + _create(client) + resp = client.get(TEMPLATES_URL) + assert resp.status_code == 200 + items = resp.json() + assert len(items) == 1 + assert items[0]["form_type"] == "state_texas" + assert items[0]["field_count"] == 2 + # Summary is a projection — no full field list. + assert "fields" not in items[0] + + +def test_get_by_id(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["template_id"] == template_id + first = body["fields"][0] + assert first["incident_mapping"] == "report_metadata.incident_number" + assert first["layout"]["x"] == 188.33 + assert first["layout"]["align"] == "left" + + +def test_get_missing_returns_404(client): + resp = client.get(f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_NOT_FOUND" + + +def test_get_invalid_uuid_returns_422(client): + assert client.get(f"{TEMPLATES_URL}/not-a-uuid").status_code == 422 + + +def test_replace_updates_fields(client): + template_id = _create(client).json()["template_id"] + + updated = _payload() + updated["display_name"] = "Texas SFM Incident Report v2" + updated["fields"] = [updated["fields"][0]] # drop one field + + resp = client.put(f"{TEMPLATES_URL}/{template_id}", json=updated) + assert resp.status_code == 200 + body = resp.json() + assert body["display_name"] == "Texas SFM Incident Report v2" + assert body["field_count"] == 1 + assert body["template_id"] == template_id + + +def test_replace_onto_a_taken_form_type_returns_409(client): + first = _create(client, form_type="tx_sfm_incident").json()["template_id"] + _create(client, form_type="tx_sfm_casualty") + + resp = client.put( + f"{TEMPLATES_URL}/{first}", json=_payload(form_type="tx_sfm_casualty") + ) + assert resp.status_code == 409 + assert resp.json()["error_code"] == "TEMPLATE_EXISTS" + + +def test_replace_keeping_its_own_form_type_is_not_a_conflict(client): + template_id = _create(client).json()["template_id"] + resp = client.put(f"{TEMPLATES_URL}/{template_id}", json=_payload()) + assert resp.status_code == 200 + + +def test_replace_missing_returns_404(client): + resp = client.put( + f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099", json=_payload() + ) + assert resp.status_code == 404 + + +def test_create_missing_required_field_returns_422(client): + body = _payload() + del body["fields"] + assert client.post(TEMPLATES_URL, json=body).status_code == 422 + + +def test_fields_endpoint(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}/fields") + assert resp.status_code == 200 + body = resp.json() + assert body["total_fields"] == 2 + assert body["required_fields"] == 1 + assert body["optional_fields"] == 1 + assert len(body["fields"]) == 2 + assert body["form_type"] == "state_texas" + + +def test_fields_required_only(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}/fields?required_only=true") + assert resp.status_code == 200 + body = resp.json() + assert body["total_fields"] == 2 + assert body["required_fields"] == 1 + assert len(body["fields"]) == 1 + assert body["fields"][0]["field_name"] == "incident_number" + + +def test_fields_missing_returns_404(client): + resp = client.get( + f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099/fields" + ) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Validation (all 422 with the contract error envelope) +# --------------------------------------------------------------------------- +def _assert_422(resp): + assert resp.status_code == 422, resp.json() + body = resp.json() + assert body["error_code"] == "VALIDATION_ERROR" + assert len(body["validation_errors"]) >= 1 + + +def test_jurisdiction_optional(client): + body = _payload() + del body["jurisdiction"] + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["jurisdiction"] is None + + +def test_static_text_field_ok(client): + body = _payload() + body["fields"].append({ + "field_name": "footer", + "field_type": "string", + "source": "static", + "required": False, + "static_text": "Generated by FireForm", + "layout": _layout(y=40.0, align="center"), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["field_count"] == 3 + + +def test_schema_field_without_mapping_rejected(client): + body = _payload() + del body["fields"][0]["incident_mapping"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_static_text_on_a_schema_field_rejected(client): + body = _payload() + body["fields"][0]["static_text"] = "x" # source is schema + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_static_field_without_text_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "footer", + "field_type": "string", + "source": "static", + "required": False, + "layout": _layout(y=40.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_mapping_on_a_manual_field_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "marshal_name", + "field_type": "string", + "source": "manual", + "required": True, + "incident_mapping": "report_metadata.incident_number", + "layout": _layout(y=120.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_manual_field_needs_nothing_else(client): + body = _payload() + body["fields"].append({ + "field_name": "marshal_name", + "field_type": "string", + "source": "manual", + "required": True, + "layout": _layout(y=120.0), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201, resp.json() + assert resp.json()["field_count"] == 3 + + +def test_open_field_without_description_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "insurance_company", + "field_type": "string", + "source": "open", + "required": False, + "layout": _layout(y=520.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_open_field_with_description_ok(client): + body = _payload() + body["fields"].append({ + "field_name": "insurance_company", + "field_type": "string", + "source": "open", + "required": False, + "description": "Name of the insurance company covering the property", + "layout": _layout(y=520.0), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201, resp.json() + + +def test_missing_source_rejected(client): + body = _payload() + del body["fields"][0]["source"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_unit_is_kept(client): + body = _payload() + body["fields"][0]["unit"] = "acres" + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["fields"][0]["unit"] == "acres" + + +def test_negative_min_value_allowed(client): + body = _payload() + body["fields"][0]["min_value"] = -40 + body["fields"][0]["max_value"] = 50 + assert client.post(TEMPLATES_URL, json=body).status_code == 201 + + +def test_enum_without_allowed_values_rejected(client): + body = _payload() + del body["fields"][1]["allowed_values"] # fire_cause is enum + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_duplicate_field_names_rejected(client): + body = _payload() + body["fields"][1]["field_name"] = body["fields"][0]["field_name"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_empty_fields_rejected(client): + body = _payload() + body["fields"] = [] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_bad_form_type_rejected(client): + _assert_422(_create(client, form_type="State Texas!")) + + +def test_min_greater_than_max_rejected(client): + body = _payload() + body["fields"][0]["min_value"] = 10 + body["fields"][0]["max_value"] = 5 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_bad_color_rejected(client): + body = _payload() + body["fields"][0]["layout"]["color"] = "black" + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_missing_required_coord_rejected(client): + body = _payload() + del body["fields"][0]["layout"]["width"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_negative_coordinate_rejected(client): + body = _payload() + body["fields"][0]["layout"]["x"] = -5 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_zero_width_rejected(client): + body = _payload() + body["fields"][0]["layout"]["width"] = 0 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_replace_validates_body(client): + template_id = _create(client).json()["template_id"] + bad = _payload() + bad["fields"][0]["layout"]["color"] = "nope" + _assert_422(client.put(f"{TEMPLATES_URL}/{template_id}", json=bad)) diff --git a/tests/test_v1_enums.py b/tests/test_v1_enums.py new file mode 100644 index 00000000..6a7ec373 --- /dev/null +++ b/tests/test_v1_enums.py @@ -0,0 +1,18 @@ +from app.api.schemas.enums import FieldSource + + +class TestFieldSource: + def test_members_and_values(self): + assert FieldSource.schema.value == "schema" + assert FieldSource.static.value == "static" + assert FieldSource.manual.value == "manual" + assert FieldSource.open.value == "open" + + def test_is_str_enum(self): + assert FieldSource.manual == "manual" + assert set(FieldSource) == { + FieldSource.schema, + FieldSource.static, + FieldSource.manual, + FieldSource.open, + } diff --git a/tests/test_v1_extraction.py b/tests/test_v1_extraction.py new file mode 100644 index 00000000..94d54feb --- /dev/null +++ b/tests/test_v1_extraction.py @@ -0,0 +1,268 @@ +"""Tests for POST /api/v1/extract/{input_id} and GET /api/v1/extract/{extract_id}. + +Endpoint tests only — dispatch is mocked (no broker) and the LLM provider's +health is patched. The real chunked worker lands in #630, so there is no task +unit here; the stub task is exercised indirectly by the POST dispatch +assertions. +""" + +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch +from uuid import UUID, uuid4 + +from app.api.schemas.enums import ExtractionStatus, InputStatus, InputType, ReportStatus +from app.services.llm.models import ProviderHealth +from app.db.repositories import ( + create_extraction, + create_incident, + create_input, + get_extraction, + get_job_by_uuid, +) +from app.models import Extraction, Incident, Input + +POST_URL = "/api/v1/extract" +GET_URL = "/api/v1/extract" + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": {"name": "Bear Creek Wildfire"}, + "location": {"city": "Reno", "state": "NV"}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _provider(up: bool) -> ProviderHealth: + """The provider health the route checks before it accepts a run.""" + return ProviderHealth( + provider="ollama", + label="Ollama", + model="qwen2.5:1.5b", + external=False, + status="healthy" if up else "unhealthy", + probed=True, + detail=None if up else "connection refused", + ) + + +def _ready_input(db, status=InputStatus.ready) -> Input: + now = datetime.now(timezone.utc) + record = Input( + input_type=InputType.text, + status=status, + transcript="Structure fire at 42 Oak St, two engines on scene, one civilian injury.", + character_count=70, + word_count=13, + created_at=now, + updated_at=now, + ) + return create_input(db, record) + + +def _completed_extraction_with_incident(db, input_id) -> tuple[Extraction, Incident]: + now = datetime.now(timezone.utc) + extraction = create_extraction( + db, + Extraction( + input_id=input_id, + status=ExtractionStatus.completed, + started_at=now, + completed_at=now, + model_used="qwen2.5:1.5b", + processing_time_seconds=42.0, + ), + ) + incident = create_incident( + db, + Incident( + extract_id=extraction.extract_id, + status=ReportStatus.draft, + incident_contract=_CONTRACT, + ), + ) + return extraction, incident + + +# --------------------------------------------------------------------------- +# POST /api/v1/extract/{input_id} +# --------------------------------------------------------------------------- + +class TestCreateExtraction: + + def _post(self, client, input_id, body=None, ollama_up=True, celery_id="celery-extract-001"): + mock_result = MagicMock() + mock_result.id = celery_id + with patch("app.api.routes.extraction.llm.health", return_value=_provider(ollama_up)), \ + patch("app.services.extraction.service.extract_task") as mock_task: + mock_task.delay.return_value = mock_result + resp = client.post(f"{POST_URL}/{input_id}", json=body) + return resp, mock_task + + def test_202_returns_required_fields(self, client, db): + inp = _ready_input(db) + resp, _ = self._post(client, inp.input_id) + assert resp.status_code == 202 + body = resp.json() + assert body["status"] == "processing" + assert body["job_type"] == "extraction" + assert body["input_id"] == str(inp.input_id) + assert "extract_id" in body + assert "job_id" in body + assert body["poll_url"] == f"/api/v1/extract/{body['extract_id']}" + assert body["estimated_seconds"] == 60 + + def test_202_creates_processing_extraction_row(self, client, db): + inp = _ready_input(db) + resp, _ = self._post(client, inp.input_id) + extraction = get_extraction(db, UUID(resp.json()["extract_id"])) + assert extraction is not None + assert extraction.status == ExtractionStatus.processing + assert extraction.input_id == inp.input_id + assert extraction.started_at is not None + + def test_202_creates_extraction_job_row(self, client, db): + inp = _ready_input(db) + resp, _ = self._post(client, inp.input_id) + job = get_job_by_uuid(db, resp.json()["job_id"]) + assert job is not None + assert job.job_type == "extraction" + assert job.celery_task_id == "celery-extract-001" + + def test_202_dispatch_called_with_extract_id_and_job_id(self, client, db): + inp = _ready_input(db) + resp, mock_task = self._post(client, inp.input_id) + body = resp.json() + mock_task.delay.assert_called_once() + args = mock_task.delay.call_args[0] + assert args[0] == body["extract_id"] + assert args[1] == body["job_id"] + + def test_202_model_override_stored_on_job(self, client, db): + inp = _ready_input(db) + resp, _ = self._post(client, inp.input_id, body={"model_override": "llama3:8b"}) + job = get_job_by_uuid(db, resp.json()["job_id"]) + assert job.model == "llama3:8b" + + def test_404_input_not_found(self, client, db): + resp, _ = self._post(client, uuid4()) + assert resp.status_code == 404 + assert resp.json()["error_code"] == "INPUT_NOT_FOUND" + + def test_409_input_not_ready(self, client, db): + inp = _ready_input(db, status=InputStatus.transcribing) + resp, _ = self._post(client, inp.input_id) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "INPUT_NOT_READY" + assert body["detail"]["current_status"] == "transcribing" + + def test_409_extraction_already_exists(self, client, db): + inp = _ready_input(db) + existing = create_extraction(db, Extraction(input_id=inp.input_id)) + # Pinned rather than left to the environment, so a developer running + # with the rerun flag on still tests the shipped behaviour. + with patch("app.api.routes.extraction.EXTRACTION_ALLOW_RERUN", False): + resp, _ = self._post(client, inp.input_id) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "EXTRACTION_EXISTS" + assert body["detail"]["existing_extract_id"] == str(existing.extract_id) + + def test_202_rerun_allowed_when_flag_is_on(self, client, db): + # Development escape hatch: the same narrative can be extracted again + # instead of having to be re-uploaded. The earlier extraction stays. + inp = _ready_input(db) + existing = create_extraction(db, Extraction(input_id=inp.input_id)) + with patch("app.api.routes.extraction.EXTRACTION_ALLOW_RERUN", True): + resp, _ = self._post(client, inp.input_id) + assert resp.status_code == 202 + assert resp.json()["extract_id"] != str(existing.extract_id) + assert get_extraction(db, existing.extract_id) is not None + + def test_503_ollama_unavailable(self, client, db): + inp = _ready_input(db) + resp, _ = self._post(client, inp.input_id, ollama_up=False) + assert resp.status_code == 503 + assert resp.json()["error_code"] == "LLM_UNAVAILABLE" + + def test_422_non_json_body_does_not_500(self, client, db): + # A wrong content-type puts the raw bytes body on the validation error; + # the error handler must still render a 422 rather than blow up on + # serializing bytes. + inp = _ready_input(db) + with patch("app.api.routes.extraction.llm.health", return_value=_provider(True)): + resp = client.post( + f"{POST_URL}/{inp.input_id}", + content=b"{}", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + assert resp.status_code == 422 + assert resp.json()["error_code"] == "VALIDATION_ERROR" + + +# --------------------------------------------------------------------------- +# GET /api/v1/extract/{extract_id} +# --------------------------------------------------------------------------- + +class TestGetExtraction: + + def test_200_processing_shape(self, client, db): + inp = _ready_input(db) + extraction = create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=ExtractionStatus.processing, + started_at=datetime.now(timezone.utc), + ), + ) + resp = client.get(f"{GET_URL}/{extraction.extract_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "processing" + assert body["extract_id"] == str(extraction.extract_id) + assert body["input_id"] == str(inp.input_id) + assert body["retry_after_seconds"] == 5 + assert "incident_contract" not in body + + def test_200_completed_shape_embeds_contract(self, client, db): + inp = _ready_input(db) + extraction, incident = _completed_extraction_with_incident(db, inp.input_id) + resp = client.get(f"{GET_URL}/{extraction.extract_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "completed" + assert body["incident_id"] == str(incident.incident_id) + assert body["model_used"] == "qwen2.5:1.5b" + assert body["processing_time_seconds"] == 42.0 + assert body["incident_contract"]["incident"]["name"] == "Bear Creek Wildfire" + assert body["incident_contract"]["location"]["city"] == "Reno" + + def test_200_failed_shape(self, client, db): + inp = _ready_input(db) + extraction = create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=ExtractionStatus.failed, + started_at=datetime.now(timezone.utc), + error_type="LLM_UNAVAILABLE", + error_detail="Ollama connection refused", + ), + ) + resp = client.get(f"{GET_URL}/{extraction.extract_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "failed" + assert body["error_type"] == "LLM_UNAVAILABLE" + assert body["error_detail"] == "Ollama connection refused" + assert body["retry_after_seconds"] is None + + def test_404_extraction_not_found(self, client, db): + resp = client.get(f"{GET_URL}/{uuid4()}") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "EXTRACT_NOT_FOUND" diff --git a/tests/test_v1_extraction_corrections.py b/tests/test_v1_extraction_corrections.py new file mode 100644 index 00000000..f1e4f0e9 --- /dev/null +++ b/tests/test_v1_extraction_corrections.py @@ -0,0 +1,302 @@ +"""Tests for PATCH /api/v1/extract/{extract_id}. + +The review-screen write path: a merge patch lands on the contract document +held by the incident row, the promoted analytics columns are recomputed from +it, and every real change is appended to the corrections trail. +""" + +from datetime import datetime, timezone +from uuid import uuid4 + +from app.api.schemas.enums import ExtractionStatus, InputStatus, InputType, ReportStatus +from app.db.repositories import ( + create_extraction, + create_incident, + create_input, + get_extraction, + get_incident, +) +from app.api.schemas.incident_contract import IncidentContract +from app.models import Extraction, Incident, Input +from app.services.extraction_review import merge_patch, patch_paths, unknown_paths + +URL = "/api/v1/extract" +MERGE_PATCH = {"Content-Type": "application/merge-patch+json"} + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": {"name": "Bear Creek Wildfire"}, + "location": {"city": "Reno", "state": "NV", "country": "US"}, + "casualties": {"total_civilian_injuries": 1, "total_responder_injuries": 0}, + "losses": {"property_loss": {"amount": 10000, "currency": "USD"}}, + "fire": {"cause_certainty": "suspected"}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _seed( + db, + contract=None, + extraction_status=ExtractionStatus.completed, + report_status=ReportStatus.draft, +) -> tuple[Extraction, Incident]: + now = datetime.now(timezone.utc) + inp = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript="Structure fire at 42 Oak St, one civilian injury.", + created_at=now, + updated_at=now, + ), + ) + extraction = create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=extraction_status, + started_at=now, + completed_at=now if extraction_status == ExtractionStatus.completed else None, + model_used="qwen2.5:1.5b", + ), + ) + incident = create_incident( + db, + Incident( + extract_id=extraction.extract_id, + status=report_status, + incident_contract=_CONTRACT if contract is None else contract, + ), + ) + return extraction, incident + + +def _patch(client, extract_id, body): + return client.patch(f"{URL}/{extract_id}", json=body, headers=MERGE_PATCH) + + +# --------------------------------------------------------------------------- +# The merge itself +# --------------------------------------------------------------------------- + +class TestMergePatch: + + def test_nested_keys_merge_not_replace(self): + target = {"losses": {"property_loss": {"amount": 1, "currency": "USD"}}} + merged = merge_patch(target, {"losses": {"property_loss": {"amount": 2}}}) + assert merged["losses"]["property_loss"] == {"amount": 2, "currency": "USD"} + + def test_null_deletes_the_key(self): + merged = merge_patch({"a": 1, "b": 2}, {"b": None}) + assert merged == {"a": 1} + + def test_null_for_absent_key_is_a_no_op(self): + assert merge_patch({"a": 1}, {"b": None}) == {"a": 1} + + def test_list_is_replaced_whole(self): + merged = merge_patch({"units": [{"id": "E1"}, {"id": "E2"}]}, {"units": [{"id": "E3"}]}) + assert merged["units"] == [{"id": "E3"}] + + def test_target_is_not_mutated(self): + target = {"incident": {"name": "old"}} + merge_patch(target, {"incident": {"name": "new"}}) + assert target["incident"]["name"] == "old" + + def test_patch_paths_walks_to_leaves(self): + paths = dict(patch_paths({"losses": {"property_loss": {"amount": 250000}}, "a": None})) + assert paths == {"losses.property_loss.amount": 250000, "a": None} + + def test_unknown_paths_reports_dotted_path(self): + assert unknown_paths({"losses": {"nope": 1}}, IncidentContract) == ["losses.nope"] + + def test_custom_fields_keys_are_open(self): + patch = {"custom_fields": {"state_texas.marshal_signature_name": "A. Ruiz"}} + assert unknown_paths(patch, IncidentContract) == [] + + +# --------------------------------------------------------------------------- +# PATCH /api/v1/extract/{extract_id} +# --------------------------------------------------------------------------- + +class TestUpdateExtraction: + + def test_200_applies_the_correction(self, client, db): + extraction, incident = _seed(db) + resp = _patch( + client, + extraction.extract_id, + {"losses": {"property_loss": {"amount": 250000}}}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "completed" + assert body["incident_id"] == str(incident.incident_id) + loss = body["incident_contract"]["losses"]["property_loss"] + # The untouched sibling survives the merge. + assert loss == {"amount": 250000, "currency": "USD"} + + def test_200_writes_the_document_to_the_incident_row(self, client, db): + extraction, incident = _seed(db) + _patch(client, extraction.extract_id, {"incident": {"name": "Bear Creek Fire"}}) + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert stored.incident_contract["incident"]["name"] == "Bear Creek Fire" + + def test_200_recomputes_promoted_columns(self, client, db): + extraction, incident = _seed(db) + _patch( + client, + extraction.extract_id, + { + "casualties": {"total_responder_injuries": 2}, + "losses": {"property_loss": {"amount": 250000}}, + "location": {"city": "Sparks"}, + }, + ) + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert stored.responder_injuries == 2 + assert stored.total_loss_amount == 250000 + assert stored.total_loss_currency == "USD" + assert stored.city == "Sparks" + + def test_200_null_deletes_a_field(self, client, db): + extraction, incident = _seed(db) + resp = _patch(client, extraction.extract_id, {"fire": {"cause_certainty": None}}) + assert resp.status_code == 200 + assert resp.json()["incident_contract"]["fire"]["cause_certainty"] is None + # The stored document drops the key outright, per RFC 7396. The response + # still carries it as null because the contract model serializes every + # field, the same way GET does. + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert "cause_certainty" not in stored.incident_contract.get("fire", {}) + + def test_200_delete_clears_the_promoted_column(self, client, db): + extraction, incident = _seed(db) + _patch(client, extraction.extract_id, {"losses": {"property_loss": None}}) + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert stored.total_loss_amount is None + assert stored.total_loss_currency is None + + def test_200_records_the_audit_trail(self, client, db): + extraction, _ = _seed(db) + resp = _patch( + client, + extraction.extract_id, + {"casualties": {"total_responder_injuries": 2}}, + ) + corrections = resp.json()["corrections"] + assert len(corrections) == 1 + entry = corrections[0] + assert entry["field_path"] == "casualties.total_responder_injuries" + assert entry["original_value"] == 0 + assert entry["corrected_value"] == 2 + assert entry["corrected_at"] is not None + + def test_200_audit_trail_records_a_delete(self, client, db): + extraction, _ = _seed(db) + resp = _patch(client, extraction.extract_id, {"fire": {"cause_certainty": None}}) + entry = resp.json()["corrections"][0] + assert entry["field_path"] == "fire.cause_certainty" + assert entry["original_value"] == "suspected" + assert entry["corrected_value"] is None + + def test_200_audit_trail_appends_across_calls(self, client, db): + extraction, _ = _seed(db) + _patch(client, extraction.extract_id, {"incident": {"name": "First"}}) + resp = _patch(client, extraction.extract_id, {"incident": {"name": "Second"}}) + db.expire_all() + stored = get_extraction(db, extraction.extract_id) + assert len(stored.corrections) == 2 + assert [c["corrected_value"] for c in resp.json()["corrections"]] == ["First", "Second"] + + def test_200_unchanged_value_is_not_recorded(self, client, db): + extraction, _ = _seed(db) + resp = _patch(client, extraction.extract_id, {"incident": {"name": "Bear Creek Wildfire"}}) + assert resp.status_code == 200 + assert not resp.json()["corrections"] + + def test_200_adds_a_custom_field(self, client, db): + extraction, _ = _seed(db) + resp = _patch( + client, + extraction.extract_id, + {"custom_fields": {"state_texas.marshal_signature_name": "A. Ruiz"}}, + ) + assert resp.status_code == 200 + custom = resp.json()["incident_contract"]["custom_fields"] + assert custom["state_texas.marshal_signature_name"] == "A. Ruiz" + + def test_404_extraction_not_found(self, client, db): + resp = _patch(client, uuid4(), {"incident": {"name": "x"}}) + assert resp.status_code == 404 + assert resp.json()["error_code"] == "EXTRACT_NOT_FOUND" + + def test_409_extraction_still_processing(self, client, db): + extraction, _ = _seed(db, extraction_status=ExtractionStatus.processing) + resp = _patch(client, extraction.extract_id, {"incident": {"name": "x"}}) + assert resp.status_code == 409 + assert resp.json()["error_code"] == "EXTRACT_NOT_COMPLETED" + + def test_409_locked_after_submission(self, client, db): + extraction, incident = _seed(db, report_status=ReportStatus.submitted) + resp = _patch(client, extraction.extract_id, {"incident": {"name": "x"}}) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "EXTRACT_LOCKED" + assert body["detail"]["report_status"] == "submitted" + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert stored.incident_contract["incident"]["name"] == "Bear Creek Wildfire" + + def test_422_invalid_enum_value(self, client, db): + extraction, _ = _seed(db) + resp = _patch(client, extraction.extract_id, {"fire": {"cause_certainty": "maybe"}}) + assert resp.status_code == 422 + body = resp.json() + assert body["error_code"] == "VALIDATION_ERROR" + assert body["validation_errors"][0]["field"] == "fire.cause_certainty" + assert body["validation_errors"][0]["value"] == "maybe" + + def test_422_unknown_field_path(self, client, db): + extraction, _ = _seed(db) + resp = _patch(client, extraction.extract_id, {"losses": {"imaginary_loss": 5}}) + assert resp.status_code == 422 + assert resp.json()["validation_errors"][0]["field"] == "losses.imaginary_loss" + + def test_422_wrong_type_for_a_field(self, client, db): + extraction, _ = _seed(db) + resp = _patch( + client, + extraction.extract_id, + {"casualties": {"total_responder_injuries": "two"}}, + ) + assert resp.status_code == 422 + assert resp.json()["validation_errors"][0]["field"] == ( + "casualties.total_responder_injuries" + ) + + def test_422_leaves_the_document_untouched(self, client, db): + extraction, incident = _seed(db) + _patch(client, extraction.extract_id, {"fire": {"cause_certainty": "maybe"}}) + db.expire_all() + stored = get_incident(db, incident.incident_id) + assert stored.incident_contract["fire"]["cause_certainty"] == "suspected" + assert get_extraction(db, extraction.extract_id).corrections is None + + def test_422_non_json_body_does_not_500(self, client, db): + extraction, _ = _seed(db) + resp = client.patch( + f"{URL}/{extraction.extract_id}", + content=b"not json", + headers={"Content-Type": "text/plain"}, + ) + assert resp.status_code == 422 + assert resp.json()["error_code"] == "VALIDATION_ERROR" diff --git a/tests/test_v1_extraction_readiness.py b/tests/test_v1_extraction_readiness.py new file mode 100644 index 00000000..8365e311 --- /dev/null +++ b/tests/test_v1_extraction_readiness.py @@ -0,0 +1,441 @@ +"""Tests for the read side of the review screen. + +GET /api/v1/extract/{extract_id}/readiness answers which registered forms can +be generated from what was extracted, and POST /api/v1/extract/{extract_id}/validate +answers the same question for one template. Both run the same engine, so the +gap rules are tested once on the engine and then confirmed through the routes. +""" + +from datetime import datetime, timezone + +from app.api.schemas.enums import ( + ExtractionStatus, + InputStatus, + InputType, + ReportStatus, + TemplateStatus, +) +from app.api.schemas.templates import TemplateField +from app.db.repositories import ( + create_extraction, + create_form_template, + create_incident, + create_input, +) +from app.models import Extraction, FormTemplate, Incident, Input +from app.services.extraction_readiness import ( + gaps_for, + is_filled, + resolve, + warnings_for, +) + +URL = "/api/v1/extract" + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": {"name": "Bear Creek Wildfire"}, + "location": {"city": "Reno", "state": "NV", "country": "US"}, + "fire": {"cause_certainty": "suspected"}, + "custom_fields": {"state_texas.marshal_signature_name": "A. Ruiz"}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _field(name, source="schema", required=True, **extra) -> dict: + field = { + "field_name": name, + "field_type": "string", + "source": source, + "required": required, + } + if source == "schema": + field.setdefault("incident_mapping", "incident.name") + if source == "static": + field.setdefault("static_text", "Reno Fire Department") + if source == "open": + field.setdefault("description", "Anything the narrative says about it") + field.update(extra) + return field + + +def _template( + db, + form_type="state_texas", + display_name="Texas SFM", + fields=None, + status=TemplateStatus.active, +) -> FormTemplate: + return create_form_template( + db, + FormTemplate( + form_type=form_type, + display_name=display_name, + fields=fields if fields is not None else [_field("incident_name")], + status=status, + ), + ) + + +def _seed( + db, + contract=None, + extraction_status=ExtractionStatus.completed, +) -> tuple[Extraction, Incident]: + now = datetime.now(timezone.utc) + inp = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript="Wildfire off Bear Creek, no injuries.", + created_at=now, + updated_at=now, + ), + ) + extraction = create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=extraction_status, + started_at=now, + completed_at=now if extraction_status == ExtractionStatus.completed else None, + model_used="qwen2.5:1.5b", + ), + ) + incident = create_incident( + db, + Incident( + extract_id=extraction.extract_id, + status=ReportStatus.draft, + incident_contract=_CONTRACT if contract is None else contract, + ), + ) + return extraction, incident + + +def _validate(client, extract_id, template_id): + return client.post(f"{URL}/{extract_id}/validate", json={"template_id": str(template_id)}) + + +# --------------------------------------------------------------------------- +# When a field counts as filled +# --------------------------------------------------------------------------- + +class TestIsFilled: + + def test_null_is_a_gap(self): + assert is_filled(None) is False + + def test_blank_string_is_a_gap(self): + assert is_filled("") is False + assert is_filled(" ") is False + + def test_empty_container_is_a_gap(self): + assert is_filled([]) is False + assert is_filled({}) is False + + def test_zero_and_false_are_values(self): + assert is_filled(0) is True + assert is_filled(False) is True + + def test_text_and_numbers_are_values(self): + assert is_filled("Reno") is True + assert is_filled(10000) is True + + +# --------------------------------------------------------------------------- +# Where a value comes from +# --------------------------------------------------------------------------- + +class TestResolve: + + def test_schema_field_reads_its_contract_path(self): + field = TemplateField.model_validate( + _field("city", incident_mapping="location.city") + ) + assert resolve(_CONTRACT, field, "state_texas") == "Reno" + + def test_missing_path_resolves_to_none(self): + field = TemplateField.model_validate( + _field("loss", incident_mapping="losses.property_loss.amount") + ) + assert resolve(_CONTRACT, field, "state_texas") is None + + def test_static_field_carries_its_own_text(self): + field = TemplateField.model_validate(_field("agency", source="static")) + assert resolve({}, field, "state_texas") == "Reno Fire Department" + + def test_manual_field_reads_the_flat_custom_fields_key(self): + field = TemplateField.model_validate( + _field("marshal_signature_name", source="manual") + ) + assert resolve(_CONTRACT, field, "state_texas") == "A. Ruiz" + + def test_manual_field_of_another_form_type_does_not_match(self): + field = TemplateField.model_validate( + _field("marshal_signature_name", source="manual") + ) + assert resolve(_CONTRACT, field, "neris") is None + + def test_contract_without_custom_fields_resolves_to_none(self): + field = TemplateField.model_validate(_field("notes", source="open")) + assert resolve({"incident": {"name": "x"}}, field, "state_texas") is None + + +# --------------------------------------------------------------------------- +# Gaps and coverage +# --------------------------------------------------------------------------- + +class TestGaps: + + def test_required_gap_blocks_and_optional_gap_does_not(self, db): + template = _template( + db, + fields=[ + _field("city", incident_mapping="location.city"), + _field("loss", required=True, incident_mapping="losses.property_loss.amount"), + _field("alarm", required=False, incident_mapping="risk_reduction.smoke_alarm"), + ], + ) + gaps = gaps_for(_CONTRACT, template) + + assert gaps.ready is False + assert [g.field_name for g in gaps.missing_required] == ["loss"] + assert [g.field_name for g in gaps.missing_recommended] == ["alarm"] + + def test_ready_when_every_required_field_resolves(self, db): + template = _template( + db, + fields=[ + _field("city", incident_mapping="location.city"), + _field("alarm", required=False, incident_mapping="risk_reduction.smoke_alarm"), + ], + ) + gaps = gaps_for(_CONTRACT, template) + + assert gaps.ready is True + assert gaps.missing_required == [] + + def test_coverage_counts_filled_over_total(self, db): + template = _template( + db, + fields=[ + _field("city", incident_mapping="location.city"), + _field("agency", source="static"), + _field("loss", incident_mapping="losses.property_loss.amount"), + _field("cause", incident_mapping="fire.cause_certainty"), + ], + ) + assert gaps_for(_CONTRACT, template).coverage_percent == 75.0 + + def test_template_without_fields_is_ready(self, db): + template = _template(db, fields=[]) + gaps = gaps_for(_CONTRACT, template) + + assert gaps.ready is True + assert gaps.coverage_percent == 0.0 + + def test_gap_points_a_schema_field_at_its_contract_path(self, db): + template = _template( + db, fields=[_field("loss", incident_mapping="losses.property_loss.amount")] + ) + gap = gaps_for(_CONTRACT, template).missing_required[0] + + assert gap.source == "schema" + assert gap.incident_mapping == "losses.property_loss.amount" + + def test_gap_points_a_manual_field_at_its_custom_fields_key(self, db): + template = _template(db, fields=[_field("chief_name", source="manual")]) + gap = gaps_for(_CONTRACT, template).missing_required[0] + + assert gap.source == "manual" + assert gap.incident_mapping == "custom_fields.state_texas.chief_name" + + def test_warnings_describe_the_recommended_gaps(self, db): + template = _template( + db, + fields=[ + _field( + "alarm", + required=False, + incident_mapping="risk_reduction.smoke_alarm", + description="NERIS recommends damage estimates", + ) + ], + ) + warnings = warnings_for(gaps_for(_CONTRACT, template).missing_recommended) + + assert len(warnings) == 1 + assert "risk_reduction.smoke_alarm" in warnings[0] + assert "NERIS recommends damage estimates" in warnings[0] + + +# --------------------------------------------------------------------------- +# POST /api/v1/extract/{extract_id}/validate +# --------------------------------------------------------------------------- + +class TestValidateExtraction: + + def test_200_valid_when_nothing_required_is_missing(self, client, db): + extraction, _ = _seed(db) + template = _template(db, fields=[_field("city", incident_mapping="location.city")]) + + resp = _validate(client, extraction.extract_id, template.template_id) + body = resp.json() + + assert resp.status_code == 200 + assert body["valid"] is True + assert body["form_type"] == "state_texas" + assert body["extract_id"] == str(extraction.extract_id) + assert body["missing_required"] == [] + assert body["field_coverage_percent"] == 100.0 + + def test_200_invalid_lists_the_blocking_fields(self, client, db): + extraction, _ = _seed(db) + template = _template( + db, + fields=[ + _field("city", incident_mapping="location.city"), + _field("loss", incident_mapping="losses.property_loss.amount"), + ], + ) + + body = _validate(client, extraction.extract_id, template.template_id).json() + + assert body["valid"] is False + assert [g["field_name"] for g in body["missing_required"]] == ["loss"] + + def test_correcting_the_contract_flips_it_to_valid(self, client, db): + extraction, _ = _seed(db) + template = _template( + db, fields=[_field("loss", incident_mapping="losses.property_loss.amount")] + ) + assert _validate(client, extraction.extract_id, template.template_id).json()["valid"] is False + + client.patch( + f"{URL}/{extraction.extract_id}", + json={"losses": {"property_loss": {"amount": 250000}}}, + headers={"Content-Type": "application/merge-patch+json"}, + ) + + assert _validate(client, extraction.extract_id, template.template_id).json()["valid"] is True + + def test_validate_checks_a_legacy_template_too(self, client, db): + extraction, _ = _seed(db) + template = _template( + db, + fields=[_field("city", incident_mapping="location.city")], + status=TemplateStatus.legacy, + ) + + resp = _validate(client, extraction.extract_id, template.template_id) + + assert resp.status_code == 200 + assert resp.json()["valid"] is True + + def test_404_when_the_extraction_is_unknown(self, client, db): + template = _template(db) + resp = _validate(client, "550e8400-e29b-41d4-a716-446655440099", template.template_id) + + assert resp.status_code == 404 + assert resp.json()["error_code"] == "EXTRACT_NOT_FOUND" + + def test_404_when_the_template_is_unknown(self, client, db): + extraction, _ = _seed(db) + resp = _validate( + client, extraction.extract_id, "550e8400-e29b-41d4-a716-446655440099" + ) + + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_NOT_FOUND" + + def test_409_while_the_extraction_is_still_running(self, client, db): + extraction, _ = _seed(db, extraction_status=ExtractionStatus.processing) + template = _template(db) + + resp = _validate(client, extraction.extract_id, template.template_id) + + assert resp.status_code == 409 + assert resp.json()["error_code"] == "EXTRACT_NOT_COMPLETED" + + +# --------------------------------------------------------------------------- +# GET /api/v1/extract/{extract_id}/readiness +# --------------------------------------------------------------------------- + +class TestReadiness: + + def test_200_reports_every_active_template(self, client, db): + extraction, _ = _seed(db) + _template(db, form_type="neris", display_name="NERIS Incident Report") + _template( + db, + form_type="state_texas", + fields=[_field("loss", incident_mapping="losses.property_loss.amount")], + ) + + resp = client.get(f"{URL}/{extraction.extract_id}/readiness") + body = resp.json() + + assert resp.status_code == 200 + assert body["extract_id"] == str(extraction.extract_id) + assert body["computed_at"] + + rows = {row["form_type"]: row for row in body["templates"]} + assert rows["neris"]["ready"] is True + assert rows["neris"]["display_name"] == "NERIS Incident Report" + assert rows["state_texas"]["ready"] is False + assert rows["state_texas"]["missing_required"][0]["field_name"] == "loss" + + def test_drafts_and_legacy_templates_stay_out(self, client, db): + extraction, _ = _seed(db) + _template(db, form_type="neris") + _template(db, form_type="old_form", status=TemplateStatus.legacy) + _template(db, form_type="wip_form", status=TemplateStatus.draft) + + body = client.get(f"{URL}/{extraction.extract_id}/readiness").json() + + assert [row["form_type"] for row in body["templates"]] == ["neris"] + + def test_empty_registry_gives_an_empty_matrix(self, client, db): + extraction, _ = _seed(db) + + body = client.get(f"{URL}/{extraction.extract_id}/readiness").json() + + assert body["templates"] == [] + + def test_readiness_agrees_with_validate(self, client, db): + extraction, _ = _seed(db) + template = _template( + db, + fields=[ + _field("city", incident_mapping="location.city"), + _field("loss", incident_mapping="losses.property_loss.amount"), + ], + ) + + row = client.get(f"{URL}/{extraction.extract_id}/readiness").json()["templates"][0] + single = _validate(client, extraction.extract_id, template.template_id).json() + + assert row["ready"] == single["valid"] + assert row["missing_required"] == single["missing_required"] + assert row["field_coverage_percent"] == single["field_coverage_percent"] + + def test_404_when_the_extraction_is_unknown(self, client, db): + resp = client.get(f"{URL}/550e8400-e29b-41d4-a716-446655440099/readiness") + + assert resp.status_code == 404 + assert resp.json()["error_code"] == "EXTRACT_NOT_FOUND" + + def test_409_while_the_extraction_is_still_running(self, client, db): + extraction, _ = _seed(db, extraction_status=ExtractionStatus.processing) + + resp = client.get(f"{URL}/{extraction.extract_id}/readiness") + + assert resp.status_code == 409 + assert resp.json()["error_code"] == "EXTRACT_NOT_COMPLETED" diff --git a/tests/test_v1_extraction_schemas.py b/tests/test_v1_extraction_schemas.py new file mode 100644 index 00000000..f697df76 --- /dev/null +++ b/tests/test_v1_extraction_schemas.py @@ -0,0 +1,192 @@ +"""Validate the extraction schemas against the examples in the contract. + +Each example below is copied from contracts/path/extraction.yaml. The point is +to catch drift: if the contract example changes shape, the matching model must +still accept it (and vice versa). No routes are exercised here, only the +Pydantic models from app/api/schemas/extraction.py. +""" + +from app.api.schemas.enums import FieldSource +from app.api.schemas.extraction import ( + ExtractionCompleted, + ExtractionProcessing, + ExtractionRequest, + ReadinessMatrix, + ValidationResult, +) +from app.api.schemas.incident_contract import IncidentContract + + +# --------------------------------------------------------------------------- +# ExtractionRequest — POST /extract/{input_id} request body +# --------------------------------------------------------------------------- + +def test_extraction_request_example(): + example = { + "model_override": "llama3:8b", + "extraction_hints": { + "incident_type": "wildland_fire", + "state": "CA", + }, + "defaults": { + "country": "US", + "timezone": "America/Los_Angeles", + "currency": "USD", + }, + } + model = ExtractionRequest.model_validate(example) + assert model.model_dump(exclude_none=True) == example + + +def test_extraction_request_hints_allow_extra_keys(): + # extraction_hints is additionalProperties: true in the contract. + model = ExtractionRequest.model_validate( + {"extraction_hints": {"incident_type": "structure_fire", "battalion": "3"}} + ) + assert model.extraction_hints.incident_type == "structure_fire" + dumped = model.model_dump(exclude_none=True) + assert dumped["extraction_hints"]["battalion"] == "3" + + +def test_extraction_request_all_fields_optional(): + assert ExtractionRequest.model_validate({}).model_dump(exclude_none=True) == {} + + +# --------------------------------------------------------------------------- +# ExtractionCompleted / ExtractionProcessing — GET /extract/{extract_id} +# --------------------------------------------------------------------------- + +def test_extraction_completed_example(): + example = { + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "input_id": "550e8400-e29b-41d4-a716-446655440001", + "incident_id": "550e8400-e29b-41d4-a716-446655440050", + "status": "completed", + "completed_at": "2024-07-15T14:31:05Z", + "incident_contract": { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "extraction_metadata": { + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "confidence_score": 0.91, + }, + "incident": {"name": "Bear Creek Wildfire"}, + }, + } + model = ExtractionCompleted.model_validate(example) + assert model.status == "completed" + assert isinstance(model.incident_contract, IncidentContract) + assert model.incident_contract.incident.name == "Bear Creek Wildfire" + + +def test_extraction_processing_example(): + example = { + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "input_id": "550e8400-e29b-41d4-a716-446655440001", + "status": "processing", + "started_at": "2024-07-15T14:30:00Z", + "retry_after_seconds": 5, + } + model = ExtractionProcessing.model_validate(example) + assert model.status == "processing" + assert model.retry_after_seconds == 5 + + +def test_extraction_processing_failed_status_allowed(): + model = ExtractionProcessing.model_validate({ + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "input_id": "550e8400-e29b-41d4-a716-446655440001", + "status": "failed", + "error_type": "LLM_TIMEOUT", + "error_detail": "Ollama did not respond within the timeout", + }) + assert model.status == "failed" + assert model.error_type == "LLM_TIMEOUT" + + +# --------------------------------------------------------------------------- +# ReadinessMatrix — GET /extract/{extract_id}/readiness +# --------------------------------------------------------------------------- + +def test_readiness_matrix_example(): + example = { + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "computed_at": "2026-07-15T14:35:00Z", + "templates": [ + { + "template_id": "550e8400-e29b-41d4-a716-446655440070", + "form_type": "neris", + "display_name": "NERIS Incident Report", + "ready": True, + "missing_required": [], + "missing_recommended": [ + { + "field_name": "smoke_alarm_presence", + "source": "schema", + "incident_mapping": "risk_reduction.smoke_alarm.presence", + } + ], + "field_coverage_percent": 94, + }, + { + "template_id": "550e8400-e29b-41d4-a716-446655440073", + "form_type": "state_texas", + "display_name": "Texas State Fire Marshal Incident Report", + "ready": False, + "missing_required": [ + { + "field_name": "marshal_signature_name", + "source": "manual", + "incident_mapping": "custom_fields.state_texas.marshal_signature_name", + "description": "Reviewing marshal's printed name, entered per incident", + }, + { + "field_name": "fire_cause", + "source": "schema", + "incident_mapping": "fire.cause_category", + }, + ], + "missing_recommended": [], + "field_coverage_percent": 78, + }, + ], + } + model = ReadinessMatrix.model_validate(example) + assert len(model.templates) == 2 + # form_type is an open string: state_texas is not in the built-in list. + assert model.templates[1].form_type == "state_texas" + assert model.templates[1].missing_required[0].source is FieldSource.manual + + +# --------------------------------------------------------------------------- +# ValidationResult — POST /extract/{extract_id}/validate +# --------------------------------------------------------------------------- + +def test_validation_result_example(): + example = { + "valid": True, + "template_id": "550e8400-e29b-41d4-a716-446655440070", + "form_type": "neris", + "extract_id": "550e8400-e29b-41d4-a716-446655440020", + "missing_required": [], + "missing_recommended": [ + { + "field_name": "smoke_alarm_presence", + "source": "schema", + "incident_mapping": "risk_reduction.smoke_alarm.presence", + }, + { + "field_name": "smoke_alarm_operation", + "source": "schema", + "incident_mapping": "risk_reduction.smoke_alarm.operation", + }, + ], + "warnings": [ + "losses.property_loss is null. NERIS recommends providing damage estimates" + ], + "field_coverage_percent": 94, + } + model = ValidationResult.model_validate(example) + assert model.valid is True + assert len(model.missing_recommended) == 2 + assert model.missing_recommended[0].source is FieldSource.schema diff --git a/tests/test_v1_extraction_worker.py b/tests/test_v1_extraction_worker.py new file mode 100644 index 00000000..809f3bad --- /dev/null +++ b/tests/test_v1_extraction_worker.py @@ -0,0 +1,518 @@ +"""Tests for the chunked extraction worker (#630). + +The model is mocked everywhere: a fake Ollama reads the chunk name out of the +prompt and answers from a canned table, so these cover routing, validation, +retry, failure handling and the deterministic post-steps without a running +Ollama. +""" + +import re +from datetime import datetime, timezone + +import pytest + +from app.api.schemas.enums import ExtractionStatus, InputStatus, InputType +from app.db.repositories import ( + create_extraction, + create_input, + create_job, + get_incident_by_extract, + get_job_by_uuid, +) +from app.models import Extraction, Input, Job +from app.services.extraction import runner as runner_module +from app.services.llm.errors import ( + LLMRateLimitError, + LLMTimeoutError, + LLMUnavailableError, +) +from app.services.extraction.defaults import ExtractionContext, apply_context, resolve_context +from app.services.extraction.prompts import build_prompt, static_prefix +from app.services.extraction.registry import Tier, chunk_registry, extractable_chunks +from app.services.extraction.router import select_chunks +from app.services.extraction.worker import run_extraction + +NARRATIVE = ( + "Structure fire at 42 Oak Street in Reno. Engine 12 dispatched, two civilians " + "injured and transported to hospital. Property loss estimated at 50000. " + "Investigation points to an electrical cause." +) + +# What the fake model answers per chunk. Anything not listed comes back empty, +# which is the honest answer for a chunk the narrative does not support. +ANSWERS = { + "incident": { + "name": "Oak Street structure fire", + "types": [{"category": "fire", "primary": True}], + "alarm_datetime": "2026-04-18T21:14:00-07:00", + "first_arrival_datetime": "2026-04-18T21:19:00-07:00", + "cleared_datetime": "2026-04-18T23:19:00-07:00", + }, + "location": {"address": "42 Oak Street", "city": "Reno", "state": "NV"}, + "casualties": {"total_civilian_injuries": 2}, + "units": [ + { + "unit_id": "E12", + "dispatched_datetime": "2026-04-18T21:14:00-07:00", + "enroute_datetime": "2026-04-18T21:16:00-07:00", + "arrived_datetime": "2026-04-18T21:19:00-07:00", + } + ], + "losses": {"property_loss": {"amount": 50000}}, +} + +_SECTION = re.compile(r"^Section: ([a-z_]+)\.", re.MULTILINE) + + +def chunk_of(prompt: str) -> str: + """The chunk a prompt is asking about.""" + match = _SECTION.search(prompt) + assert match, "every chunk prompt names its section" + return match.group(1) + + +def fake_llm(answers=None, calls=None): + """A stand-in for llm.generate_json that answers from a table.""" + table = ANSWERS if answers is None else answers + + def _call(prompt: str, model: str | None = None, gate=None): + name = chunk_of(prompt) + if calls is not None: + calls.append((name, prompt)) + value = table.get(name, {}) + return {name: value() if callable(value) else value} + + return _call + + +@pytest.fixture +def mock_llm(monkeypatch): + """Patch the model call the runner makes. Yields a setter for the answers.""" + + def _install(answers=None, calls=None, side_effect=None): + target = side_effect or fake_llm(answers, calls) + monkeypatch.setattr(runner_module.llm, "generate_json", target) + return target + + return _install + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def seed(db, transcript: str = NARRATIVE) -> tuple[Extraction, Job]: + now = datetime.now(timezone.utc) + record = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript=transcript, + created_at=now, + updated_at=now, + ), + ) + extraction = create_extraction( + db, + Extraction( + input_id=record.input_id, + status=ExtractionStatus.processing, + started_at=now, + created_at=now, + updated_at=now, + ), + ) + job = create_job(db, Job(celery_task_id="task-1", job_type="extraction", status="queued")) + return extraction, job + + +# --------------------------------------------------------------------------- +# Registry and routing +# --------------------------------------------------------------------------- + +class TestRegistry: + def test_every_chunk_carries_a_tier(self): + registry = chunk_registry() + assert registry, "the contract should yield chunks" + assert all(isinstance(spec.tier, Tier) for spec in registry.values()) + + def test_manual_chunks_are_never_extracted(self): + names = {spec.name for spec in extractable_chunks()} + assert "attachments" not in names + assert "report_metadata" not in names + assert "custom_fields" not in names + + def test_gated_chunks_declare_triggers(self): + gated = [s for s in chunk_registry().values() if s.tier is Tier.gated] + assert gated + assert all(spec.triggers for spec in gated) + + def test_list_chunks_are_flagged(self): + assert chunk_registry()["units"].is_list is True + assert chunk_registry()["incident"].is_list is False + + +class TestRouter: + def test_core_chunks_always_run(self): + selected = {spec.name for spec in select_chunks("nothing much happened")} + assert {"incident", "dispatch", "location", "units"} <= selected + + def test_gated_chunk_runs_only_on_evidence(self): + without = {spec.name for spec in select_chunks("Assisted a resident with a lift.")} + with_evidence = {spec.name for spec in select_chunks("Brush fire burned four hectares.")} + assert "wildland" not in without + assert "wildland" in with_evidence + + def test_core_chunks_come_before_gated(self): + tiers = [spec.tier for spec in select_chunks(NARRATIVE)] + assert tiers == sorted(tiers, key=lambda t: [Tier.core, Tier.gated, Tier.background].index(t)) + + +class TestPrompts: + def test_prefix_is_identical_across_incidents(self): + spec = chunk_registry()["casualties"] + first = build_prompt(spec, "one narrative", ["context"]) + second = build_prompt(spec, "a different narrative", ["context"]) + prefix = static_prefix(spec.name, spec.model, spec.is_list, spec.description) + assert first.startswith(prefix) and second.startswith(prefix) + + def test_narrative_sits_at_the_end(self): + spec = chunk_registry()["incident"] + prompt = build_prompt(spec, NARRATIVE, ["context line"]) + assert prompt.rstrip().endswith(NARRATIVE) + + def test_enum_values_are_spelled_out(self): + spec = chunk_registry()["incident"] + prompt = build_prompt(spec, NARRATIVE, []) + assert "hazardous_conditions" in prompt + + +# --------------------------------------------------------------------------- +# The run +# --------------------------------------------------------------------------- + +class TestHappyPath: + def test_extraction_completes_and_writes_a_draft_incident(self, db, mock_llm): + mock_llm() + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "completed" + assert extraction.status == ExtractionStatus.completed + assert extraction.completed_at is not None + assert extraction.processing_time_seconds is not None + # The incident row owns the document, so the working copy is cleared. + assert extraction.partial_result is None + + incident = get_incident_by_extract(db, extraction.extract_id) + assert incident is not None + contract = incident.incident_contract + assert contract["location"]["city"] == "Reno" + assert contract["incident"]["name"] == "Oak Street structure fire" + assert contract["schema_name"] == "fireform_incident_contract" + + def test_promoted_columns_are_recomputed_from_the_contract(self, db, mock_llm): + mock_llm() + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + incident = get_incident_by_extract(db, extraction.extract_id) + assert incident.city == "Reno" + assert incident.state == "NV" + assert incident.civilian_injuries == 2 + assert incident.incident_category == "fire" + assert incident.total_loss_amount == 50000 + assert incident.call_to_arrival_seconds == 300 + assert incident.on_scene_duration_seconds == 7200 + + def test_job_finishes(self, db, mock_llm): + mock_llm() + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + stored = get_job_by_uuid(db, job.job_id) + assert stored.status == "completed" + assert stored.progress_percent == 100 + assert stored.result_url.endswith(str(extraction.extract_id)) + + def test_only_routed_chunks_are_asked_about(self, db, mock_llm): + calls: list[tuple[str, str]] = [] + mock_llm(calls=calls) + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + asked = {name for name, _ in calls} + assert "casualties" in asked + assert "wildland" not in asked + + def test_metadata_records_the_run(self, db, mock_llm): + mock_llm() + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + contract = get_incident_by_extract(db, extraction.extract_id).incident_contract + metadata = contract["extraction_metadata"] + assert metadata["extract_id"] == str(extraction.extract_id) + assert metadata["llm_model"] + assert 0 <= metadata["completeness"]["overall_percent"] <= 100 + + +class TestRetryAndFailure: + def test_a_rejected_answer_is_retried_once(self, db, mock_llm): + attempts = {"casualties": 0} + + def answers_with_one_miss(): + attempts["casualties"] += 1 + if attempts["casualties"] == 1: + return {"total_civilian_injuries": "two civilians"} + return {"total_civilian_injuries": 2} + + calls: list[tuple[str, str]] = [] + mock_llm({**ANSWERS, "casualties": answers_with_one_miss}, calls=calls) + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + assert attempts["casualties"] == 2 + retry_prompt = [p for name, p in calls if name == "casualties"][1] + assert "previous answer was rejected" in retry_prompt + contract = get_incident_by_extract(db, extraction.extract_id).incident_contract + assert contract["casualties"]["total_civilian_injuries"] == 2 + + def test_a_bad_field_is_dropped_and_the_rest_of_the_chunk_kept(self, db, mock_llm): + # One invented enum in a sub-field used to cost the whole section. + broken = { + **ANSWERS, + "losses": { + "property_loss": {"amount": 50000, "currency": "USD"}, + "estimate_method": "a wild guess", + }, + } + mock_llm(broken) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "completed" + assert "losses" not in result["failed_chunks"] + contract = get_incident_by_extract(db, extraction.extract_id).incident_contract + assert contract["losses"]["property_loss"]["amount"] == 50000 + assert "estimate_method" not in contract["losses"] + assert "losses.estimate_method" in contract["extraction_metadata"]["completeness"]["missing_fields"] + + def test_salvage_keeps_the_right_list_entries(self, db, mock_llm): + # Two bad entries in one list: deleting them must not shift the good one. + units = { + **ANSWERS, + "units": [ + {"unit_id": "E12", "response_mode": "warp_speed"}, + {"unit_id": "E13"}, + {"unit_id": "E14", "response_mode": "teleport"}, + ], + } + mock_llm(units) + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + stored = get_incident_by_extract(db, extraction.extract_id).incident_contract["units"] + assert [unit["unit_id"] for unit in stored] == ["E12", "E13", "E14"] + assert all("response_mode" not in unit for unit in stored) + + def test_a_chunk_with_nothing_salvageable_is_left_empty(self, db, mock_llm): + # Its only field is unusable, so what survives the salvage pass is empty. + mock_llm({**ANSWERS, "casualties": {"total_civilian_injuries": "loads"}}) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "completed" + contract = get_incident_by_extract(db, extraction.extract_id).incident_contract + assert "casualties" not in contract + missing = contract["extraction_metadata"]["completeness"]["missing_fields"] + assert "casualties" in missing + assert "casualties.total_civilian_injuries" in missing + + def test_an_unusable_chunk_shape_is_reported_as_failed(self, db, mock_llm): + # A scalar where the section's object belongs: there is nothing to keep. + def scalar_casualties(prompt, model=None, gate=None): + name = chunk_of(prompt) + return {name: "two people hurt" if name == "casualties" else ANSWERS.get(name, {})} + + mock_llm(side_effect=scalar_casualties) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "completed" + assert "casualties" in result["failed_chunks"] + assert "casualties" not in get_incident_by_extract(db, extraction.extract_id).incident_contract + + def test_a_timed_out_chunk_is_not_retried(self, db, mock_llm): + calls: list[str] = [] + + def slow_casualties(prompt, model=None, gate=None): + name = chunk_of(prompt) + calls.append(name) + if name == "casualties": + raise LLMTimeoutError("the provider did not answer within 300s") + return {name: ANSWERS.get(name, {})} + + mock_llm(side_effect=slow_casualties) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + # One attempt only: the same prompt on the same model takes the same time. + assert calls.count("casualties") == 1 + assert "casualties" in result["failed_chunks"] + assert extraction.status == ExtractionStatus.completed + + def test_run_fails_when_every_chunk_is_rejected(self, db, mock_llm): + def always_broken(prompt, model=None, gate=None): + # A scalar where the chunk's object belongs: rejected every time. + return {chunk_of(prompt): "not an object"} + + mock_llm(side_effect=always_broken) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "failed" + assert extraction.status == ExtractionStatus.failed + assert extraction.error_type == "EXTRACTION_FAILED" + assert get_incident_by_extract(db, extraction.extract_id) is None + assert get_job_by_uuid(db, job.job_id).status == "failed" + + def test_ollama_down_fails_the_run(self, db, mock_llm): + def unreachable(prompt, model=None, gate=None): + raise LLMUnavailableError("could not reach Ollama at http://ollama:11434") + + mock_llm(side_effect=unreachable) + extraction, job = seed(db) + + with pytest.raises(LLMUnavailableError): + run_extraction(db, extraction.extract_id, job.job_id) + + assert extraction.status == ExtractionStatus.failed + assert extraction.error_type == "LLM_UNAVAILABLE" + assert "could not reach Ollama" in extraction.error_detail + assert get_job_by_uuid(db, job.job_id).error["error_code"] == "LLM_UNAVAILABLE" + + def test_a_rate_limited_run_fails_but_keeps_what_it_had(self, db, mock_llm): + """A quota that will not lift stops the run. It does not erase it.""" + later = next(spec.name for spec in select_chunks(NARRATIVE) if spec.tier is not Tier.core) + + def rate_limited(prompt, model=None, gate=None): + name = chunk_of(prompt) + if name == later: + raise LLMRateLimitError("still rate limiting after 11 attempts", 10.0) + return {name: ANSWERS.get(name, {})} + + mock_llm(side_effect=rate_limited) + extraction, job = seed(db) + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "failed" + assert result["retry_after_seconds"] == 10.0 + assert extraction.status == ExtractionStatus.failed + assert extraction.error_type == "LLM_RATE_LIMITED" + assert extraction.partial_result["incident"]["name"] + assert get_job_by_uuid(db, job.job_id).error["error_code"] == "LLM_RATE_LIMITED" + + def test_empty_transcript_fails_before_any_call(self, db, mock_llm): + calls: list[tuple[str, str]] = [] + mock_llm(calls=calls) + extraction, job = seed(db, transcript=" ") + + result = run_extraction(db, extraction.extract_id, job.job_id) + + assert result["status"] == "failed" + assert extraction.error_type == "EMPTY_INPUT" + assert calls == [] + + def test_a_missing_extraction_is_a_no_op(self, db, mock_llm): + from uuid import uuid4 + + mock_llm() + assert run_extraction(db, uuid4(), "job-does-not-exist")["status"] == "missing" + + +# --------------------------------------------------------------------------- +# Deterministic post-steps +# --------------------------------------------------------------------------- + +class TestDeterministicSteps: + def test_request_defaults_win_over_config(self): + context = resolve_context({"country": "IN", "timezone": "Asia/Kolkata", "currency": "INR"}) + assert (context.country, context.timezone, context.currency) == ("IN", "Asia/Kolkata", "INR") + + def test_unknown_timezone_falls_back_to_utc(self): + context = resolve_context({"timezone": "Mars/Olympus"}) + assert str(context.zone) == "UTC" + + def test_defaults_fill_country_currency_and_offset(self): + context = ExtractionContext( + country="IN", + timezone="Asia/Kolkata", + currency="INR", + now=datetime.now(timezone.utc), + ) + filled = apply_context( + { + "location": {"city": "Pune"}, + "losses": {"property_loss": {"amount": 1200}}, + "incident": {"alarm_datetime": "2026-04-18T21:14:00"}, + }, + context, + ) + assert filled["location"]["country"] == "IN" + assert filled["losses"]["property_loss"]["currency"] == "INR" + assert filled["incident"]["alarm_datetime"].endswith("+05:30") + assert filled["incident"]["timezone"] == "Asia/Kolkata" + + def test_blank_answers_are_dropped_but_zero_is_kept(self): + context = ExtractionContext("US", "UTC", "USD", datetime.now(timezone.utc)) + filled = apply_context( + { + "incident": {"name": "", "special_modifiers": [], "chimney_fire": False}, + "casualties": {"total_civilian_injuries": 0}, + "structure": {"floors": {"above_grade": ""}}, + }, + context, + ) + assert "name" not in filled["incident"] + assert "special_modifiers" not in filled["incident"] + assert filled["incident"]["chimney_fire"] is False + assert filled["casualties"]["total_civilian_injuries"] == 0 + assert "structure" not in filled + + def test_currency_already_stated_is_left_alone(self): + context = ExtractionContext("US", "UTC", "USD", datetime.now(timezone.utc)) + filled = apply_context({"losses": {"property_loss": {"amount": 10, "currency": "GBP"}}}, context) + assert filled["losses"]["property_loss"]["currency"] == "GBP" + + def test_unit_turnout_and_travel_are_computed(self, db, mock_llm): + mock_llm() + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + unit = get_incident_by_extract(db, extraction.extract_id).incident_contract["units"][0] + assert unit["turnout_seconds"] == 120 + assert unit["travel_seconds"] == 180 + + def test_computed_timings_beat_a_stated_duration(self, db, mock_llm): + # Models do state a duration that contradicts the times they just gave. + stated = {**ANSWERS, "units": [{**ANSWERS["units"][0], "turnout_seconds": 570}]} + mock_llm(stated) + extraction, job = seed(db) + run_extraction(db, extraction.extract_id, job.job_id) + + unit = get_incident_by_extract(db, extraction.extract_id).incident_contract["units"][0] + assert unit["turnout_seconds"] == 120 + + def test_a_stated_duration_survives_when_there_is_nothing_to_compute(self): + context = ExtractionContext("US", "UTC", "USD", datetime.now(timezone.utc)) + filled = apply_context({"units": [{"unit_id": "E12", "turnout_seconds": 95}]}, context) + assert filled["units"][0]["turnout_seconds"] == 95 diff --git a/tests/test_v1_form_fill_worker.py b/tests/test_v1_form_fill_worker.py new file mode 100644 index 00000000..d73af213 --- /dev/null +++ b/tests/test_v1_form_fill_worker.py @@ -0,0 +1,290 @@ +"""Tests for the batch form-fill worker (app/services/form_fill_worker.py). + +Runs the real ReportLab-draw / pypdf-merge pipeline against the minimal valid +PDF from conftest — no mocking of the drawing itself, only the filesystem +location (redirected to tmp_path). Covers: field resolution onto pdf/json +output, the field_mapping_summary shape, status transitions, and per-form +failure isolation (one bad form doesn't sink the batch or the job). +""" + +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest +from pypdf import PdfReader + +from app.api.schemas.enums import ( + ExtractionStatus, + FormStatus, + InputStatus, + InputType, + JobStatus, + ReportStatus, +) +from app.db.repositories import ( + create_extraction, + create_form_template, + create_generated_form, + create_incident, + create_input, + create_job, + get_job_by_uuid, +) +from app.models import Extraction, Form, FormTemplate, Incident, Input, Job +from app.services.form_fill_worker import fill_one, run_batch_fill + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": {"name": "Bear Creek Wildfire"}, + "location": {"city": "Reno", "state": "NV"}, + "custom_fields": {"neris.marshal_signature_name": "A. Ruiz"}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _layout(page=0, x=50, y=700, width=200, height=20, **extra) -> dict: + return {"page": page, "x": x, "y": y, "width": width, "height": height, **extra} + + +def _field(name, source="schema", required=True, layout=None, **extra) -> dict: + field = { + "field_name": name, + "field_type": "string", + "source": source, + "required": required, + "layout": layout, + } + if source == "schema": + field.setdefault("incident_mapping", "incident.name") + if source == "static": + field.setdefault("static_text", "Reno Fire Department") + if source == "manual": + pass + field.update(extra) + return field + + +def _incident(db, contract=None) -> Incident: + now = datetime.now(timezone.utc) + inp = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript="Wildfire off Bear Creek, no injuries.", + created_at=now, + updated_at=now, + ), + ) + extraction = create_extraction( + db, + Extraction(input_id=inp.input_id, status=ExtractionStatus.completed, started_at=now, completed_at=now), + ) + return create_incident( + db, + Incident( + extract_id=extraction.extract_id, + status=ReportStatus.draft, + incident_contract=_CONTRACT if contract is None else contract, + ), + ) + + +def _template(db, form_type="neris", fields=None, pdf_template_ref=None) -> FormTemplate: + return create_form_template( + db, + FormTemplate( + form_type=form_type, + display_name=form_type.upper(), + fields=fields if fields is not None else [_field("incident_name", layout=_layout())], + pdf_template_ref=pdf_template_ref, + ), + ) + + +def _form(db, incident, template, batch_id=None, **kwargs) -> Form: + defaults = dict( + template_id=template.template_id, + incident_id=incident.incident_id, + batch_id=batch_id, + form_type=template.form_type, + status=FormStatus.queued, + ) + return create_generated_form(db, Form(**{**defaults, **kwargs})) + + +@pytest.fixture +def output_dir(tmp_path, monkeypatch): + """Redirect the template-source lookup and the fill output to tmp_path, + same pattern test_templates_pdf.py uses for the upload flow.""" + monkeypatch.setattr("app.services.form_templates.DATA_DIR", tmp_path) + monkeypatch.setattr("app.services.form_fill_worker.DATA_DIR", tmp_path) + generated = tmp_path / "forms" / "generated" + monkeypatch.setattr("app.services.form_fill_worker.FORMS_OUTPUT_DIR", generated) + return tmp_path + + +def _seed_template_pdf(tmp_path, pdf_bytes, name="template.pdf") -> str: + """Write the source PDF under tmp_path and return its DATA_DIR-relative ref.""" + path = tmp_path / "templates" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(pdf_bytes) + return str(path.relative_to(tmp_path)) + + +# --------------------------------------------------------------------------- +# fill_one +# --------------------------------------------------------------------------- + +class TestFillOne: + + def test_fills_schema_static_and_manual_fields(self, db, output_dir, pdf_bytes): + ref = _seed_template_pdf(output_dir, pdf_bytes) + incident = _incident(db) + template = _template( + db, + fields=[ + _field("incident_name", source="schema", incident_mapping="incident.name", layout=_layout(y=700)), + _field("agency", source="static", layout=_layout(y=650)), + _field("marshal_signature_name", source="manual", required=False, layout=None), + ], + pdf_template_ref=ref, + ) + form = _form(db, incident, template) + + fill_one(db, form) + + assert form.status == FormStatus.completed + assert form.pdf_ready is True + assert form.json_ready is True + assert form.completed_at is not None + + # pdf_path is DATA_DIR-relative and the file is a real, readable PDF. + assert not form.pdf_path.startswith("/") + written = output_dir / form.pdf_path + assert written.is_file() + reader = PdfReader(str(written)) + assert len(reader.pages) == 1 + + # agency_fields covers every field, placed or not. + assert form.json_data["incident_name"] == "Bear Creek Wildfire" + assert form.json_data["agency"] == "Reno Fire Department" + assert form.json_data["marshal_signature_name"] == "A. Ruiz" + + summary = form.field_mapping_summary + assert summary["total_form_fields"] == 3 + assert summary["fields_filled"] == 3 + assert summary["fields_blank"] == 0 + assert summary["coverage_percent"] == 100.0 + + def test_unplaced_field_has_no_layout_but_is_still_in_json(self, db, output_dir, pdf_bytes): + ref = _seed_template_pdf(output_dir, pdf_bytes) + incident = _incident(db) + template = _template( + db, + fields=[_field("incident_name", layout=None)], + pdf_template_ref=ref, + ) + form = _form(db, incident, template) + + fill_one(db, form) + + assert form.status == FormStatus.completed + assert form.json_data["incident_name"] == "Bear Creek Wildfire" + + def test_missing_required_field_still_fills_a_blank_box(self, db, output_dir, pdf_bytes): + """Filling doesn't gate on readiness — that's the generate-time skip check.""" + ref = _seed_template_pdf(output_dir, pdf_bytes) + incident = _incident(db, contract={"schema_version": "1.1.0", "schema_name": "fireform_incident_contract"}) + template = _template( + db, + fields=[_field("incident_name", source="schema", incident_mapping="incident.name", layout=_layout())], + pdf_template_ref=ref, + ) + form = _form(db, incident, template) + + fill_one(db, form) + + assert form.status == FormStatus.completed + assert form.json_data["incident_name"] is None + assert form.field_mapping_summary["fields_blank"] == 1 + assert form.field_mapping_summary["coverage_percent"] == 0.0 + + def test_missing_template_pdf_raises(self, db, output_dir): + incident = _incident(db) + template = _template( + db, + fields=[_field("incident_name", layout=_layout())], + pdf_template_ref="templates/does-not-exist.pdf", + ) + form = _form(db, incident, template) + + with pytest.raises(Exception): + fill_one(db, form) + + +# --------------------------------------------------------------------------- +# run_batch_fill — batch orchestration and per-form failure isolation +# --------------------------------------------------------------------------- + +class TestRunBatchFill: + + def _job(self, db) -> Job: + return create_job(db, Job(celery_task_id="task-1", job_type="batch_form_generation", status="queued")) + + def test_all_forms_complete_job_completed(self, db, output_dir, pdf_bytes): + ref = _seed_template_pdf(output_dir, pdf_bytes) + incident = _incident(db) + template = _template(db, fields=[_field("incident_name", layout=_layout())], pdf_template_ref=ref) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id) + _form(db, incident, template, batch_id=batch_id) + job = self._job(db) + + result = run_batch_fill(db, batch_id, job.job_id) + + assert result["completed"] == 2 + assert result["failed"] == 0 + refreshed = get_job_by_uuid(db, job.job_id) + assert refreshed.status == JobStatus.completed + assert refreshed.progress_percent == 100 + + def test_one_bad_form_does_not_sink_the_batch(self, db, output_dir, pdf_bytes): + """One form's template PDF is missing; the other form in the same + batch still completes, and the Job still finishes as completed.""" + good_ref = _seed_template_pdf(output_dir, pdf_bytes, name="good.pdf") + good_template = _template(db, fields=[_field("incident_name", layout=_layout())], pdf_template_ref=good_ref) + bad_template = _template( + db, + form_type="cal_fire_ics209", + fields=[_field("incident_name", layout=_layout())], + pdf_template_ref="templates/missing.pdf", + ) + incident = _incident(db) + batch_id = uuid4() + good_form = _form(db, incident, good_template, batch_id=batch_id) + bad_form = _form(db, incident, bad_template, batch_id=batch_id) + job = self._job(db) + + result = run_batch_fill(db, batch_id, job.job_id) + + assert result["completed"] == 1 + assert result["failed"] == 1 + + from app.db.repositories import get_form + assert get_form(db, good_form.form_id).status == FormStatus.completed + assert get_form(db, bad_form.form_id).status == FormStatus.failed + + refreshed_job = get_job_by_uuid(db, job.job_id) + assert refreshed_job.status == JobStatus.completed + assert refreshed_job.progress_percent == 100 + + def test_empty_batch_completes_cleanly(self, db, output_dir): + job = self._job(db) + result = run_batch_fill(db, uuid4(), job.job_id) + assert result == {"batch_id": result["batch_id"], "completed": 0, "failed": 0} + assert get_job_by_uuid(db, job.job_id).status == JobStatus.completed diff --git a/tests/test_v1_form_generation.py b/tests/test_v1_form_generation.py new file mode 100644 index 00000000..ef2e5626 --- /dev/null +++ b/tests/test_v1_form_generation.py @@ -0,0 +1,784 @@ +"""Tests for POST /forms/generate, GET /forms/batch/{batch_id}, GET /forms/{form_id}, +GET /forms/{form_id}/pdf and GET /forms/{form_id}/json. + +Dispatch is mocked (no broker) — the actual fill is covered by +tests/test_v1_form_fill_worker.py. These cover the write path (queued vs +skipped split, 404s, the NO_FORMS_TO_GENERATE case) and the read endpoints +against Form rows seeded directly. +""" + +from datetime import datetime, timezone +from io import BytesIO +from unittest.mock import MagicMock, patch +from uuid import UUID, uuid4 +from zipfile import ZipFile + +from app.api.schemas.enums import ( + ExtractionStatus, + FormStatus, + InputStatus, + InputType, + ReportStatus, + TemplateStatus, +) +from app.core.config import API_PREFIX +from app.db.repositories import ( + create_extraction, + create_form_template, + create_generated_form, + create_incident, + create_input, +) +from app.models import Extraction, Form, FormTemplate, Incident, Input + +FORMS_URL = f"{API_PREFIX}/forms" + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": {"name": "Bear Creek Wildfire"}, + "location": {"city": "Reno", "state": "NV"}, + "custom_fields": {"neris.marshal_signature_name": "A. Ruiz"}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _field(name, source="schema", required=True, layout=None, **extra) -> dict: + field = { + "field_name": name, + "field_type": "string", + "source": source, + "required": required, + "layout": layout, + } + if source == "schema": + field.setdefault("incident_mapping", "incident.name") + if source == "static": + field.setdefault("static_text", "Reno Fire Department") + if source == "open": + field.setdefault("description", "Anything the narrative says about it") + field.update(extra) + return field + + +def _incident(db, contract=None, incident_number=None) -> Incident: + now = datetime.now(timezone.utc) + inp = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript="Wildfire off Bear Creek, no injuries.", + created_at=now, + updated_at=now, + ), + ) + extraction = create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=ExtractionStatus.completed, + started_at=now, + completed_at=now, + ), + ) + return create_incident( + db, + Incident( + extract_id=extraction.extract_id, + status=ReportStatus.draft, + incident_contract=_CONTRACT if contract is None else contract, + incident_number=incident_number, + ), + ) + + +def _template(db, form_type="neris", fields=None, status=TemplateStatus.active) -> FormTemplate: + return create_form_template( + db, + FormTemplate( + form_type=form_type, + display_name=form_type.upper(), + status=status, + fields=fields if fields is not None else [_field("incident_name")], + ), + ) + + +def _form(db, incident, template, batch_id=None, **kwargs) -> Form: + defaults = dict( + template_id=template.template_id, + incident_id=incident.incident_id, + batch_id=batch_id, + form_type=template.form_type, + status=FormStatus.queued, + ) + return create_generated_form(db, Form(**{**defaults, **kwargs})) + + +class NoCelery: + """Stand-in for the fill task so nothing is dispatched to a broker.""" + + def __enter__(self): + self._patch = patch("app.services.form_generation.generate_forms_batch_task") + self.task = self._patch.__enter__() + self.task.delay.return_value = MagicMock(id="celery-batch-1") + return self.task + + def __exit__(self, *exc): + self._patch.__exit__(*exc) + + +# --------------------------------------------------------------------------- +# POST /forms/generate +# --------------------------------------------------------------------------- + +class TestGenerateForms: + + def test_202_all_ready_queues_every_template(self, client, db): + incident = _incident(db) + template = _template(db, fields=[_field("incident_name")]) + + with NoCelery() as task: + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(incident.incident_id), "template_ids": [str(template.template_id)]}, + ) + + assert resp.status_code == 202 + body = resp.json() + assert body["status"] == "processing" + assert body["incident_id"] == str(incident.incident_id) + assert len(body["forms_queued"]) == 1 + assert body["forms_queued"][0]["template_id"] == str(template.template_id) + assert body["forms_queued"][0]["form_type"] == "neris" + assert body["forms_skipped"] == [] + assert body["poll_url"] == f"/api/v1/forms/batch/{body['batch_id']}" + assert body["estimated_seconds"] == 10 + task.delay.assert_called_once() + dispatched_batch_id, dispatched_job_id = task.delay.call_args[0] + assert dispatched_batch_id == body["batch_id"] + assert isinstance(dispatched_job_id, str) and dispatched_job_id + + def test_not_ready_template_is_skipped_with_reason(self, client, db): + # A not-ready template alone would 422 NO_FORMS_TO_GENERATE (covered + # separately below) — pair it with a ready one so the skip path is + # exercised inside a batch that still succeeds. + incident = _incident(db) + ready = _template(db, form_type="neris", fields=[_field("incident_name")]) + not_ready = _template( + db, + form_type="state_texas", + fields=[_field("marshal_signature_name", source="manual", required=True)], + ) + + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={ + "incident_id": str(incident.incident_id), + "template_ids": [str(ready.template_id), str(not_ready.template_id)], + }, + ) + + assert resp.status_code == 202 + body = resp.json() + assert len(body["forms_queued"]) == 1 + assert len(body["forms_skipped"]) == 1 + skipped = body["forms_skipped"][0] + assert skipped["template_id"] == str(not_ready.template_id) + assert skipped["reason"] == "Not ready: marshal_signature_name (manual) has no value" + + def test_force_partial_queues_a_not_ready_template(self, client, db): + incident = _incident(db, contract={"schema_version": "1.1.0", "schema_name": "fireform_incident_contract"}) + template = _template( + db, + form_type="state_texas", + fields=[_field("marshal_signature_name", source="manual", required=True)], + ) + + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={ + "incident_id": str(incident.incident_id), + "template_ids": [str(template.template_id)], + "options": {"force_partial": True}, + }, + ) + + assert resp.status_code == 202 + body = resp.json() + assert body["forms_skipped"] == [] + assert len(body["forms_queued"]) == 1 + + def test_mixed_batch_splits_queued_and_skipped(self, client, db): + incident = _incident(db) + ready = _template(db, form_type="neris", fields=[_field("incident_name")]) + not_ready = _template( + db, + form_type="cal_fire_ics209", + fields=[_field("something_missing", source="schema", incident_mapping="does.not.exist")], + ) + + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={ + "incident_id": str(incident.incident_id), + "template_ids": [str(ready.template_id), str(not_ready.template_id)], + }, + ) + + body = resp.json() + assert len(body["forms_queued"]) == 1 + assert len(body["forms_skipped"]) == 1 + assert body["forms_queued"][0]["template_id"] == str(ready.template_id) + assert body["forms_skipped"][0]["template_id"] == str(not_ready.template_id) + + def test_creates_queued_form_rows_in_db(self, client, db, test_engine): + from sqlmodel import Session, select + + incident = _incident(db) + template = _template(db) + + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(incident.incident_id), "template_ids": [str(template.template_id)]}, + ) + batch_id = UUID(resp.json()["batch_id"]) + + with Session(test_engine) as session: + rows = list(session.exec(select(Form).where(Form.batch_id == batch_id))) + assert len(rows) == 1 + assert rows[0].status == FormStatus.queued + assert rows[0].incident_id == incident.incident_id + assert rows[0].template_id == template.template_id + + def test_404_incident_not_found(self, client, db): + template = _template(db) + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(uuid4()), "template_ids": [str(template.template_id)]}, + ) + assert resp.status_code == 404 + assert resp.json()["error_code"] == "INCIDENT_NOT_FOUND" + + def test_404_template_not_found(self, client, db): + incident = _incident(db) + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(incident.incident_id), "template_ids": [str(uuid4())]}, + ) + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_NOT_FOUND" + + def test_404_on_bad_template_leaves_no_partial_batch(self, client, db, test_engine): + """A bad template_id anywhere in the list 404s before any Form row is written.""" + from sqlmodel import Session, select + + incident = _incident(db) + good = _template(db) + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={ + "incident_id": str(incident.incident_id), + "template_ids": [str(good.template_id), str(uuid4())], + }, + ) + assert resp.status_code == 404 + with Session(test_engine) as session: + rows = list(session.exec(select(Form))) + assert rows == [] + + def test_422_empty_template_ids_rejected(self, client, db): + incident = _incident(db) + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(incident.incident_id), "template_ids": []}, + ) + assert resp.status_code == 422 + + def test_422_no_forms_to_generate_when_all_skipped(self, client, db): + incident = _incident(db, contract={"schema_version": "1.1.0", "schema_name": "fireform_incident_contract"}) + template = _template( + db, + form_type="state_texas", + fields=[_field("marshal_signature_name", source="manual", required=True)], + ) + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={"incident_id": str(incident.incident_id), "template_ids": [str(template.template_id)]}, + ) + assert resp.status_code == 422 + assert resp.json()["error_code"] == "NO_FORMS_TO_GENERATE" + assert resp.json()["message"] == "None of the selected templates are ready" + assert resp.json()["detail"]["skipped"] + + +# --------------------------------------------------------------------------- +# POST /forms/generate with template_ids omitted +# --------------------------------------------------------------------------- + +class TestGenerateEverythingReady: + + def _generate_all(self, client, incident, **options): + body = {"incident_id": str(incident.incident_id)} + if options: + body["options"] = options + with NoCelery(): + return client.post(f"{FORMS_URL}/generate", json=body) + + def test_202_queues_every_ready_active_template(self, client, db): + incident = _incident(db) + _template(db, form_type="neris") + _template(db, form_type="nfirs_basic") + + resp = self._generate_all(client, incident) + + assert resp.status_code == 202 + queued = {f["form_type"] for f in resp.json()["forms_queued"]} + assert queued == {"neris", "nfirs_basic"} + + def test_legacy_and_draft_templates_are_not_candidates(self, client, db): + incident = _incident(db) + _template(db, form_type="neris") + _template(db, form_type="old_county", status=TemplateStatus.legacy) + _template(db, form_type="half_built", status=TemplateStatus.draft) + + resp = self._generate_all(client, incident) + + body = resp.json() + assert [f["form_type"] for f in body["forms_queued"]] == ["neris"] + assert body["forms_skipped"] == [] + + def test_a_legacy_template_still_generates_when_asked_for_by_id(self, client, db): + incident = _incident(db) + template = _template(db, form_type="old_county", status=TemplateStatus.legacy) + + with NoCelery(): + resp = client.post( + f"{FORMS_URL}/generate", + json={ + "incident_id": str(incident.incident_id), + "template_ids": [str(template.template_id)], + }, + ) + + assert resp.status_code == 202 + assert len(resp.json()["forms_queued"]) == 1 + + def test_not_ready_template_is_skipped_with_a_reason(self, client, db): + incident = _incident(db) + _template(db, form_type="neris") + _template( + db, + form_type="state_texas", + fields=[_field("marshal_signature_name", source="manual", required=True)], + ) + + body = self._generate_all(client, incident).json() + + assert [f["form_type"] for f in body["forms_queued"]] == ["neris"] + assert len(body["forms_skipped"]) == 1 + assert body["forms_skipped"][0]["form_type"] == "state_texas" + assert "marshal_signature_name" in body["forms_skipped"][0]["reason"] + + def test_force_partial_queues_the_not_ready_ones_too(self, client, db): + incident = _incident(db) + _template(db, form_type="neris") + _template( + db, + form_type="state_texas", + fields=[_field("marshal_signature_name", source="manual", required=True)], + ) + + body = self._generate_all(client, incident, force_partial=True).json() + + assert {f["form_type"] for f in body["forms_queued"]} == {"neris", "state_texas"} + assert body["forms_skipped"] == [] + + def test_422_when_the_registry_has_nothing_ready(self, client, db): + incident = _incident(db) + _template(db, form_type="old_county", status=TemplateStatus.legacy) + + resp = self._generate_all(client, incident) + + assert resp.status_code == 422 + assert resp.json()["error_code"] == "NO_FORMS_TO_GENERATE" + assert resp.json()["message"] == "No templates were selected and none are ready" + + +# --------------------------------------------------------------------------- +# GET /forms/batch/{batch_id} +# --------------------------------------------------------------------------- + +class TestBatchStatus: + + def test_processing_when_some_forms_still_queued(self, client, db): + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.queued) + _form(db, incident, template, batch_id=batch_id, status=FormStatus.completed) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "processing" + assert body["total"] == 2 + assert body["completed"] == 1 + assert body["failed"] == 0 + assert body["download_url"] is None + + def test_completed_when_all_terminal_with_no_failures(self, client, db): + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.completed) + _form(db, incident, template, batch_id=batch_id, status=FormStatus.completed) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}") + assert resp.json()["status"] == "completed" + assert resp.json()["download_url"] == f"/api/v1/forms/batch/{batch_id}/download" + + def test_completed_when_terminal_with_a_partial_failure(self, client, db): + """One failed form doesn't fail the batch — matches the per-form isolation design.""" + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.completed) + _form(db, incident, template, batch_id=batch_id, status=FormStatus.failed) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}") + body = resp.json() + assert body["status"] == "completed" + assert body["completed"] == 1 + assert body["failed"] == 1 + + def test_failed_when_every_form_failed(self, client, db): + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.failed) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}") + assert resp.json()["status"] == "failed" + assert resp.json()["download_url"] is None + + def test_404_unknown_batch(self, client, db): + resp = client.get(f"{FORMS_URL}/batch/{uuid4()}") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "BATCH_NOT_FOUND" + + +# --------------------------------------------------------------------------- +# GET /forms/batch/{batch_id}/download +# --------------------------------------------------------------------------- + +class TestBatchDownload: + + def _pdf_form(self, db, tmp_path, pdf_bytes, incident, template, batch_id, name, **kwargs): + pdf_file = tmp_path / "forms" / "generated" / name + pdf_file.parent.mkdir(parents=True, exist_ok=True) + pdf_file.write_bytes(pdf_bytes) + return _form( + db, incident, template, + batch_id=batch_id, + status=FormStatus.completed, + pdf_ready=True, + pdf_path=f"forms/generated/{name}", + **kwargs, + ) + + def test_202_while_the_batch_is_still_generating(self, client, db): + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.queued) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + assert resp.status_code == 202 + assert resp.json()["status"] == "processing" + assert resp.json()["retry_after_seconds"] == 5 + + def test_500_when_every_form_failed(self, client, db): + incident = _incident(db) + template = _template(db) + batch_id = uuid4() + _form(db, incident, template, batch_id=batch_id, status=FormStatus.failed) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + assert resp.status_code == 500 + assert resp.json()["error_code"] == "FORM_GENERATION_FAILED" + + def test_404_unknown_batch(self, client, db): + resp = client.get(f"{FORMS_URL}/batch/{uuid4()}/download") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "BATCH_NOT_FOUND" + + def test_200_bundles_every_pdf_under_its_download_name( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db, incident_number="FF-2024-CA-0157") + batch_id = uuid4() + self._pdf_form( + db, tmp_path, pdf_bytes, incident, _template(db, form_type="neris"), batch_id, "a.pdf" + ) + self._pdf_form( + db, tmp_path, pdf_bytes, incident, _template(db, form_type="nfirs"), batch_id, "b.pdf" + ) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/zip" + assert 'filename="fireform_batch_FF-2024-CA-0157.zip"' in resp.headers[ + "content-disposition" + ] + with ZipFile(BytesIO(resp.content)) as archive: + assert sorted(archive.namelist()) == [ + "neris_FF-2024-CA-0157.pdf", + "nfirs_FF-2024-CA-0157.pdf", + ] + assert archive.read("neris_FF-2024-CA-0157.pdf") == pdf_bytes + + def test_a_failed_form_is_left_out_of_the_zip( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db, incident_number="FF-2024-CA-0157") + batch_id = uuid4() + self._pdf_form( + db, tmp_path, pdf_bytes, incident, _template(db, form_type="neris"), batch_id, "a.pdf" + ) + _form( + db, incident, _template(db, form_type="nfirs"), + batch_id=batch_id, status=FormStatus.failed, + ) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + + assert resp.status_code == 200 + with ZipFile(BytesIO(resp.content)) as archive: + assert archive.namelist() == ["neris_FF-2024-CA-0157.pdf"] + + def test_a_pdf_missing_off_disk_is_left_out_rather_than_failing( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db, incident_number="FF-2024-CA-0157") + batch_id = uuid4() + self._pdf_form( + db, tmp_path, pdf_bytes, incident, _template(db, form_type="neris"), batch_id, "a.pdf" + ) + _form( + db, incident, _template(db, form_type="nfirs"), + batch_id=batch_id, + status=FormStatus.completed, + pdf_ready=True, + pdf_path="forms/generated/gone.pdf", + ) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + + assert resp.status_code == 200 + with ZipFile(BytesIO(resp.content)) as archive: + assert archive.namelist() == ["neris_FF-2024-CA-0157.pdf"] + + def test_zip_name_falls_back_to_the_batch_id_without_an_incident_number( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db) + batch_id = uuid4() + form = self._pdf_form( + db, tmp_path, pdf_bytes, incident, _template(db), batch_id, "a.pdf" + ) + + resp = client.get(f"{FORMS_URL}/batch/{batch_id}/download") + + assert f'filename="fireform_batch_{batch_id}.zip"' in resp.headers["content-disposition"] + with ZipFile(BytesIO(resp.content)) as archive: + assert archive.namelist() == [f"{form.form_id}.pdf"] + + +# --------------------------------------------------------------------------- +# GET /forms/{form_id} +# --------------------------------------------------------------------------- + +class TestGetForm: + + def test_200_returns_form_record(self, client, db): + incident = _incident(db) + template = _template(db) + summary = { + "total_form_fields": 10, + "fields_filled": 8, + "fields_blank": 2, + "coverage_percent": 80.0, + } + form = _form( + db, incident, template, + status=FormStatus.completed, + pdf_ready=True, + json_ready=True, + field_mapping_summary=summary, + ) + + resp = client.get(f"{FORMS_URL}/{form.form_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["form_id"] == str(form.form_id) + assert body["template_id"] == str(template.template_id) + assert body["form_type"] == "neris" + assert body["status"] == "completed" + assert body["incident_id"] == str(incident.incident_id) + assert body["pdf_ready"] is True + assert body["json_ready"] is True + assert body["field_mapping_summary"]["coverage_percent"] == 80.0 + + def test_404_unknown_form(self, client, db): + resp = client.get(f"{FORMS_URL}/{uuid4()}") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "FORM_NOT_FOUND" + + +# --------------------------------------------------------------------------- +# GET /forms/{form_id}/pdf +# --------------------------------------------------------------------------- + +class TestGetFormPdf: + + def test_202_while_still_generating(self, client, db): + incident = _incident(db) + template = _template(db) + form = _form(db, incident, template, status=FormStatus.generating) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert resp.status_code == 202 + assert resp.json()["status"] == "generating" + + def test_500_when_form_failed(self, client, db): + incident = _incident(db) + template = _template(db) + form = _form(db, incident, template, status=FormStatus.failed) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert resp.status_code == 500 + assert resp.json()["error_code"] == "PDF_GENERATION_FAILED" + + def _completed_pdf_form(self, db, tmp_path, pdf_bytes, incident, template): + pdf_file = tmp_path / "forms" / "generated" / "x.pdf" + pdf_file.parent.mkdir(parents=True, exist_ok=True) + pdf_file.write_bytes(pdf_bytes) + return _form( + db, incident, template, + status=FormStatus.completed, + pdf_ready=True, + pdf_path="forms/generated/x.pdf", + ) + + def test_200_serves_the_pdf_file(self, client, db, monkeypatch, tmp_path, pdf_bytes): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db, incident_number="FF-2024-CA-0157") + template = _template(db) + form = self._completed_pdf_form(db, tmp_path, pdf_bytes, incident, template) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/pdf" + assert resp.content == pdf_bytes + assert 'filename="neris_FF-2024-CA-0157.pdf"' in resp.headers["content-disposition"] + + def test_filename_falls_back_to_form_id_without_an_incident_number( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db) + template = _template(db) + form = self._completed_pdf_form(db, tmp_path, pdf_bytes, incident, template) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert f'filename="{form.form_id}.pdf"' in resp.headers["content-disposition"] + + def test_filename_strips_characters_an_incident_number_should_not_carry( + self, client, db, monkeypatch, tmp_path, pdf_bytes + ): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db, incident_number='../2024 "07"/0157') + template = _template(db) + form = self._completed_pdf_form(db, tmp_path, pdf_bytes, incident, template) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert 'filename="neris_2024-07-0157.pdf"' in resp.headers["content-disposition"] + + def test_404_path_escaping_data_dir_is_rejected(self, client, db, monkeypatch, tmp_path): + monkeypatch.setattr("app.services.form_generation.DATA_DIR", tmp_path) + incident = _incident(db) + template = _template(db) + form = _form( + db, incident, template, + status=FormStatus.completed, + pdf_ready=True, + pdf_path="../../etc/passwd", + ) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/pdf") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# GET /forms/{form_id}/json +# --------------------------------------------------------------------------- + +class TestGetFormJson: + + def test_200_returns_agency_fields(self, client, db): + incident = _incident(db) + template = _template(db) + form = _form( + db, incident, template, + status=FormStatus.completed, + json_ready=True, + json_data={"incident_name": "Bear Creek Wildfire"}, + ) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/json") + assert resp.status_code == 200 + body = resp.json() + assert body["form_id"] == str(form.form_id) + assert body["agency_fields"]["incident_name"] == "Bear Creek Wildfire" + assert body["form_version"] == template.version + + def test_202_while_still_generating(self, client, db): + incident = _incident(db) + template = _template(db) + form = _form(db, incident, template, status=FormStatus.queued) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/json") + assert resp.status_code == 202 + assert resp.json()["status"] == "queued" + assert resp.json()["retry_after_seconds"] == 5 + + def test_500_when_form_failed(self, client, db): + incident = _incident(db) + template = _template(db) + form = _form(db, incident, template, status=FormStatus.failed) + + resp = client.get(f"{FORMS_URL}/{form.form_id}/json") + assert resp.status_code == 500 + assert resp.json()["error_code"] == "FORM_GENERATION_FAILED" + + def test_404_unknown_form(self, client, db): + resp = client.get(f"{FORMS_URL}/{uuid4()}/json") + assert resp.status_code == 404 diff --git a/tests/test_v1_incidents.py b/tests/test_v1_incidents.py new file mode 100644 index 00000000..73c7207f --- /dev/null +++ b/tests/test_v1_incidents.py @@ -0,0 +1,510 @@ +"""Tests for the five incident endpoints (contracts/path/incidents.yaml). + +Rows are seeded directly rather than driven through extraction, so these cover +the CRUD surface itself: finalizing the draft, list filtering/paging/sorting, +the full record shape, metadata updates, and soft delete. + +The submitted-status lock is deliberately not built yet, so the tests here +assert the current behaviour: status moves freely and a submitted incident is +still editable. +""" + +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +from app.api.schemas.enums import ( + ExtractionStatus, + FormStatus, + IncidentCategory, + InputStatus, + InputType, + ReportStatus, +) +from app.core.config import API_PREFIX +from app.db.repositories import ( + create_extraction, + create_form_template, + create_generated_form, + create_incident, + create_input, +) +from app.models import Extraction, Form, FormTemplate, Incident, Input + +INCIDENTS_URL = f"{API_PREFIX}/incidents" + +_CONTRACT = { + "schema_version": "1.1.0", + "schema_name": "fireform_incident_contract", + "incident": { + "name": "Bear Creek Wildfire", + "alarm_datetime": "2024-07-10T13:52:00-07:00", + "types": [{"primary": True, "category": "fire", "subcategory": "wildland_fire"}], + }, + "location": {"city": "Reno", "state": "NV", "country": "US"}, + "casualties": {"total_civilian_injuries": 2}, +} + + +# --------------------------------------------------------------------------- +# Seed helpers +# --------------------------------------------------------------------------- + +def _extraction(db, status=ExtractionStatus.completed) -> Extraction: + now = datetime.now(timezone.utc) + inp = create_input( + db, + Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript="Wildfire off Bear Creek, two injured.", + created_at=now, + updated_at=now, + ), + ) + return create_extraction( + db, + Extraction( + input_id=inp.input_id, + status=status, + started_at=now, + completed_at=now if status == ExtractionStatus.completed else None, + ), + ) + + +def _incident(db, extraction=None, **kwargs) -> Incident: + """A draft incident with the promoted columns already filled, as the + extraction worker would leave it.""" + extraction = extraction or _extraction(db) + defaults = dict( + extract_id=extraction.extract_id, + status=ReportStatus.draft, + incident_contract=_CONTRACT, + incident_name="Bear Creek Wildfire", + incident_type="wildland_fire", + incident_category=IncidentCategory.fire, + incident_datetime=datetime(2024, 7, 10, 13, 52), + city="Reno", + state="NV", + country="US", + civilian_injuries=2, + ) + defaults.update(kwargs) + return create_incident(db, Incident(**defaults)) + + +def _form(db, incident, form_type="neris") -> Form: + template = create_form_template( + db, + FormTemplate( + form_type=f"{form_type}-{uuid4().hex[:6]}", + display_name=form_type.upper(), + fields=[], + ), + ) + return create_generated_form( + db, + Form( + form_type=form_type, + status=FormStatus.completed, + template_id=template.template_id, + incident_id=incident.incident_id, + ), + ) + + +# --------------------------------------------------------------------------- +# POST /incidents +# --------------------------------------------------------------------------- + +class TestCreateIncident: + def test_finalizes_the_existing_draft(self, client, db): + incident = _incident(db) + body = { + "extract_id": str(incident.extract_id), + "incident_number": "CA-SQF-2024-0421", + "tags": ["wildland", "mutual_aid"], + } + response = client.post(INCIDENTS_URL, json=body) + + assert response.status_code == 201 + payload = response.json() + # The same row, not a second one. + assert payload["incident_id"] == str(incident.incident_id) + assert payload["incident_number"] == "CA-SQF-2024-0421" + assert payload["tags"] == ["wildland", "mutual_aid"] + assert payload["status"] == "draft" + + def test_promoted_fields_are_returned(self, client, db): + incident = _incident(db) + payload = client.post( + INCIDENTS_URL, json={"extract_id": str(incident.extract_id)} + ).json() + + assert payload["incident_name"] == "Bear Creek Wildfire" + assert payload["incident_type"] == "wildland_fire" + assert payload["incident_category"] == "fire" + assert payload["analytics"]["city"] == "Reno" + assert payload["analytics"]["civilian_injuries"] == 2 + + def test_is_idempotent(self, client, db): + incident = _incident(db) + body = {"extract_id": str(incident.extract_id), "incident_number": "CA-1"} + + first = client.post(INCIDENTS_URL, json=body) + second = client.post(INCIDENTS_URL, json=body) + + assert first.status_code == 201 + assert second.status_code == 201 + assert first.json()["incident_id"] == second.json()["incident_id"] + + def test_omitted_fields_leave_the_draft_alone(self, client, db): + incident = _incident(db, incident_number="CA-1", tags=["existing"]) + payload = client.post( + INCIDENTS_URL, json={"extract_id": str(incident.extract_id)} + ).json() + + assert payload["incident_number"] == "CA-1" + assert payload["tags"] == ["existing"] + + def test_unknown_extract_id_is_404(self, client, db): + response = client.post(INCIDENTS_URL, json={"extract_id": str(uuid4())}) + + assert response.status_code == 404 + assert response.json()["error_code"] == "EXTRACT_NOT_FOUND" + + def test_extraction_without_a_draft_is_409(self, client, db): + extraction = _extraction(db, status=ExtractionStatus.processing) + response = client.post( + INCIDENTS_URL, json={"extract_id": str(extraction.extract_id)} + ) + + assert response.status_code == 409 + assert response.json()["error_code"] == "EXTRACTION_NOT_COMPLETED" + + def test_duplicate_incident_number_is_409(self, client, db): + _incident(db, incident_number="CA-SQF-2024-0421") + other = _incident(db) + + response = client.post( + INCIDENTS_URL, + json={ + "extract_id": str(other.extract_id), + "incident_number": "CA-SQF-2024-0421", + }, + ) + + assert response.status_code == 409 + assert response.json()["error_code"] == "DUPLICATE_INCIDENT_NUMBER" + + def test_a_deleted_incident_frees_its_number(self, client, db): + _incident( + db, + incident_number="CA-SQF-2024-0421", + deleted_at=datetime.now(timezone.utc), + ) + other = _incident(db) + + response = client.post( + INCIDENTS_URL, + json={ + "extract_id": str(other.extract_id), + "incident_number": "CA-SQF-2024-0421", + }, + ) + + assert response.status_code == 201 + + +# --------------------------------------------------------------------------- +# GET /incidents +# --------------------------------------------------------------------------- + +class TestListIncidents: + def test_returns_rows_and_pagination(self, client, db): + incident = _incident(db) + _form(db, incident) + _form(db, incident) + + payload = client.get(INCIDENTS_URL).json() + + assert payload["pagination"] == { + "total": 1, + "page": 1, + "per_page": 20, + "total_pages": 1, + "has_next": False, + "has_prev": False, + } + row = payload["data"][0] + assert row["incident_name"] == "Bear Creek Wildfire" + assert row["incident_type"] == "wildland_fire" + assert row["incident_category"] == "fire" + assert row["city"] == "Reno" + assert row["forms_count"] == 2 + + def test_forms_count_is_zero_without_forms(self, client, db): + _incident(db) + assert client.get(INCIDENTS_URL).json()["data"][0]["forms_count"] == 0 + + def test_excludes_soft_deleted(self, client, db): + _incident(db) + _incident(db, deleted_at=datetime.now(timezone.utc)) + + payload = client.get(INCIDENTS_URL).json() + + assert payload["pagination"]["total"] == 1 + assert len(payload["data"]) == 1 + + def test_filters_by_status_and_category(self, client, db): + _incident(db, status=ReportStatus.approved) + _incident(db, status=ReportStatus.draft) + _incident(db, incident_category=IncidentCategory.ems) + + approved = client.get(INCIDENTS_URL, params={"status": "approved"}).json() + ems = client.get(INCIDENTS_URL, params={"incident_category": "ems"}).json() + + assert approved["pagination"]["total"] == 1 + assert ems["pagination"]["total"] == 1 + + def test_date_bounds_are_inclusive(self, client, db): + _incident(db, incident_datetime=datetime(2024, 7, 9, 23, 0)) + _incident(db, incident_datetime=datetime(2024, 7, 10, 13, 52)) + _incident(db, incident_datetime=datetime(2024, 7, 11, 0, 30)) + + payload = client.get( + INCIDENTS_URL, params={"date_from": "2024-07-10", "date_to": "2024-07-10"} + ).json() + + assert payload["pagination"]["total"] == 1 + assert payload["data"][0]["incident_datetime"].startswith("2024-07-10T13:52") + + def test_rows_without_a_datetime_are_dropped_by_a_date_filter(self, client, db): + _incident(db, incident_datetime=None) + + payload = client.get(INCIDENTS_URL, params={"date_from": "2024-07-10"}).json() + + assert payload["pagination"]["total"] == 0 + + def test_sort_order(self, client, db): + early = datetime(2024, 7, 1, 8, 0) + late = datetime(2024, 7, 20, 8, 0) + _incident(db, incident_datetime=early) + _incident(db, incident_datetime=late) + + desc = client.get(INCIDENTS_URL).json()["data"] + asc = client.get(INCIDENTS_URL, params={"sort": "date_asc"}).json()["data"] + + assert desc[0]["incident_datetime"].startswith("2024-07-20") + assert asc[0]["incident_datetime"].startswith("2024-07-01") + + def test_rows_without_a_datetime_sort_last(self, client, db): + _incident(db, incident_datetime=None) + _incident(db, incident_datetime=datetime(2024, 7, 1, 8, 0)) + + rows = client.get(INCIDENTS_URL).json()["data"] + + assert rows[0]["incident_datetime"] is not None + assert rows[-1]["incident_datetime"] is None + + def test_paging(self, client, db): + base = datetime(2024, 7, 1, 8, 0) + for offset in range(3): + _incident(db, incident_datetime=base + timedelta(days=offset)) + + page_one = client.get(INCIDENTS_URL, params={"per_page": 2}).json() + page_two = client.get(INCIDENTS_URL, params={"per_page": 2, "page": 2}).json() + + assert page_one["pagination"] == { + "total": 3, + "page": 1, + "per_page": 2, + "total_pages": 2, + "has_next": True, + "has_prev": False, + } + assert len(page_one["data"]) == 2 + assert len(page_two["data"]) == 1 + assert page_two["pagination"]["has_next"] is False + assert page_two["pagination"]["has_prev"] is True + + def test_empty_list(self, client, db): + payload = client.get(INCIDENTS_URL).json() + + assert payload["data"] == [] + assert payload["pagination"]["total"] == 0 + assert payload["pagination"]["total_pages"] == 0 + + def test_reversed_date_range_is_422(self, client, db): + response = client.get( + INCIDENTS_URL, params={"date_from": "2024-07-20", "date_to": "2024-07-10"} + ) + + assert response.status_code == 422 + + def test_bad_date_format_is_422(self, client, db): + assert client.get(INCIDENTS_URL, params={"date_from": "15/07/2024"}).status_code == 422 + + def test_per_page_over_the_cap_is_422(self, client, db): + assert client.get(INCIDENTS_URL, params={"per_page": 101}).status_code == 422 + + def test_unknown_sort_is_422(self, client, db): + assert client.get(INCIDENTS_URL, params={"sort": "name_asc"}).status_code == 422 + + +# --------------------------------------------------------------------------- +# GET /incidents/{incident_id} +# --------------------------------------------------------------------------- + +class TestGetIncident: + def test_returns_contract_and_forms(self, client, db): + incident = _incident(db) + form = _form(db, incident) + + payload = client.get(f"{INCIDENTS_URL}/{incident.incident_id}").json() + + assert payload["incident_contract"]["incident"]["name"] == "Bear Creek Wildfire" + assert [f["form_id"] for f in payload["forms"]] == [str(form.form_id)] + assert [f["form_id"] for f in payload["forms_generated"]] == [str(form.form_id)] + + def test_submission_log_is_empty_by_default(self, client, db): + incident = _incident(db) + payload = client.get(f"{INCIDENTS_URL}/{incident.incident_id}").json() + + assert payload["submission_log"] == [] + + def test_submission_log_is_read_from_the_contract(self, client, db): + contract = dict(_CONTRACT) + contract["submission_log"] = [ + {"form_type": "neris", "submitted_to": "State FMO", "status": "accepted"} + ] + incident = _incident(db, incident_contract=contract) + + payload = client.get(f"{INCIDENTS_URL}/{incident.incident_id}").json() + + assert payload["submission_log"][0]["submitted_to"] == "State FMO" + + def test_soft_deleted_is_still_readable(self, client, db): + incident = _incident(db, deleted_at=datetime.now(timezone.utc)) + + response = client.get(f"{INCIDENTS_URL}/{incident.incident_id}") + + assert response.status_code == 200 + assert response.json()["deleted_at"] is not None + + def test_unknown_id_is_404(self, client, db): + response = client.get(f"{INCIDENTS_URL}/{uuid4()}") + + assert response.status_code == 404 + assert response.json()["error_code"] == "INCIDENT_NOT_FOUND" + + +# --------------------------------------------------------------------------- +# PATCH /incidents/{incident_id} +# --------------------------------------------------------------------------- + +class TestUpdateIncident: + def test_updates_metadata(self, client, db): + incident = _incident(db) + + payload = client.patch( + f"{INCIDENTS_URL}/{incident.incident_id}", + json={"status": "approved", "tags": ["reviewed"], "notes": "Ready to go."}, + ).json() + + assert payload["status"] == "approved" + assert payload["tags"] == ["reviewed"] + assert payload["notes"] == "Ready to go." + + def test_omitted_fields_are_untouched(self, client, db): + incident = _incident(db, notes="Original note", tags=["wildland"]) + + payload = client.patch( + f"{INCIDENTS_URL}/{incident.incident_id}", json={"status": "under_review"} + ).json() + + assert payload["notes"] == "Original note" + assert payload["tags"] == ["wildland"] + + def test_does_not_touch_the_contract(self, client, db): + incident = _incident(db) + + client.patch( + f"{INCIDENTS_URL}/{incident.incident_id}", json={"notes": "Checked."} + ) + + db.refresh(incident) + assert incident.incident_contract == _CONTRACT + assert incident.incident_name == "Bear Creek Wildfire" + + def test_duplicate_number_is_409(self, client, db): + _incident(db, incident_number="CA-1") + target = _incident(db) + + response = client.patch( + f"{INCIDENTS_URL}/{target.incident_id}", json={"incident_number": "CA-1"} + ) + + assert response.status_code == 409 + assert response.json()["error_code"] == "DUPLICATE_INCIDENT_NUMBER" + + def test_keeping_its_own_number_is_allowed(self, client, db): + incident = _incident(db, incident_number="CA-1") + + response = client.patch( + f"{INCIDENTS_URL}/{incident.incident_id}", + json={"incident_number": "CA-1", "notes": "Same number."}, + ) + + assert response.status_code == 200 + + def test_submitted_is_still_editable_for_now(self, client, db): + """The submitted lock is deferred, so this documents current behaviour.""" + incident = _incident(db, status=ReportStatus.submitted) + + response = client.patch( + f"{INCIDENTS_URL}/{incident.incident_id}", json={"notes": "Late edit."} + ) + + assert response.status_code == 200 + + def test_unknown_id_is_404(self, client, db): + response = client.patch(f"{INCIDENTS_URL}/{uuid4()}", json={"notes": "x"}) + + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# DELETE /incidents/{incident_id} +# --------------------------------------------------------------------------- + +class TestDeleteIncident: + def test_soft_deletes(self, client, db): + incident = _incident(db) + + payload = client.delete(f"{INCIDENTS_URL}/{incident.incident_id}").json() + + assert payload["incident_id"] == str(incident.incident_id) + assert payload["recoverable"] is True + assert payload["deleted_at"] is not None + + def test_the_row_survives(self, client, db): + incident = _incident(db) + + client.delete(f"{INCIDENTS_URL}/{incident.incident_id}") + + db.refresh(incident) + assert incident.deleted_at is not None + assert incident.incident_contract == _CONTRACT + + def test_deleting_twice_is_409(self, client, db): + incident = _incident(db) + client.delete(f"{INCIDENTS_URL}/{incident.incident_id}") + + response = client.delete(f"{INCIDENTS_URL}/{incident.incident_id}") + + assert response.status_code == 409 + assert response.json()["error_code"] == "ALREADY_DELETED" + + def test_unknown_id_is_404(self, client, db): + assert client.delete(f"{INCIDENTS_URL}/{uuid4()}").status_code == 404 diff --git a/tests/test_v1_models.py b/tests/test_v1_models.py index 6584ac36..5b732a56 100644 --- a/tests/test_v1_models.py +++ b/tests/test_v1_models.py @@ -15,6 +15,7 @@ ExtractionStatus, FormStatus, FormType, + IncidentCategory, InputStatus, InputType, JobStatus, @@ -22,7 +23,7 @@ PeriodType, ReportStatus, ) -from app.models import Extraction, Form, Incident, Input, Report +from app.models import Extraction, Form, FormTemplate, Incident, Input, Report # --------------------------------------------------------------------------- @@ -46,6 +47,26 @@ def _extraction(db: Session, input_id: UUID, **kwargs) -> "Extraction": return row +def _incident(db: Session, **kwargs) -> Incident: + inp = _input(db) + ext = _extraction(db, inp.input_id) + defaults = dict(extract_id=ext.extract_id) + row = Incident(**{**defaults, **kwargs}) + db.add(row) + db.commit() + db.refresh(row) + return row + + +def _form_template(db: Session, **kwargs) -> FormTemplate: + defaults = dict(form_type=f"test_form_{uuid4().hex[:8]}", display_name="Test Form", fields=[]) + row = FormTemplate(**{**defaults, **kwargs}) + db.add(row) + db.commit() + db.refresh(row) + return row + + # --------------------------------------------------------------------------- # Input # --------------------------------------------------------------------------- @@ -123,21 +144,21 @@ def test_defaults_on_create(self, db): assert isinstance(ext.extract_id, UUID) assert ext.input_id == inp.input_id assert ext.status == ExtractionStatus.processing - assert ext.incident_contract is None + assert ext.partial_result is None assert ext.corrections is None assert ext.started_at is None - def test_incident_contract_json_roundtrip(self, db): + def test_partial_result_json_roundtrip(self, db): inp = _input(db) - contract = { + partial = { "schema_version": "1.1.0", "incident": {"name": "Structure Fire Main St", "types": []}, "location": {"address": "123 Main St", "state": "CA"}, } - ext = _extraction(db, inp.input_id, incident_contract=contract) + ext = _extraction(db, inp.input_id, partial_result=partial) fetched = db.get(Extraction, ext.extract_id) - assert fetched.incident_contract["schema_version"] == "1.1.0" - assert fetched.incident_contract["location"]["state"] == "CA" + assert fetched.partial_result["schema_version"] == "1.1.0" + assert fetched.partial_result["location"]["state"] == "CA" def test_corrections_json_roundtrip(self, db): inp = _input(db) @@ -224,19 +245,60 @@ def test_soft_delete(self, db): fetched = db.get(Incident, row.incident_id) assert fetched.deleted_at is not None - def test_incident_date_field(self, db): - row = self._make(db, incident_date=date(2026, 5, 15)) - fetched = db.get(Incident, row.incident_id) - assert fetched.incident_date == date(2026, 5, 15) - def test_all_nullable_fields_default_none(self, db): row = self._make(db) assert row.incident_number is None assert row.incident_name is None assert row.incident_type is None - assert row.incident_date is None assert row.notes is None + def test_incident_contract_json_roundtrip(self, db): + contract = { + "schema_version": "1.1.0", + "incident": {"name": "Structure Fire Main St", "types": []}, + "location": {"address": "123 Main St", "state": "CA"}, + } + row = self._make(db, incident_contract=contract) + fetched = db.get(Incident, row.incident_id) + assert fetched.incident_contract["schema_version"] == "1.1.0" + assert fetched.incident_contract["location"]["state"] == "CA" + + def test_promoted_columns_default_none(self, db): + row = self._make(db) + assert row.incident_contract is None + assert row.incident_category is None + assert row.incident_datetime is None + assert row.city is None + assert row.civilian_injuries is None + assert row.area_burned_ha is None + assert row.total_loss_amount is None + assert row.call_to_arrival_seconds is None + + def test_promoted_columns_roundtrip(self, db): + row = self._make( + db, + incident_category=IncidentCategory.fire, + incident_datetime=datetime(2026, 5, 15, 14, 30, tzinfo=timezone.utc), + city="Oakland", + state="CA", + country="US", + civilian_injuries=2, + responder_fatalities=0, + people_evacuated=40, + structures_destroyed=3, + area_burned_ha=12.5, + total_loss_amount=250000.0, + total_loss_currency="USD", + call_to_arrival_seconds=312, + ) + fetched = db.get(Incident, row.incident_id) + assert fetched.incident_category == IncidentCategory.fire + assert fetched.city == "Oakland" + assert fetched.civilian_injuries == 2 + assert fetched.area_burned_ha == pytest.approx(12.5) + assert fetched.total_loss_currency == "USD" + assert fetched.call_to_arrival_seconds == 312 + # --------------------------------------------------------------------------- # Form @@ -245,10 +307,11 @@ def test_all_nullable_fields_default_none(self, db): class TestFormModel: def _make(self, db, **kwargs): - inp = _input(db) - ext = _extraction(db, inp.input_id) + template = _form_template(db) + incident = _incident(db) defaults = dict( - extract_id=ext.extract_id, + template_id=template.template_id, + incident_id=incident.incident_id, form_type=FormType.nfirs_basic, ) row = Form(**{**defaults, **kwargs}) @@ -263,7 +326,9 @@ def test_defaults_on_create(self, db): assert row.status == FormStatus.queued assert row.pdf_ready is False assert row.json_ready is False - assert row.incident_id is None + assert isinstance(row.template_id, UUID) + assert isinstance(row.incident_id, UUID) + assert row.batch_id is None assert row.job_id is None assert row.completed_at is None @@ -287,15 +352,11 @@ def test_json_data_roundtrip(self, db): assert fetched.json_data["FDID"] == "CA99901" def test_incident_fk_links_correctly(self, db): - inp = _input(db) - ext = _extraction(db, inp.input_id) - incident = Incident(extract_id=ext.extract_id) - db.add(incident) - db.commit() - db.refresh(incident) + template = _form_template(db) + incident = _incident(db) form = Form( - extract_id=ext.extract_id, + template_id=template.template_id, form_type=FormType.neris, incident_id=incident.incident_id, ) @@ -304,6 +365,27 @@ def test_incident_fk_links_correctly(self, db): db.refresh(form) assert form.incident_id == incident.incident_id + def test_template_fk_links_correctly(self, db): + template = _form_template(db) + incident = _incident(db) + + form = Form( + template_id=template.template_id, + form_type=FormType.neris, + incident_id=incident.incident_id, + ) + db.add(form) + db.commit() + db.refresh(form) + assert form.template_id == template.template_id + + def test_batch_id_roundtrip(self, db): + """batch_id is a plain UUID grouping key — no Batch table, no FK.""" + batch_id = uuid4() + row = self._make(db, batch_id=batch_id) + fetched = db.get(Form, row.form_id) + assert fetched.batch_id == batch_id + def test_job_id_stored_without_fk(self, db): """job_id is a plain UUID — can store any UUID without a FK constraint.""" arbitrary_uuid = uuid4() @@ -312,10 +394,14 @@ def test_job_id_stored_without_fk(self, db): assert fetched.job_id == arbitrary_uuid def test_all_form_types_accepted(self, db): - inp = _input(db) - ext = _extraction(db, inp.input_id) + template = _form_template(db) + incident = _incident(db) for ft in FormType: - form = Form(extract_id=ext.extract_id, form_type=ft) + form = Form( + template_id=template.template_id, + incident_id=incident.incident_id, + form_type=ft, + ) db.add(form) db.commit() results = db.exec(select(Form)).all() diff --git a/tests/test_v1_system.py b/tests/test_v1_system.py index b7b8dfcb..f41b2945 100644 --- a/tests/test_v1_system.py +++ b/tests/test_v1_system.py @@ -1,8 +1,8 @@ """Tests for GET /api/v1/health, /schema/incident, /schema/incident/versions. -External dependencies (Ollama, Whisper, disk, DB) are mocked at the route-module -boundary. The health check uses engine.connect() directly (not the get_db -dependency) so we mock system_mod.engine explicitly in every test. +External dependencies (the LLM provider, Whisper, disk, DB) are mocked at the +route-module boundary. The health check uses engine.connect() directly (not the +get_db dependency) so we mock system_mod.engine explicitly in every test. """ from unittest.mock import MagicMock @@ -11,6 +11,7 @@ import requests as requests_lib import app.api.routes.system as system_mod +from app.services.llm.models import ModelInfo, ProviderHealth # --------------------------------------------------------------------------- @@ -53,6 +54,37 @@ def _mock_shutil(free_bytes: int = 120 * 1024 ** 3, raise_oserror: bool = False) return mock +def _fake_llm( + monkeypatch, + status="healthy", + provider="ollama", + model="llama3:8b", + external=False, + probed=True, + response_time_ms=15, + detail=None, + models=("llama3:8b", "mistral:7b"), +): + """Stand in for the LLM module the health route asks.""" + report = ProviderHealth( + provider=provider, + label=provider.title(), + model=model, + external=external, + status=status, + probed=probed, + detail=detail, + response_time_ms=response_time_ms, + ) + monkeypatch.setattr(system_mod.llm, "health", lambda: report) + monkeypatch.setattr( + system_mod.llm, + "list_models", + lambda: [ModelInfo(name=name, default=name == model) for name in models], + ) + return report + + def _make_mock_response(json_data=None, status_code=200): m = MagicMock() m.status_code = status_code @@ -117,6 +149,11 @@ def _fake_get_whisper_down(url, timeout=None): class TestHealthEndpoint: + @pytest.fixture(autouse=True) + def _healthy_llm(self, monkeypatch): + """A working local provider, unless a test says otherwise.""" + _fake_llm(monkeypatch) + def test_all_healthy(self, client, monkeypatch): monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) @@ -132,21 +169,33 @@ def test_all_healthy(self, client, monkeypatch): comps = body["components"] assert comps["database"]["status"] == "healthy" - assert comps["ollama"]["status"] == "healthy" + assert comps["llm"]["status"] == "healthy" assert comps["whisper"]["status"] == "healthy" assert comps["storage"]["status"] == "healthy" - def test_ollama_down_returns_200_degraded(self, client, monkeypatch): + def test_llm_component_names_the_provider_and_model(self, client, monkeypatch): + monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) + monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) + monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_all_healthy) + + component = client.get("/api/v1/health").json()["components"]["llm"] + assert component["provider"] == "ollama" + assert component["model"] == "llama3:8b" + assert component["external"] is False + assert component["probed"] is True + + def test_llm_down_returns_200_degraded(self, client, monkeypatch): monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_ollama_down) + _fake_llm(monkeypatch, status="unhealthy", detail="connection refused") resp = client.get("/api/v1/health") assert resp.status_code == 200 body = resp.json() assert body["status"] == "degraded" - assert body["components"]["ollama"]["status"] == "unhealthy" + assert body["components"]["llm"]["status"] == "unhealthy" assert body["components"]["database"]["status"] == "healthy" assert body["components"]["whisper"]["status"] == "healthy" @@ -161,7 +210,7 @@ def test_whisper_down_returns_200_degraded(self, client, monkeypatch): body = resp.json() assert body["status"] == "degraded" assert body["components"]["whisper"]["status"] == "unhealthy" - assert body["components"]["ollama"]["status"] == "healthy" + assert body["components"]["llm"]["status"] == "healthy" def test_db_down_returns_503_unhealthy(self, client, monkeypatch): monkeypatch.setattr(system_mod, "engine", _mock_engine_down()) @@ -192,18 +241,7 @@ def test_db_execute_fails_returns_503_unhealthy(self, client, monkeypatch): assert body["components"]["database"]["status"] == "unhealthy" assert "query failed" in body["components"]["database"]["detail"] - def test_current_load_not_fabricated(self, client, monkeypatch): - """current_load must be absent or null — never a made-up value.""" - monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) - monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) - monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_all_healthy) - - resp = client.get("/api/v1/health") - assert resp.status_code == 200 - ollama_comp = resp.json()["components"]["ollama"] - assert ollama_comp.get("current_load") is None - - def test_ollama_slow_returns_degraded(self, client, monkeypatch): + def test_slow_llm_returns_degraded(self, client, monkeypatch): """Patch _SLOW_MS to -1 so any measured elapsed time triggers degraded, without depending on wall-clock timing in CI. """ @@ -216,44 +254,39 @@ def test_ollama_slow_returns_degraded(self, client, monkeypatch): assert resp.status_code == 200 body = resp.json() assert body["status"] == "degraded" - assert body["components"]["ollama"]["status"] == "degraded" - - def test_models_available_and_loaded_flag(self, client, monkeypatch): - """models_available is built from /api/tags; loaded=True only for models in /api/ps.""" - monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) - monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) - monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_all_healthy) + assert body["components"]["llm"]["status"] == "degraded" - resp = client.get("/api/v1/health") - assert resp.status_code == 200 - models = resp.json()["components"]["ollama"]["models_available"] - assert len(models) == 2 - llama = next(m for m in models if m["name"] == "llama3:8b") - mistral = next(m for m in models if m["name"] == "mistral:7b") - assert llama["loaded"] is True - assert mistral["loaded"] is False - assert llama["quantization"] == "Q4_K_M" - assert llama["size_gb"] == pytest.approx(4.66, abs=0.1) - - def test_model_loaded_reflects_running_model(self, client, monkeypatch): + def test_models_available_lists_what_the_provider_serves(self, client, monkeypatch): monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_all_healthy) resp = client.get("/api/v1/health") assert resp.status_code == 200 - ollama = resp.json()["components"]["ollama"] - assert ollama["model_loaded"] == "llama3:8b" + assert resp.json()["components"]["llm"]["models_available"] == [ + "llama3:8b", + "mistral:7b", + ] - def test_ollama_version_present(self, client, monkeypatch): + def test_a_hosted_provider_is_flagged_and_not_listed(self, client, monkeypatch): + """A hosted provider costs quota to ask, so health does not ask it.""" monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy()) monkeypatch.setattr(system_mod, "shutil", _mock_shutil()) monkeypatch.setattr("app.api.routes.system.requests.get", _fake_get_all_healthy) - - resp = client.get("/api/v1/health") - assert resp.status_code == 200 - ollama = resp.json()["components"]["ollama"] - assert ollama["ollama_version"] == "0.3.0" + _fake_llm( + monkeypatch, + provider="gemini", + model="gemini-2.0-flash", + external=True, + probed=False, + response_time_ms=None, + detail="hosted provider, not probed to avoid spending quota", + ) + + component = client.get("/api/v1/health").json()["components"]["llm"] + assert component["external"] is True + assert component["probed"] is False + assert component.get("models_available") is None def test_storage_disk_free_present(self, client, monkeypatch): monkeypatch.setattr(system_mod, "engine", _mock_engine_healthy())