diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index c21bf595c..c58f542d6 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -73,8 +73,8 @@ jobs: run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py apply --create-missing-roles - name: Verify the managed v2 roles hold their declared privileges run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - - name: Test v2 metadata publication, resource routes, the countries list, reports and simulations - run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py + - name: Test v2 metadata publication, resource routes, the countries list, reports, simulations and users' report runs + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py tests/integration/test_v2_report_reads_by_run.py tests/integration/test_v2_user_report_runs.py tests/integration/test_v2_user_report_run_requesters.py tests/integration/test_v2_user_budget_window_runs.py tests/integration/test_v2_forced_report_reruns.py tests/integration/test_v2_user_report_views.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning, user associations, saved detailed-save records and user types diff --git a/changelog.d/user-report-runs.added.md b/changelog.d/user-report-runs.added.md new file mode 100644 index 000000000..c755311c8 --- /dev/null +++ b/changelog.d/user-report-runs.added.md @@ -0,0 +1 @@ +`/v2/reports` records each user's report runs and reads a report as each user sees it. `POST /v2/reports` and `POST /v2/reports/{report_id}/runs` take an optional user credential (a bearer access token, the `PolicyEngine-Unauthenticated-User` header, or a server forwarding either); with one, the request is recorded in `user_report_run_associations` for the run it ended on, created or joined (each year's report run for a budget window), and the user's saved reports of the report take that time as `last_run_at`. Both answer the report with the run the simulation service named, which needn't be its newest, and name it as `run_id` when the report can't be read back (502) or the request can't be recorded (503); only admin users may force a re-run. `GET /v2/reports/{report_id}?run_id=` reads a chosen run; without `run_id`, a read answers the requesting user's newest recorded run, else the newest succeeded run that wasn't forced, else the newest run that wasn't forced. A report's label and description are the requesting user's saved report's, null without one; a create's `label` and `description` are optional and no longer sent to the simulation service. A saved detailed-save report's `last_run_at` is its owner's newest request, and a sign-in that merges a browser into an existing account moves the browser's run records with it. diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 7208bfe98..118fdf47b 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -239,6 +239,12 @@ def _default_v2_stored_report_service_factory() -> V2StoredReportResourceService from policyengine_api.data.v2.database import ( get_v2_session_factory, ) + from policyengine_api.services.v2.report_run_records.database_session import ( + ReportRunRecordDatabaseSession, + ) + from policyengine_api.services.v2.report_run_records.services import ( + ReportRunRecordService, + ) from policyengine_api.services.v2.stored_reports.database_session import ( StoredReportDatabaseSession, ) @@ -250,6 +256,9 @@ def _default_v2_stored_report_service_factory() -> V2StoredReportResourceService StoredReportDatabaseSession(get_v2_session_factory()), current_versions=_current_package_versions, submissions=_default_simulation_submissions(), + run_records=ReportRunRecordService( + ReportRunRecordDatabaseSession(get_v2_session_factory()) + ), ) @@ -290,6 +299,26 @@ def _default_v2_stored_simulation_service_factory() -> ( ) +def _user_authorization( + request: Request, credentials: HTTPAuthorizationCredentials | None +) -> str | None: + """Return the user's bearer credential, or None when the request sends none. + + When the credential check before routing found a server credential, it + is the one that server forwards (see ``user_auth_support``). + """ + caller = current_request_caller() + if caller is not None and caller.kind is CallerKind.SERVER: + # Authorization carries the server's own token; the user is the one + # the server forwards. + return forwarded_user_credential( + request.headers.getlist(FORWARDED_USER_AUTHORIZATION_HEADER) + ) + if credentials is not None: + return f"Bearer {credentials.credentials}" + return None + + def _default_require_user( request: Request, credentials: Annotated[ @@ -302,17 +331,28 @@ def _default_require_user( When the credential check before routing found a server credential, the account is the user that server forwards (see ``user_auth_support``). """ - caller = current_request_caller() - if caller is not None and caller.kind is CallerKind.SERVER: - # Authorization carries the server's own token; the user is the one - # the server forwards. - user_authorization = forwarded_user_credential( - request.headers.getlist(FORWARDED_USER_AUTHORIZATION_HEADER) - ) - elif credentials is not None: - user_authorization = f"Bearer {credentials.credentials}" - else: - user_authorization = None + return authenticate_account_request( + _user_authorization(request, credentials), unauthenticated_user + ) + + +def _default_optional_user( + request: Request, + credentials: Annotated[ + HTTPAuthorizationCredentials | None, Depends(_user_bearer_scheme) + ], + unauthenticated_user: Annotated[str | None, Depends(_unauthenticated_user_scheme)], +) -> AccountPrincipal | None: + """Authenticate the request's account, or return None when it names no user. + + A request sending no user credential, such as an API caller's, or a + server's forwarding none, acts as nobody. One that sends a credential is + authenticated exactly as ``_default_require_user`` authenticates it, and + refused the same way when it is not valid. + """ + user_authorization = _user_authorization(request, credentials) + if user_authorization is None and unauthenticated_user is None: + return None return authenticate_account_request(user_authorization, unauthenticated_user) @@ -498,6 +538,8 @@ class NativeRouteDependencies: # fails closed (401/403/503); tests replace it with a callable returning a # fixed principal. require_user: Callable[..., AccountPrincipal] = _default_require_user + # The account of a route a user may call without signing in, or None. + optional_user: Callable[..., AccountPrincipal | None] = _default_optional_user @classmethod def defaults(cls) -> NativeRouteDependencies: @@ -529,4 +571,5 @@ def defaults(cls) -> NativeRouteDependencies: v2_account_service_factory=_default_v2_account_service_factory, v2_user_type_service_factory=_default_v2_user_type_service_factory, require_user=_default_require_user, + optional_user=_default_optional_user, ) diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index fef9d0c39..c099267e2 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -191,29 +191,59 @@ def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: class V2StoredReportResourceService(Protocol): """Route-independent report operations for one request.""" - def get_report(self, *, country_id: str, report_id: UUID) -> StoredReportRead: - """Return the country's report, its newest run and that run's result. - - The report is an economy report, a household report or a budget - window. Raise ``StoredReportNotFoundError`` when the country has no - such report, and ``EconomyResultUnreadableError``, - ``HouseholdResultUnreadableError`` or + def get_report( + self, + *, + country_id: str, + report_id: UUID, + run_id: UUID | None = None, + requester: AccountPrincipal | None = None, + ) -> StoredReportRead: + """Return the country's report, a run and that run's result, as a user sees it. + + The run is ``run_id``; without it, the requester's newest recorded + run, else the report's newest succeeded unforced run, else its newest + unforced run. The label and description are the requester's saved + report's. The report is an economy report, a household report or a + budget window. Raise ``StoredReportNotFoundError`` when the country + has no such report, ``ReportRunNotFoundError`` when it has no run + ``run_id``, ``StandardUserRequiresSignInError`` for an + unauthenticated credential naming a signed-in user, and + ``EconomyResultUnreadableError``, ``HouseholdResultUnreadableError`` or ``BudgetWindowResultUnreadableError`` when a succeeded run's result cannot be read. """ ... def list_reports( - self, *, country_id: str, report_ids: Sequence[UUID] + self, + *, + country_id: str, + report_ids: Sequence[UUID], + requester: AccountPrincipal | None = None, ) -> tuple[StoredReportSummaryRead, ...]: - """Return the country's reports of every kind among those named, in order.""" + """Return the country's reports of every kind among those named, in order. + + Each is read as ``get_report`` reads it without a run, as the + requester sees it. + """ ... - def create_report(self, report_input: ReportCreationInput) -> StoredReportRead: + def create_report( + self, + report_input: ReportCreationInput, + *, + requester: AccountPrincipal | None = None, + ) -> StoredReportRead: """Create the report through the simulation service; return it as stored. - Raise ``EconomyCalculationsUnsupportedError`` for an economy report in - a country without economy reports, + It is answered with the run the simulation service named, created or + joined. With a ``requester``, the request is recorded for its user + once the simulation service has answered. Raise + ``StandardUserRequiresSignInError`` for an unauthenticated credential + that names a signed-in user, before anything is asked; + ``EconomyCalculationsUnsupportedError`` for an economy report in a + country without economy reports, ``HouseholdCalculationCountryError`` for a household report in a country without household calculations, ``HouseholdNotFoundError`` for a household the country does not store, @@ -223,16 +253,27 @@ def create_report(self, report_input: ReportCreationInput) -> StoredReportRead: ``PolicyParametersMissingError`` for a policy that sets parameters the running catalog version lacks, ``SubmissionStorageError`` when the database or its catalog cannot be read for those checks, the - simulation service's errors, and ``CreatedReportUnreadableError`` when - the report it created cannot be read back. + simulation service's errors, ``ReportRunNotRecordedError`` when the + request could not be recorded, and ``CreatedReportUnreadableError`` + when the report it created cannot be read back. """ ... def rerun_report( - self, *, country_id: str, report_id: UUID, force: bool + self, + *, + country_id: str, + report_id: UUID, + force: bool, + requester: AccountPrincipal | None = None, ) -> StoredReportRead: - """Run the country's report again; return it with its new run as its newest. + """Run the country's report again; return it with the run the request ended on. + That is the run the simulation service named, created or joined. + With a ``requester``, the request is recorded for its user once the + simulation service has answered, and + ``StandardUserRequiresSignInError`` is raised, before anything is + asked, for an unauthenticated credential that names a signed-in user. Raise ``StoredReportNotFoundError`` when the country has no such report; the errors of the country check a new report of its kind makes (economy calculations, or household calculations for a household @@ -240,8 +281,11 @@ def rerun_report( ``create_report`` raises them (a household report's household is not checked again); the simulation service's errors, among them ``SimulationServiceConflictError`` when it will not run the report - again now, and ``RerunReportUnreadableError`` when the report cannot - be read back once its new run was started. + again now, ``ReportRunNotRecordedError`` when the request could not be + recorded, and ``RerunReportUnreadableError`` when the report cannot + be read back once its run was started, or when a budget window's + answer names no window run (its ``run_id`` is then None, and nothing + is recorded). """ ... diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 3cf52c748..a8a497cc1 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -85,10 +85,12 @@ class ReportSimulationItem(StrictReportAPIModel): class ReportDefinitionItem(StrictReportAPIModel): """What every report item gives: its definition and the current versions. - ``current_versions`` are the versions the Public API runs now, so a - newest run on other versions is out of date. ``options`` are the options - the report was created with. ``label`` is null for a report the simulation - service shares between requests, which records no one caller's label. + ``label`` and ``description`` are the requesting user's saved report's + name and description, null without a user credential or a saved report: + a report the simulation service shares between requests records no one + caller's. ``current_versions`` are the versions the Public API runs now, + so a run on other versions is out of date. ``options`` are the options + the report was created with. """ id: UUID @@ -106,9 +108,11 @@ class ReportDefinitionItem(StrictReportAPIModel): class ReportSummaryItem(ReportDefinitionItem): """An economy report comparing a baseline and a reform simulation. - Both simulations use the region and dataset named. ``status`` is the - newest run's, or ``not_run`` for a report with no run. ``run`` is the - newest run, whatever its status and versions. + Both simulations use the region and dataset named. ``run`` is the run + the read answers: the one it names, else the requesting user's newest + recorded run, else the newest succeeded run that wasn't forced, else the + newest run that wasn't forced, whatever its versions. ``status`` is that + run's, or ``not_run`` without one. """ kind: Literal["economy"] @@ -124,10 +128,10 @@ def from_summary(cls, read: ReportSummaryRead) -> ReportSummaryItem: class ReportItem(ReportSummaryItem): - """An economy report with its newest run's result. + """An economy report with its run's result. ``result`` is the single-year economy result, in the format the app reads - today, once the newest run has succeeded, and null otherwise. + today, once the run read has succeeded, and null otherwise. """ result: EconomyResult | None @@ -143,9 +147,9 @@ class HouseholdReportSummaryItem(ReportDefinitionItem): Both simulations calculate the household ``household_id`` in its default year, ``year``; ``reform_simulation`` is null for a report of the - baseline alone. ``status`` is the newest run's, or ``not_run`` for a - report with no run. ``run`` is the newest run, whatever its status and - versions, naming each simulation's run. + baseline alone. ``run`` is the run the read answers, chosen as for an + economy report, naming each simulation's run; ``status`` is that run's, + or ``not_run`` without one. """ kind: Literal["household"] @@ -201,9 +205,9 @@ class HouseholdReportFailureItem(StrictReportAPIModel): class HouseholdReportItem(HouseholdReportSummaryItem): - """A household report with its newest run's outcome. + """A household report with its run's outcome. - ``result`` holds the calculated households once the newest run has + ``result`` holds the calculated households once the run read has succeeded, and ``failure`` each side's error once it has failed; each is null otherwise. """ @@ -262,12 +266,13 @@ class BudgetWindowReportSummaryItem(StrictReportAPIModel): The window is ``budget_window_size`` years from ``year``, in the region and dataset named. ``baseline_simulation`` and ``reform_simulation`` are the first year's; each year's run names that year's own simulation runs. - ``status`` is the newest run's, or ``not_run`` for a - report with no run. ``run`` is the newest budget-window run, whatever its - status and versions, with its years; ``current_versions`` are the - versions the Public API runs now, so a newest run on other versions is - out of date. ``options`` are the options the report was created with. - ``label`` is null for a shared report, which records no one caller's label. + ``run`` is the budget-window run the read answers, chosen as for an + economy report (a recalculated window run is a forced one), with its + years; ``status`` is that run's, or ``not_run`` without one. + ``current_versions`` are the versions the Public API runs now, so a run + on other versions is out of date. ``options`` are the options the report + was created with. ``label`` and ``description`` are the requesting user's + saved report's, null otherwise. """ id: UUID @@ -296,11 +301,11 @@ def from_summary( class BudgetWindowReportItem(BudgetWindowReportSummaryItem): - """A budget window with its newest run's result. + """A budget window with its run's result. ``result`` is the budget-window result, in the format the app reads today: each year's budget figures, in year order, and their totals. It is - there once the newest run has succeeded, and null otherwise. + there once the run read has succeeded, and null otherwise. """ result: BudgetWindowResult | None @@ -377,11 +382,14 @@ class ReportListResponse(ReportSuccessResponse[ReportListResult]): class CreatedReportUnreadableResponse(V2ErrorResponse): """The 502 answer when the created report cannot be read back. - The simulation service created ``report_id``, so the caller reads it - with ``GET /v2/reports/{report_id}`` rather than submitting it again. + The simulation service created ``report_id`` with the run ``run_id`` (a + report run, or a budget window's budget-window run), so the caller reads + it with ``GET /v2/reports/{report_id}?run_id={run_id}`` rather than + submitting it again. """ report_id: UUID + run_id: UUID class UnavailableAfterReportCreatedResponse(V2ErrorResponse): @@ -394,6 +402,36 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): report_id: UUID +class ReportRunNotRecordedResponse(V2ErrorResponse): + """The 503 answer when a user's request for a run could not be recorded. + + The simulation service created or joined the run ``run_id`` of + ``report_id`` (a report run, or a budget window's budget-window run), so + the caller reads it with ``GET /v2/reports/{report_id}?run_id={run_id}``. + Sending the request again records it, once the simulation service finds + runs by key, since the request then ends on the same run. + """ + + report_id: UUID + run_id: UUID + + +UNAUTHENTICATED_REQUESTER = ( + "The request sent a user credential that is not valid: a bearer access " + "token that is invalid, or an unauthenticated-user header that is not a " + "canonical UUID or names a signed-in user. Nothing was started." +) +MACHINE_REQUESTER = ( + "A server forwarded a machine client's token as the user's: these routes " + "record requests for users only. Nothing was started." +) +NOT_RECORDED = ( + "Or the simulation service started the run, but the user's request for it " + "could not be recorded: the answer gives its report_id and run_id, to read " + "with GET /v2/reports/{report_id}?run_id={run_id}." +) + + # Every report route can answer these. REPORT_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 422: error_response("The request does not match the schema."), @@ -401,10 +439,23 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): 503: error_response("The v2 database or the simulation service is unavailable."), } READ_RESPONSES: dict[int | str, dict[str, Any]] = { - 404: error_response("No report has this ID in this country."), + 401: error_response( + "The request sent a user credential that is not valid: a bearer access " + "token that is invalid, or an unauthenticated-user header that is not a " + "canonical UUID or names a signed-in user." + ), + 403: error_response( + "A server forwarded a machine client's token as the user's: reports are " + "read as users see them." + ), + 404: error_response( + "No report has this ID in this country, or run_id is not one of the " + "report's runs (a budget window's budget-window runs; any other " + "report's own report runs)." + ), 422: error_response("The request does not match the schema."), 500: error_response( - "The newest run succeeded but its stored result cannot be read (for a " + "The run read succeeded but its stored result cannot be read (for a " "household report, a simulation run's calculated household; for a " "budget window, a year has no succeeded run or no budget figures), or " "the operation could not be completed." @@ -412,6 +463,8 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): } CREATE_RESPONSES: dict[int | str, dict[str, Any]] = { 400: error_response("The body and query name different countries."), + 401: error_response(UNAUTHENTICATED_REQUESTER), + 403: error_response(MACHINE_REQUESTER), 404: error_response( "A policy the report names, or a household report's household, is not " "stored in the country." @@ -450,12 +503,15 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): "again, which creates a second report if the first request went through. " "Or it created the report, but its answer cannot be read. Or it created " "the report, but the report cannot be read back: the answer gives its " - "report_id, to read with GET /v2/reports/{report_id} rather than " - "submitting it again." + "report_id and run_id, to read with GET " + "/v2/reports/{report_id}?run_id={run_id} rather than submitting it " + "again." ), }, 503: { - "model": V2ErrorResponse | UnavailableAfterReportCreatedResponse, + "model": V2ErrorResponse + | UnavailableAfterReportCreatedResponse + | ReportRunNotRecordedResponse, "description": ( "The simulation service created nothing: it cannot be reached or is not " "configured; it answered a readable 503 naming no record, and the message " @@ -468,7 +524,7 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): "be read. Or the service created the report, then " "Modal refused the run's call and the service failed the run: its 503 " "named the report, with its status and code, and the answer gives its " - "report_id." + f"report_id. {NOT_RECORDED}" ), }, } diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index 681e81851..c4b888fe9 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -1,7 +1,10 @@ """Native FastAPI routes for v2 reports. The routes need no user credential: a user keeps a report through -``/v2/user-reports``. Reports, their runs and results are read from the +``/v2/user-reports``. A create or re-run may send one (a bearer access +token, the unauthenticated-user header, or a server's forwarded user), and +the request is then recorded for that user, naming the run it ended on. +Reports, their runs and results are read from the detailed-save tables the simulation service and its Modal workers write; reading never calls the simulation service and never starts a run. A report is an economy report, a household report or a budget window, which its @@ -12,19 +15,24 @@ first. """ -from __future__ import annotations - -from typing import TYPE_CHECKING, Annotated, TypeVar +# No postponed annotations: FastAPI resolves each route's dependency +# annotations at runtime, and the requester's names a dependency local to the +# router builder. +from collections.abc import Callable +from typing import Annotated, Any, TypeVar from uuid import UUID from fastapi import APIRouter, Depends from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2StoredReportResourceService, _default_v2_stored_report_service_factory, ) from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.errors_support import ( + BEARER_CHALLENGE, V2ErrorMapping, v2_error_response, ) @@ -40,6 +48,7 @@ ReportDetailResult, ReportListResponse, ReportListResult, + ReportRunNotRecordedResponse, UnavailableAfterReportCreatedResponse, report_item, report_summary_item, @@ -69,6 +78,13 @@ ReportDetailQuery, ReportRunsQuery, ) +from policyengine_api.services.v2.identity.types import AccountPrincipal +from policyengine_api.services.v2.identity.validators import ( + StandardUserRequiresSignInError, +) +from policyengine_api.services.v2.report_run_records.validators import ( + ReportRunRecordStorageError, +) from policyengine_api.services.v2.simulation_service.types import ( SimulationServiceConflictError, SimulationServiceUnavailableError, @@ -84,19 +100,15 @@ ) from policyengine_api.services.v2.stored_reports.validators import ( CreatedReportUnreadableError, + ForcedRerunRequiresAdminError, + ForcedRerunRequiresSignInError, + ReportRunNotFoundError, + ReportRunNotRecordedError, ReportStorageError, RerunReportUnreadableError, StoredReportNotFoundError, ) -if TYPE_CHECKING: - from collections.abc import Callable - - from policyengine_api.fastapi_routes.dependencies import ( - NativeRouteDependencies, - V2StoredReportResourceService, - ) - MAXIMUM_REPORT_REQUEST_BYTES = 16 * 1024 enforce_report_request_size = request_size_limit( MAXIMUM_REPORT_REQUEST_BYTES, label="16 KiB" @@ -117,10 +129,13 @@ # Refusals a caller can act on carry their own message and are not logged. REPORT_REFUSALS: tuple[type[Exception], ...] = ( StoredReportNotFoundError, + ReportRunNotFoundError, + ForcedRerunRequiresAdminError, *SUBMISSION_REFUSALS, ) REPORT_ERROR_MAPPINGS = ( - V2ErrorMapping((StoredReportNotFoundError,), 404), + V2ErrorMapping((StoredReportNotFoundError, ReportRunNotFoundError), 404), + V2ErrorMapping((ForcedRerunRequiresAdminError,), 403), *SUBMISSION_ERROR_MAPPINGS, V2ErrorMapping( ( @@ -132,7 +147,11 @@ "The report's stored result cannot be read", ), # SUBMISSION_ERROR_MAPPINGS answers a missing v2 database configuration. - V2ErrorMapping((ReportStorageError,), 503, "V2 report storage is unavailable"), + V2ErrorMapping( + (ReportStorageError, ReportRunRecordStorageError), + 503, + "V2 report storage is unavailable", + ), ) @@ -159,18 +178,84 @@ def _report_operation( ) +def _read_path(report_id: UUID, country_id: str, run_id: UUID) -> str: + """Return the path that reads the report with its run ``run_id``.""" + return f"/v2/reports/{report_id}?country_id={country_id}&run_id={run_id}" + + +def _rerun_unreadable_message(error: RerunReportUnreadableError) -> str: + """Say where to read the run a re-run started, or its report's runs. + + A budget window's answer that named no window run names no run to read, + so the message points to the report's run list. + """ + started = f"The simulation service started a run of report {error.report_id}" + if error.run_id is None: + return ( + f"{started}, but its answer named no budget-window run. Find the run " + f"with GET /v2/reports/{error.report_id}/runs?country_id=" + f"{error.country_id} rather than running it again." + ) + return ( + f"{started}, but the report could not be read back. Read it with GET " + f"{_read_path(error.report_id, error.country_id, error.run_id)} rather " + "than running it again." + ) + + +def _requester_refused( + error: StandardUserRequiresSignInError | ForcedRerunRequiresSignInError, +) -> JSONResponse: + """Answer a browser credential naming a signed-in user, as user routes do.""" + return v2_error_response( + 401, str(error), headers={"WWW-Authenticate": BEARER_CHALLENGE} + ) + + +# The two routes that record a request take its user's credential, but +# optionally: FastAPI lists the schemes a credential may use, and an empty +# requirement says a request may send none. +OPTIONAL_USER_CREDENTIAL: dict[str, Any] = {"security": [{}]} + + +def _not_recorded(error: ReportRunNotRecordedError) -> JSONResponse: + """Answer a run whose request could not be recorded with 503 naming the run. + + The failure logged is the record's own, the error's cause. + """ + cause = error.__cause__ + return unavailable_after_creation_answer( + "v2_report_run_not_recorded", + cause if isinstance(cause, Exception) else error, + ReportRunNotRecordedResponse( + message=str(error), + report_id=error.report_id, + run_id=error.run_id, + ), + ) + + def _create_report( - service: V2StoredReportResourceService, body: ReportCreateRequest + service: V2StoredReportResourceService, + body: ReportCreateRequest, + requester: AccountPrincipal | None, ) -> ReportDetailResponse | JSONResponse: """Create the report and answer it as stored. - A report the simulation service created answers with its ``report_id`` - when it cannot be read back (502), and when the service failed its run - and answered 503. Every other failure is raised, for the route's error - mappings to answer. + A report the simulation service created answers 502 with its + ``report_id`` and ``run_id`` when it cannot be read back, and 503 with + its ``report_id`` when the service failed its run; one whose request + could not be recorded for its user answers 503 with its ``report_id`` + and ``run_id``. A requester's unauthenticated credential naming a + signed-in user answers 401. Every other failure is raised, for the + route's error mappings to answer. """ try: - report = service.create_report(body) + report = service.create_report(body, requester=requester) + except StandardUserRequiresSignInError as error: + return _requester_refused(error) + except ReportRunNotRecordedError as error: + return _not_recorded(error) except CreatedReportUnreadableError as error: return created_record_unreadable_answer( "v2_created_report_unreadable", @@ -179,10 +264,11 @@ def _create_report( message=( f"The simulation service created report {error.report_id}, " "but it could not be read back. Read it with GET " - f"/v2/reports/{error.report_id}?country_id={error.country_id} " + f"{_read_path(error.report_id, error.country_id, error.run_id)} " "rather than submitting it again." ), report_id=error.report_id, + run_id=error.run_id, ), ) except SimulationServiceUnavailableError as error: @@ -204,18 +290,30 @@ def _rerun_report( country_id: str, report_id: UUID, force: bool, + requester: AccountPrincipal | None, ) -> ReportDetailResponse | JSONResponse: - """Run the report again and answer it with its new run. + """Run the report again and answer it with the run the request ended on. The simulation service's refusal to run it again now answers 409 with - the run it names, and a report that cannot be read back once its new run - started answers 502 with its ``report_id``. Every other failure is + the run it names; a report that cannot be read back once its run started + answers 502 with its ``report_id`` and ``run_id`` (null, pointing to the + report's run list, when a budget window's answer named no window run); + and a run whose request could not be recorded for its user answers 503 + with its ``report_id`` and ``run_id``. A requester's unauthenticated + credential naming a signed-in user answers 401. Every other failure is raised, for the route's error mappings to answer. """ try: report = service.rerun_report( - country_id=country_id, report_id=report_id, force=force + country_id=country_id, + report_id=report_id, + force=force, + requester=requester, ) + except (StandardUserRequiresSignInError, ForcedRerunRequiresSignInError) as error: + return _requester_refused(error) + except ReportRunNotRecordedError as error: + return _not_recorded(error) except SimulationServiceConflictError as error: conflict = ReportRerunConflictResponse(message=str(error), run_id=error.run_id) return JSONResponse(status_code=409, content=conflict.model_dump(mode="json")) @@ -224,20 +322,18 @@ def _rerun_report( "v2_rerun_report_unreadable", error, RerunReportUnreadableResponse( - message=( - "The simulation service started a new run of report " - f"{error.report_id}, but the report could not be read back. " - f"Read it with GET /v2/reports/{error.report_id}?country_id=" - f"{error.country_id} rather than running it again." - ), + message=_rerun_unreadable_message(error), report_id=error.report_id, + run_id=error.run_id, ), ) return ReportDetailResponse(result=ReportDetailResult(item=report_item(report))) def _add_report_run_routes( - router: APIRouter, service_factory: Callable[[], V2StoredReportResourceService] + router: APIRouter, + service_factory: Callable[[], V2StoredReportResourceService], + optional_user: Callable[..., AccountPrincipal | None], ) -> None: """Add the routes that run a report again and list its runs.""" @@ -246,6 +342,7 @@ def _add_report_run_routes( response_model=ReportDetailResponse, status_code=201, responses=RERUN_RESPONSES, + openapi_extra=OPTIONAL_USER_CREDENTIAL, summary="Run a report again", description=( "Checks the report as a new report of its kind is checked (the " @@ -253,19 +350,26 @@ def _add_report_run_routes( "household report, and each policy its simulations name is stored and " "sets only parameters the running catalog version has), then has the " "simulation service create the report's new run on its default " - "worker's PolicyEngine.py version, and start it. Every earlier run " - "is kept. Without force, each side reuses a " - "run of its own simulation that succeeded on those versions; with " - "force, both simulations are calculated again. A household report's " - "sides are always calculated again, so force changes nothing for it. " - "A budget window's new " - "run is a budget-window run with a report run for each year, and " + "worker's PolicyEngine.py version, and start it, or join the run " + "already pending, running or succeeded on those versions, once the " + "simulation service finds runs by key. Every earlier run is kept. " + "Only an admin user may send force (401 without a user credential, " + "403 below admin, before anything is asked). Without force, each " + "side reuses a run of its own simulation that succeeded on those " + "versions; with force, both simulations are calculated again. A " + "household report's sides are always calculated again, so force " + "changes nothing for it. A budget window's new run is a " + "budget-window run with a report run for each year, and " "with force each year calculates its simulations again. Answers the " - "report as stored, whose newest run is then the new one, since no " - "other run " - "starts while it is pending or running: a report with a run pending " - "or running answers 409 naming that run. Read the report again to " - "follow the run." + "report as stored, with the run the simulation service named (a " + "budget window's budget-window run), which needn't be its newest. A " + "simulation service that refuses because a run is pending or " + "running answers 409 naming that run. Read the report again with " + "run_id to follow the run. With a user credential (a bearer access " + "token, or the unauthenticated-user header of a signed-out browser), " + "the request is recorded for that user, naming the run it ended on " + "(trigger manual), or each year's report run of a budget window's " + "run; without one, nothing is recorded." ), ) def rerun_report( @@ -273,6 +377,7 @@ def rerun_report( body: ReportRerunRequest, query: ReportRunsQueryDependency, _size: Annotated[None, Depends(enforce_report_request_size)], + requester: Annotated[AccountPrincipal | None, Depends(optional_user)], ) -> ReportDetailResponse | JSONResponse: def rerun() -> ReportDetailResponse | JSONResponse: return _rerun_report( @@ -280,6 +385,7 @@ def rerun() -> ReportDetailResponse | JSONResponse: country_id=query.country_id, report_id=report_id, force=body.force, + requester=requester, ) return _report_operation(rerun) @@ -316,12 +422,14 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: """Build the report routes without opening a connection.""" router = APIRouter(prefix="/v2", responses=REPORT_ERROR_RESPONSES) service_factory = _service_factory(dependencies) + optional_user = dependencies.optional_user @router.post( "/reports", response_model=ReportDetailResponse, status_code=201, responses=CREATE_RESPONSES, + openapi_extra=OPTIONAL_USER_CREDENTIAL, summary="Create an economy report, a household report or a budget window", description=( "Checks that the country has economy reports and stores each named " @@ -336,45 +444,67 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: "default year; its simulations each calculate that household in its " "default year. A budget window (kind budget_window) is checked as an " "economy report is; the simulation service creates a budget-window " - "run with a report run for each year, and starts its years. " - "Answers the report as stored, with its " - "pending run; read it again to follow the run. A created report " - "that cannot be read back answers 502 with its report_id, and one " - "whose run the simulation service failed before answering 503 " - "answers 503 with its report_id." + "run with a report run for each year, and starts its years. label " + "and description are optional and stored nowhere: a report's name and " + "description are each user's saved report's (/v2/user-reports). " + "Answers the report as stored, with the run the simulation service " + "named: its new pending run, or, once the simulation service finds " + "reports and runs by key, the existing run the request joined, which " + "needn't be the newest. Read it again with run_id to follow the run. " + "With a user credential (a bearer access token, or the " + "unauthenticated-user header of a signed-out browser), the request is " + "recorded for that user, naming the run it ended on (trigger " + "initial), or each year's report run of a budget window's run; " + "without one, nothing is recorded. A created report that cannot be " + "read back answers 502 with its report_id and run_id, and one whose " + "run the simulation service failed before answering 503 answers " + "503 with its report_id." ), ) def create_report( body: ReportCreateRequest, query: ReportCreateQueryDependency, _size: Annotated[None, Depends(enforce_report_request_size)], + requester: Annotated[AccountPrincipal | None, Depends(optional_user)], ) -> ReportDetailResponse | JSONResponse: if body.country_id != query.country_id: return v2_error_response(400, "Body country_id must match query country_id") def create() -> ReportDetailResponse | JSONResponse: - return _create_report(service_factory(), body) + return _create_report(service_factory(), body, requester) return _report_operation(create) @router.get( "/reports", response_model=ReportListResponse, + responses={401: READ_RESPONSES[401], 403: READ_RESPONSES[403]}, summary="List chosen reports", description=( "Lists the country's reports among the report_id values (at most " - "25), in the order named and once each, with their newest runs, and " - "each budget window's years, but without results, read in two " - "queries. Unknown reports and other countries' are left out." + "25), in the order named and once each, each with the run a read of " + "it without run_id answers, and each budget window's years, but " + "without results. With a user credential, each report's label and " + "description are that user's saved report's, and a report the user " + "asked to run shows the run of their newest request; without one, " + "the label is null. Unknown reports and other countries' are left " + "out." ), + openapi_extra=OPTIONAL_USER_CREDENTIAL, ) def list_reports( query: ReportCollectionQueryDependency, + requester: Annotated[AccountPrincipal | None, Depends(optional_user)], ) -> ReportListResponse | JSONResponse: - def read() -> ReportListResponse: - reports = service_factory().list_reports( - country_id=query.country_id, report_ids=query.report_id - ) + def read() -> ReportListResponse | JSONResponse: + try: + reports = service_factory().list_reports( + country_id=query.country_id, + report_ids=query.report_id, + requester=requester, + ) + except StandardUserRequiresSignInError as error: + return _requester_refused(error) return ReportListResponse( result=ReportListResult( items=[report_summary_item(report) for report in reports] @@ -387,11 +517,20 @@ def read() -> ReportListResponse: "/reports/{report_id}", response_model=ReportDetailResponse, responses=READ_RESPONSES, + openapi_extra=OPTIONAL_USER_CREDENTIAL, summary="Read a report", description=( - "Returns the report, its newest run whatever its status and " - "versions, the versions the Public API runs now, and, once the newest " - "run has succeeded, its result. An economy report's run is one report " + "Returns the report as the requesting user sees it: its run named by " + "run_id (404 when it is not one of the report's runs); without " + "run_id, the run of the user's newest request for one, else its " + "newest succeeded run that wasn't forced, else its newest run that " + "wasn't forced, on whatever versions (a forced run is never the " + "default, and a newer run someone else started is listed by GET " + "/v2/reports/{report_id}/runs, not shown in its place); the " + "versions the Public API runs now; and, once that run has succeeded, " + "its result. Its label and description are the requesting user's " + "saved report's name and description, null without a user " + "credential or a saved report. An economy report's run is one report " "run and its result today's economy result. A household report's " "result is each simulation's calculated household, as " "/v2/household-calculations answers one, and a failed run gives each " @@ -405,16 +544,23 @@ def read() -> ReportListResponse: def get_report( report_id: UUID, query: ReportDetailQueryDependency, + requester: Annotated[AccountPrincipal | None, Depends(optional_user)], ) -> ReportDetailResponse | JSONResponse: - def read() -> ReportDetailResponse: - report = service_factory().get_report( - country_id=query.country_id, report_id=report_id - ) + def read() -> ReportDetailResponse | JSONResponse: + try: + report = service_factory().get_report( + country_id=query.country_id, + report_id=report_id, + run_id=query.run_id, + requester=requester, + ) + except StandardUserRequiresSignInError as error: + return _requester_refused(error) return ReportDetailResponse( result=ReportDetailResult(item=report_item(report)) ) return _report_operation(read) - _add_report_run_routes(router, service_factory) + _add_report_run_routes(router, service_factory, optional_user) return router diff --git a/policyengine_api/fastapi_routes/v2/reports/run_models.py b/policyengine_api/fastapi_routes/v2/reports/run_models.py index 2cb9efbde..4680fa289 100644 --- a/policyengine_api/fastapi_routes/v2/reports/run_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/run_models.py @@ -12,9 +12,13 @@ V2MissingParametersErrorResponse, ) from policyengine_api.fastapi_routes.v2.reports.response_models import ( + MACHINE_REQUESTER, + NOT_RECORDED, + UNAUTHENTICATED_REQUESTER, BudgetWindowRunItem, PackageVersionsItem, ReportRunItem, + ReportRunNotRecordedResponse, ReportSuccessResponse, StrictReportAPIModel, ) @@ -129,13 +133,18 @@ class ReportRerunConflictResponse(V2ErrorResponse): class RerunReportUnreadableResponse(V2ErrorResponse): - """The 502 answer when the report cannot be read back after its new run started. - - The simulation service started the run, so the caller reads the report - with ``GET /v2/reports/{report_id}`` rather than running it again. + """The 502 answer when the run a re-run started cannot be answered. + + Either the report cannot be read back with the run ``run_id``, which the + caller then reads with ``GET /v2/reports/{report_id}?run_id={run_id}``, + or a budget window's answer named no budget-window run, when ``run_id`` + is null and the caller finds the run in ``GET + /v2/reports/{report_id}/runs``. Either way the run exists, so the caller + doesn't run the report again. """ report_id: UUID + run_id: UUID | None NOT_FOUND = "No report has this ID in this country." @@ -143,6 +152,14 @@ class RerunReportUnreadableResponse(V2ErrorResponse): 404: error_response(NOT_FOUND), } RERUN_RESPONSES: dict[int | str, dict[str, Any]] = { + 401: error_response( + f"{UNAUTHENTICATED_REQUESTER} Or the request forces a re-run without a " + "user credential: only an admin user may force one." + ), + 403: error_response( + f"{MACHINE_REQUESTER} Or the request forces a re-run, and its user is " + "not an admin: only an admin user may force one." + ), 404: error_response( f"{NOT_FOUND} Or a policy a simulation of the report names is no longer " "stored in the country." @@ -181,19 +198,26 @@ class RerunReportUnreadableResponse(V2ErrorResponse): "see whether it has a new run before running it again; while that " "run is pending or running, running it again answers 409. Or it " "started the run, but the report cannot be read back: the answer " - "gives its report_id." + "gives its report_id and run_id, to read with GET " + "/v2/reports/{report_id}?run_id={run_id}. Or it started a budget " + "window's run, but its answer named no budget-window run: run_id " + "is null, and the message points to GET " + "/v2/reports/{report_id}/runs." + ), + }, + 503: { + "model": V2ErrorResponse | ReportRunNotRecordedResponse, + "description": ( + "The simulation service started nothing that runs: it cannot be reached " + "or is not configured; it answered a readable 503, and the message gives " + "its status and code, and names the new run it failed when Modal refused " + "the run's call; it answered a 400 about its own state rather than a row " + "the report names (such as a catalog version missing while a release " + "deploys), and the message gives its code and reason; it answered 400, " + "409 or 422 in a shape the Public API cannot read; or it answered 401, " + "403, 404, 405, 413 or 429, which it sends before writing anything. Or " + "the v2 database, or the catalog the policies are checked against, " + f"cannot be read. {NOT_RECORDED}" ), }, - 503: error_response( - "The simulation service started nothing that runs: it cannot be reached " - "or is not configured; it answered a readable 503, and the message gives " - "its status and code, and names the new run it failed when Modal refused " - "the run's call; it answered a 400 about its own state rather than a row " - "the report names (such as a catalog version missing while a release " - "deploys), and the message gives its code and reason; it answered 400, " - "409 or 422 in a shape the Public API cannot read; or it answered 401, " - "403, 404, 405, 413 or 429, which it sends before writing anything. Or " - "the v2 database, or the catalog the policies are checked against, " - "cannot be read." - ), } diff --git a/policyengine_api/fastapi_routes/v2/user_reports/routes.py b/policyengine_api/fastapi_routes/v2/user_reports/routes.py index b11374bb6..c99bdc0c7 100644 --- a/policyengine_api/fastapi_routes/v2/user_reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_reports/routes.py @@ -145,7 +145,10 @@ def build_v2_user_report_router( "budget window as budget_window_years, or null for a report without " "a window of two or more years; otherwise it is refused with 400. An " "optional user_policy_id must name the same user's saved policy in " - "the same country." + "the same country. A saved report naming a detailed-save report " + "answers last_run_at as the owner's newest request for one of that " + "report's runs through /v2/reports, or null, whatever the request " + "sends; one naming only a v1 report output keeps the value sent." ), ) def create_user_report( @@ -203,6 +206,12 @@ def read() -> UserReportPageResponse: response_model=UserReportDetailResponse, responses=SAVED_REPORT_WRITE_RESPONSES, summary="Update or repoint one of the current user's saved reports", + description=( + "Updates the fields sent. A saved report that names a detailed-save " + "report after the update, and whose report_id or last_run_at the " + "request sends, answers last_run_at as the owner's newest request for " + "one of that report's runs through /v2/reports, or null." + ), ) def patch_user_report_route( association_id: UUID, diff --git a/policyengine_api/libs/simulation_report_reruns.py b/policyengine_api/libs/simulation_report_reruns.py index 75d4204fb..8f8d2e636 100644 --- a/policyengine_api/libs/simulation_report_reruns.py +++ b/policyengine_api/libs/simulation_report_reruns.py @@ -11,9 +11,11 @@ The Public API reads the report in its country before asking, so it never asks about a report it does not have. The simulation service answers: -- 201 or 202 naming the report: the new run exists. One whose body cannot - be read, or that names another report, is - SimulationServiceAnswerUnreadableError; +- 201 or 202 naming the report and the run the request ended on + (``report_run_id``, and a budget window's ``budget_window_run_id``): a + new run, or, once the service finds runs by key, an existing one the + request joined. One whose body cannot be read, that names no run, or that + names another report, is SimulationServiceAnswerUnreadableError; - 409 with ``code`` and ``detail`` when it will not run the report again now: ``run_in_progress``, naming the run pending or running in ``run_id``, or ``rerun_not_supported`` for a report it does not run again. @@ -83,6 +85,7 @@ SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, + StartedReportRun, ) if TYPE_CHECKING: @@ -105,6 +108,8 @@ class _RerunStarted(AnswerModel): report_id: UUID + report_run_id: UUID + budget_window_run_id: UUID | None = None class _RerunRefusal(Refusal): @@ -256,14 +261,22 @@ def _raise_for_refusal(answer: SimulationAnswer, report_id: UUID) -> NoReturn: raise SimulationServiceAnswerUnreadableError(_possibly_started(answer)) -def _require_started(answer: SimulationAnswer, report_id: UUID) -> None: - """Raise unless the started answer can be read and names the report.""" +def _require_started(answer: SimulationAnswer, report_id: UUID) -> StartedReportRun: + """Return the run the started answer names, once it names the report. + + Raise SimulationServiceAnswerUnreadableError for an answer that cannot + be read, names no run, or names another report. + """ try: started = _RerunStarted.model_validate_json(answer.content) except ValidationError as error: raise SimulationServiceAnswerUnreadableError(UNREADABLE_START) from error if started.report_id != report_id: raise SimulationServiceAnswerUnreadableError(UNREADABLE_START) + return StartedReportRun( + report_run_id=started.report_run_id, + budget_window_run_id=started.budget_window_run_id, + ) def start_report_rerun( @@ -271,13 +284,13 @@ def start_report_rerun( rerun: ReportRerunInput, *, send: Callable[[httpx.Client, UUID, dict[str, Any]], SimulationAnswer], -) -> None: - """Have the simulation service create and start the report's new run. +) -> StartedReportRun: + """Have the simulation service start the report's run; return the run it names. - Send through ``send`` on ``http_client``, and raise the errors the module - docstring lists for the service's answers. A connection never made, or a - gateway token that cannot be fetched, is - SimulationServiceUnavailableError. + The run is a new one, or one the request joined. Send through ``send`` + on ``http_client``, and raise the errors the module docstring lists for + the service's answers. A connection never made, or a gateway token that + cannot be fetched, is SimulationServiceUnavailableError. """ answer = send_to_simulation_service( lambda client, body: send(client, rerun.report_id, body), @@ -286,6 +299,5 @@ def start_report_rerun( submit_again_sentence=READ_BEFORE_RUNNING_AGAIN, ) if answer.status_code in CREATED_STATUSES: - _require_started(answer, rerun.report_id) - return + return _require_started(answer, rerun.report_id) _raise_for_refusal(answer, rerun.report_id) diff --git a/policyengine_api/libs/simulation_reports.py b/policyengine_api/libs/simulation_reports.py index e7a8609ff..ef3a8d0f2 100644 --- a/policyengine_api/libs/simulation_reports.py +++ b/policyengine_api/libs/simulation_reports.py @@ -12,18 +12,19 @@ A simulation request names the country, ``simulation_type`` (``economy``), the year, the region code, the dataset name (null for the region's default), and the stored policy and dynamic (null for current law). A report request -names the country, ``report_type`` (``economy``), the label and description, -the year, region and dataset its two simulations share, each simulation's +names the country, ``report_type`` (``economy``), the year, region and +dataset its two simulations share, each simulation's policy and dynamic, ``include_cliffs``, and the SPM settings only when the caller gave them. A budget window goes to the same ``POST /reports`` as ``report_type`` ``budget_window``, naming its first year and ``budget_window_size`` in place of the year, and no ``include_cliffs``. A household report goes there too, as ``report_type`` ``household``, naming -the label and description, the stored household, each simulation's policy +the stored household, each simulation's policy (the reform null for a report of the baseline alone), but no year: the simulation service calculates the household's default year, adding every variable the household lacks. None names a PolicyEngine.py version: the -simulation service runs its default worker's. The simulation service +simulation service runs its default worker's. None names a label or a +description, which belong to users' saved reports. The simulation service answers: - 201 or 202 with the created record's ID and its first run's ID: for a @@ -159,6 +160,7 @@ HouseholdReportCreationInput, PolicyChoice, ReportRerunInput, + StartedReportRun, ) SIMULATIONS_PATH = "/simulations" @@ -232,13 +234,13 @@ def _report_definition( ) -> dict[str, Any]: """Return the request fields economy reports and budget windows both send. - ``spm`` is sent only when the caller gave it. + ``spm`` is sent only when the caller gave it. A report's label and + description are never sent: they belong to users' saved reports, and + the simulation service stores neither. """ body: dict[str, Any] = { "country": report.country_id, "report_type": report.kind, - "label": report.label, - "description": report.description, "year": report.year, "region_code": report.region_code, "dataset_name": report.dataset_name, @@ -282,8 +284,6 @@ def household_report_request(report: HouseholdReportCreationInput) -> dict[str, return { "country": report.country_id, "report_type": report.kind, - "label": report.label, - "description": report.description, "household_id": str(report.household_id), "baseline": _household_policy_choice(report.baseline), "reform": ( @@ -605,13 +605,15 @@ def submit_budget_window( budget_window_run_id=created.budget_window_run_id, ) - def rerun_report(self, rerun: ReportRerunInput) -> None: - """Have the service create and start the stored report's new run. + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: + """Have the service start the stored report's run; return the run it names. Raise the errors ``policyengine_api.libs.simulation_report_reruns`` lists for the service's answers. """ - start_report_rerun(self._http_client, rerun, send=self._send_report_rerun) + return start_report_rerun( + self._http_client, rerun, send=self._send_report_rerun + ) @lru_cache(maxsize=1) diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 568e148c8..cc3a5f205 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -46,7 +46,8 @@ from collections.abc import Iterable, Mapping DEFAULT_QUERY_LIMIT = 100 -# A report list reads every named report and its newest run in one query. +# A report list reads every named report and the run its read answers in a few +# queries. MAXIMUM_LISTED_REPORTS = 25 MAXIMUM_QUERY_LIMIT = 500 MAXIMUM_USER_ID_LENGTH = 255 @@ -430,7 +431,23 @@ class ReportCreateQuery(CountryQuery): class ReportDetailQuery(CountryQuery): - """Query contract for reading one report in a country.""" + """Query contract for reading one report in a country. + + ``run_id`` names the run to read: a report run of an economy or household + report, or a budget-window run of a budget window. Without it the report + is read with the requesting user's newest recorded run, else its newest + succeeded run that wasn't forced, else its newest run that wasn't forced. + """ + + run_id: ResourceId | None = Field( + default=None, + description=( + "The report's run to read: one of its report runs, or a budget " + "window's budget-window run. Without it, the requesting user's " + "newest recorded run, else the newest succeeded run that wasn't " + "forced, else the newest run that wasn't forced." + ), + ) class ReportRunsQuery(CountryQuery): diff --git a/policyengine_api/services/v2/identity/services.py b/policyengine_api/services/v2/identity/services.py index 83c7998d4..8690f3a30 100644 --- a/policyengine_api/services/v2/identity/services.py +++ b/policyengine_api/services/v2/identity/services.py @@ -76,6 +76,15 @@ require_provisioned_user, require_unauthenticated_user, ) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + creates as record_creates, +) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + reads as record_reads, +) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + updates as record_updates, +) if TYPE_CHECKING: from datetime import datetime @@ -301,6 +310,41 @@ def resolve_principal_user_for_read_in_session( ) +def _move_report_run_records( + session: Session, + *, + from_user_id: UUID, + to_user_id: UUID, + moved_saved_report_ids: set[UUID], +) -> None: + """Copy one user's report-run records to another before the first is deleted. + + A run the other user already has a record of keeps that record. Then + each of the other user's saved reports of a report whose saved report + moved in (``moved_saved_report_ids``) or whose record was copied takes + the other user's newest request for one of that report's runs as its + ``last_run_at``, or null without one, whatever it held before. + """ + copied_run_ids = record_creates.create_report_run_records( + session, + user_id=to_user_id, + records=record_reads.read_report_run_records(session, user_id=from_user_id), + ) + report_ids = moved_saved_report_ids | record_reads.read_report_ids_of_runs( + session, report_run_ids=copied_run_ids + ) + # In a set order, so two merges into one account never wait on each other. + for report_id in sorted(report_ids): + record_updates.update_saved_reports_last_run_at( + session, + user_id=to_user_id, + report_id=report_id, + last_run_at=record_reads.read_newest_report_request( + session, user_id=to_user_id, report_id=report_id + ), + ) + + def convert_unauthenticated_user_in_session( session: Session, *, @@ -319,8 +363,11 @@ def convert_unauthenticated_user_in_session( - No user with the identity yet: the unauthenticated user becomes it in place, keeping its ID and therefore every association it owns. - The identity already has a user: every association moves to that user - unchanged, the unauthenticated user is deleted, and the identity's user - keeps its stored type. + unchanged, the unauthenticated user's report-run records are copied to + it (a run it already has a record of keeps that record), the saved + reports involved take its newest request for their report as + ``last_run_at``, the unauthenticated user is deleted, and the + identity's user keeps its stored type. - No unauthenticated user, or the ID names a signed-in user (one already converted, or anyone else's): nothing is touched, and the identity's user is resolved as on any request. @@ -372,10 +419,20 @@ def convert_unauthenticated_user_in_session( owner = standard_provisioning.resolve_in_session( session, country_id=country_id, now=seen_at ) + moved_saved_report_ids = record_reads.read_saved_report_ids( + session, user_id=unauthenticated_user.id + ) reassign_user_associations( session, from_user_id=unauthenticated_user.id, to_user_id=owner.user_id, ) + # The deletion below takes the browser's own records with it (CASCADE). + _move_report_run_records( + session, + from_user_id=unauthenticated_user.id, + to_user_id=owner.user_id, + moved_saved_report_ids=moved_saved_report_ids, + ) delete_unauthenticated_user(session, user_id=unauthenticated_user.id) return AccountConversion(owner=owner, outcome=AccountConversionOutcome.MERGED) diff --git a/policyengine_api/services/v2/report_run_records/__init__.py b/policyengine_api/services/v2/report_run_records/__init__.py new file mode 100644 index 000000000..649f6d875 --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/__init__.py @@ -0,0 +1 @@ +"""Each user's requests for v2 report runs, which the report routes record.""" diff --git a/policyengine_api/services/v2/report_run_records/database_connectors/__init__.py b/policyengine_api/services/v2/report_run_records/database_connectors/__init__.py new file mode 100644 index 000000000..0df0f39bc --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-only connector functions for users' report-run records.""" diff --git a/policyengine_api/services/v2/report_run_records/database_connectors/creates.py b/policyengine_api/services/v2/report_run_records/database_connectors/creates.py new file mode 100644 index 000000000..10dbcc009 --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/database_connectors/creates.py @@ -0,0 +1,53 @@ +"""Database inserts of users' report-run records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import col + +from policyengine_api.data.v2.models import UserReportRunAssociation + +if TYPE_CHECKING: + from collections.abc import Sequence + from uuid import UUID + + from sqlmodel import Session + + from policyengine_api.services.v2.report_run_records.types import ( + ReportRunRecordRead, + ) + + +def create_report_run_records( + session: Session, *, user_id: UUID, records: Sequence[ReportRunRecordRead] +) -> list[UUID]: + """Record each request for the user whose run they have no record of yet. + + Each row takes its record's run, ``requested_at`` and ``trigger``, with a + new ID. A run the user already has a record of keeps that record (``ON + CONFLICT (user_id, report_run_id) DO NOTHING``). Return the runs this + call recorded. + """ + if not records: + return [] + statement = ( + insert(UserReportRunAssociation) + .values( + [ + { + "id": uuid4(), + "user_id": user_id, + "report_run_id": record.report_run_id, + "requested_at": record.requested_at, + "trigger": record.trigger, + } + for record in records + ] + ) + .on_conflict_do_nothing(index_elements=["user_id", "report_run_id"]) + .returning(col(UserReportRunAssociation.report_run_id)) + ) + return list(session.scalars(statement).all()) diff --git a/policyengine_api/services/v2/report_run_records/database_connectors/reads.py b/policyengine_api/services/v2/report_run_records/database_connectors/reads.py new file mode 100644 index 000000000..e74f3048f --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/database_connectors/reads.py @@ -0,0 +1,88 @@ +"""Database reads of users' report-run records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import func, select +from sqlmodel import col + +from policyengine_api.data.v2.models import ( + ReportRun, + UserReportAssociation, + UserReportRunAssociation, +) +from policyengine_api.services.v2.report_run_records.types import ReportRunRecordRead + +if TYPE_CHECKING: + from collections.abc import Collection + from datetime import datetime + from uuid import UUID + + from sqlmodel import Session + + +def read_report_ids_of_runs( + session: Session, *, report_run_ids: Collection[UUID] +) -> set[UUID]: + """Return the reports these report runs belong to.""" + if not report_run_ids: + return set() + statement = select(col(ReportRun.report_id)).where( + col(ReportRun.id).in_(report_run_ids) + ) + return set(session.scalars(statement).all()) + + +def read_saved_report_ids(session: Session, *, user_id: UUID) -> set[UUID]: + """Return the detailed-save reports the user's saved reports name.""" + statement = select(col(UserReportAssociation.report_id)).where( + col(UserReportAssociation.user_id) == user_id, + col(UserReportAssociation.report_id).is_not(None), + ) + return {report_id for report_id in session.scalars(statement) if report_id} + + +def read_report_run_records( + session: Session, *, user_id: UUID +) -> list[ReportRunRecordRead]: + """Return every report-run record of the user, in no set order.""" + statement = select( + col(UserReportRunAssociation.report_run_id), + col(UserReportRunAssociation.requested_at), + col(UserReportRunAssociation.trigger), + ).where(col(UserReportRunAssociation.user_id) == user_id) + return [ + ReportRunRecordRead( + report_run_id=report_run_id, requested_at=requested_at, trigger=trigger + ) + for report_run_id, requested_at, trigger in session.execute(statement).all() + ] + + +def read_newest_report_request( + session: Session, *, user_id: UUID, report_id: UUID +) -> datetime | None: + """Return the user's newest request time among the report's runs, or None.""" + statement = ( + select(func.max(col(UserReportRunAssociation.requested_at))) + .join(ReportRun, col(ReportRun.id) == UserReportRunAssociation.report_run_id) + .where( + col(UserReportRunAssociation.user_id) == user_id, + col(ReportRun.report_id) == report_id, + ) + ) + newest_request: datetime | None = session.execute(statement).scalar_one() + return newest_request + + +def read_budget_window_year_run_ids( + session: Session, *, budget_window_run_id: UUID +) -> list[UUID]: + """Return the report run of each year of the budget-window run, in year order.""" + statement = ( + select(col(ReportRun.id)) + .where(col(ReportRun.budget_window_run_id) == budget_window_run_id) + .order_by(col(ReportRun.year), col(ReportRun.id)) + ) + return list(session.scalars(statement).all()) diff --git a/policyengine_api/services/v2/report_run_records/database_connectors/updates.py b/policyengine_api/services/v2/report_run_records/database_connectors/updates.py new file mode 100644 index 000000000..acb8d1673 --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/database_connectors/updates.py @@ -0,0 +1,69 @@ +"""Database updates of saved reports' last run times from users' requests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, update +from sqlmodel import col + +from policyengine_api.data.v2.models import UserReportAssociation + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from sqlmodel import Session + + +def advance_saved_reports_last_run_at( + session: Session, *, user_id: UUID, report_id: UUID, requested_at: datetime +) -> None: + """Set the user's saved reports of the report to ``requested_at`` when later. + + A saved report whose ``last_run_at`` is already later, as one written by + a concurrent newer request, keeps it, so the time never moves back; one + with none takes ``requested_at``. ``updated_at`` is unchanged, since the + user changed nothing. + """ + session.execute( + update(UserReportAssociation) + .where( + col(UserReportAssociation.user_id) == user_id, + col(UserReportAssociation.report_id) == report_id, + or_( + col(UserReportAssociation.last_run_at).is_(None), + col(UserReportAssociation.last_run_at) < requested_at, + ), + ) + .values( + last_run_at=requested_at, + updated_at=col(UserReportAssociation.updated_at), + ) + .execution_options(synchronize_session=False) + ) + + +def update_saved_reports_last_run_at( + session: Session, + *, + user_id: UUID, + report_id: UUID, + last_run_at: datetime | None, +) -> None: + """Set the user's saved reports of the report to ``last_run_at``, even earlier. + + ``updated_at`` is unchanged, since the user changed nothing. + """ + session.execute( + update(UserReportAssociation) + .where( + col(UserReportAssociation.user_id) == user_id, + col(UserReportAssociation.report_id) == report_id, + ) + .values( + last_run_at=last_run_at, + updated_at=col(UserReportAssociation.updated_at), + ) + .execution_options(synchronize_session=False) + ) diff --git a/policyengine_api/services/v2/report_run_records/database_session.py b/policyengine_api/services/v2/report_run_records/database_session.py new file mode 100644 index 000000000..1c2efd42d --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/database_session.py @@ -0,0 +1,32 @@ +"""Database session lifetime management for users' report-run records.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class ReportRunRecordDatabaseSession: + """Open read sessions and transactions for the report-run record service.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + """Yield a session whose transaction commits on exit and rolls back on error.""" + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/report_run_records/services.py b/policyengine_api/services/v2/report_run_records/services.py new file mode 100644 index 000000000..7381b1086 --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/services.py @@ -0,0 +1,152 @@ +"""Record each user's requests for v2 report runs. + +The report routes record a request once the simulation service has answered +which run it ended on, created or joined: the requesting user's +``user_report_run_associations`` row for that run, written in one +transaction with the user's provisioning (a signed-in identity's user, or a +signed-out browser's unauthenticated user, as any user write provisions +it) and with the ``last_run_at`` of the user's saved reports naming the +report. Before the simulation service is asked, the requester is checked, +so a browser credential that names a signed-in user is refused before +anything is created, and the user it has then is the one the record names: +a browser whose user signing in merged into an account while the simulation +service was asked is not recorded under a new user re-created with its ID, +and one converted in place meanwhile is recorded as the account it became. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.services.v2.identity.database_connectors.reads import ( + read_user, +) +from policyengine_api.services.v2.identity.services import ( + find_principal_user_in_session, + resolve_principal_user_in_session, +) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + creates as record_creates, +) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + reads as record_reads, +) +from policyengine_api.services.v2.report_run_records.database_connectors import ( + updates as record_updates, +) +from policyengine_api.services.v2.report_run_records.types import ( + CheckedRequester, + ReportRunRecordRead, +) +from policyengine_api.services.v2.report_run_records.validators import ( + ReportRunRecordStorageError, + require_requester_user, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + from uuid import UUID + + from policyengine_api.services.v2.identity.types import AccountPrincipal + from policyengine_api.services.v2.report_run_records.database_session import ( + ReportRunRecordDatabaseSession, + ) + from policyengine_api.services.v2.report_run_records.types import ( + ReportRunRequest, + ) + + +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise ReportRunRecordStorageError for a failed database read or write.""" + try: + yield + except SQLAlchemyError as error: + msg = "V2 report storage is unavailable" + raise ReportRunRecordStorageError(msg) from error + + +class ReportRunRecordService: + """Record the report runs users ask for.""" + + def __init__(self, database_session: ReportRunRecordDatabaseSession) -> None: + """Read and write through ``database_session``.""" + self._database_session = database_session + + def check_requester(self, principal: AccountPrincipal) -> CheckedRequester: + """Check a principal before anything starts, and name its user if it has one. + + Nothing is written. Raise StandardUserRequiresSignInError for an + unauthenticated credential that names a signed-in user, and + ReportRunRecordStorageError when the database cannot be read. + """ + with _storage_failures(), self._database_session.read() as session: + found = find_principal_user_in_session(session, principal) + if found is None: + return CheckedRequester(principal=principal, user_id=None) + return CheckedRequester( + principal=principal, user_id=found.user_id, user_type=found.user_type + ) + + def record(self, requester: CheckedRequester, request: ReportRunRequest) -> UUID: + """Record the request for each run it ended on; return the user it names. + + It is written in one transaction. + A budget window's request records each year's report run of its + budget-window run, read in the same transaction. The record names the + user the check found, held ``FOR SHARE`` so a + sign-in merging or converting it waits, and kept as it is when it + was converted in place meanwhile; with none found, the principal's + user is provisioned as any user write provisions it. A run the user + already has a record of keeps it. When this request recorded a run, + the user's saved reports naming the report take its ``requested_at`` + as their ``last_run_at`` if it is later. Raise RequesterUserGoneError + when the user the check found no longer exists (merged into an + account, or its account deleted), StandardUserRequiresSignInError + when an unauthenticated credential with no user then names a + signed-in one, and ReportRunRecordStorageError when the database + cannot be written. + """ + with _storage_failures(), self._database_session.transaction() as session: + if requester.user_id is not None: + user_id = require_requester_user( + requester.user_id, + exists=read_user(session, requester.user_id, for_share=True) + is not None, + ) + else: + user_id = resolve_principal_user_in_session( + session, + requester.principal, + country_id=request.country_id, + now=request.requested_at, + ).user_id + recorded = record_creates.create_report_run_records( + session, + user_id=user_id, + records=[ + ReportRunRecordRead( + report_run_id=report_run_id, + requested_at=request.requested_at, + trigger=request.trigger, + ) + for report_run_id in ( + (request.report_run_id,) + if request.report_run_id is not None + else record_reads.read_budget_window_year_run_ids( + session, budget_window_run_id=request.run_id + ) + ) + ], + ) + if recorded: + record_updates.advance_saved_reports_last_run_at( + session, + user_id=user_id, + report_id=request.report_id, + requested_at=request.requested_at, + ) + return user_id diff --git a/policyengine_api/services/v2/report_run_records/types.py b/policyengine_api/services/v2/report_run_records/types.py new file mode 100644 index 000000000..c8d2d74d9 --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/types.py @@ -0,0 +1,103 @@ +"""Data the report-run record service and its callers exchange. + +A user's request for a report run is recorded once per user and run +(``user_report_run_associations``): a user asking again for a run they have +keeps their first record, and its ``requested_at``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from policyengine_api.data.v2.models import ReportRunTrigger, UserType + from policyengine_api.services.v2.identity.types import AccountPrincipal + + +@dataclass(frozen=True) +class ReportRunRequest: + """One user's request for a run of a report: what to record. + + The request ended on, created or joined, either an economy or household + report's report run (``report_run_id``) or a budget window's + budget-window run (``budget_window_run_id``), whose every year's report + run is recorded; exactly one is set. ``requested_at`` is when the Public + API received the request, the time the user sees; ``trigger`` is + ``initial`` for a create and ``manual`` for a re-run. ``country_id`` is + the report's country, the primary country of a signed-in user + provisioned by this request. + """ + + country_id: str + report_id: UUID + requested_at: datetime + trigger: ReportRunTrigger + report_run_id: UUID | None = None + budget_window_run_id: UUID | None = None + + def __post_init__(self) -> None: + """Require exactly one of the report run and the budget-window run.""" + if (self.report_run_id is None) == (self.budget_window_run_id is None): + msg = "A request names a report run or a budget-window run, not both" + raise ValueError(msg) + + @property + def run_id(self) -> UUID: + """The run a read of the report names to answer this request's run.""" + run_id = self.report_run_id or self.budget_window_run_id + if run_id is None: # pragma: no cover # __post_init__ requires one + msg = "A request names a run" + raise ValueError(msg) + return run_id + + +@dataclass(frozen=True) +class CheckedRequester: + """A requester checked before the simulation service is asked. + + ``user_id`` and ``user_type`` are those of the user the principal had + then, or None when it had none yet (a signed-out browser's first write, + or a signed-in identity's first request), in which case the record + provisions one. + """ + + principal: AccountPrincipal + user_id: UUID | None + user_type: UserType | None = None + + +@dataclass(frozen=True) +class ReportRunRecordRead: + """One stored record of a user's request: the run, when, and how.""" + + report_run_id: UUID + requested_at: datetime + trigger: ReportRunTrigger + + +class ReportRunRecorder(Protocol): + """Record the report runs users ask for.""" + + def check_requester(self, principal: AccountPrincipal) -> CheckedRequester: + """Check a principal before anything starts, and name its user if it has one. + + Raise StandardUserRequiresSignInError for an unauthenticated + credential that names a signed-in user, and ReportRunRecordStorageError + when the database cannot be read. + """ + ... + + def record(self, requester: CheckedRequester, request: ReportRunRequest) -> UUID: + """Record the request for the user the check found, or a new one if none. + + Return the ID of the user the request was recorded for. + + The user's saved reports naming the report get the request's time as + their ``last_run_at`` when it is newer. Raise RequesterUserGoneError + when the user the check found no longer exists. + """ + ... diff --git a/policyengine_api/services/v2/report_run_records/validators.py b/policyengine_api/services/v2/report_run_records/validators.py new file mode 100644 index 000000000..a8c9825fe --- /dev/null +++ b/policyengine_api/services/v2/report_run_records/validators.py @@ -0,0 +1,30 @@ +"""Errors of the report-run record service.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from uuid import UUID + + +class ReportRunRecordStorageError(RuntimeError): + """The detailed-save database could not be read or written for a record.""" + + +class RequesterUserGoneError(LookupError): + """The user found before the simulation service was asked no longer exists. + + A signed-out browser's user is deleted when signing in merges it into an + existing account, and any user by its own account deletion. Its UUID now + names nobody, so the request is not recorded rather than recorded for a + user re-created with that ID. + """ + + +def require_requester_user(user_id: UUID, *, exists: bool) -> UUID: + """Return the user's ID, or raise RequesterUserGoneError once it is gone.""" + if not exists: + msg = "The requesting user no longer exists, so the request was not recorded" + raise RequesterUserGoneError(msg) + return user_id diff --git a/policyengine_api/services/v2/simulation_service/services.py b/policyengine_api/services/v2/simulation_service/services.py index 74f1bc133..908d80134 100644 --- a/policyengine_api/services/v2/simulation_service/services.py +++ b/policyengine_api/services/v2/simulation_service/services.py @@ -65,6 +65,7 @@ ReportRerunInput, ReportRerunKind, SimulationServiceClient, + StartedReportRun, SubmittedBudgetWindow, SubmittedReport, SubmittedSimulation, @@ -221,8 +222,11 @@ def submit_budget_window( ) return self._client_factory().submit_budget_window(report) - def rerun_report(self, rerun: ReportRerunInput) -> None: - """Check the stored report, then have the simulation service run it again. + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: + """Check the stored report, have the simulation service run it again. + + Return the run the simulation service named: a new one, or one the + request joined. The country is checked as for a new report of the report's kind, then the policies of the report's simulations, the baseline's first. A @@ -238,7 +242,7 @@ def rerun_report(self, rerun: ReportRerunInput) -> None: rerun.policy_ids, require_country=_country_check(rerun.kind), ) - self._client_factory().rerun_report(rerun) + return self._client_factory().rerun_report(rerun) def _country_check(kind: ReportRerunKind) -> CountryCheck: diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index e52999829..80bad5ebf 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -49,6 +49,11 @@ CatalogKey = Annotated[str, Field(min_length=1, max_length=255, pattern=r"^\S+$")] ReportLabel = Annotated[str, Field(min_length=1, max_length=255)] ReportDescription = Annotated[str, Field(max_length=10_000)] +# A create's label and description are checked when sent, then dropped. +UNSTORED_FIELD_DESCRIPTION = ( + "Optional, and stored nowhere: a report's label and description are the " + "name and description of each user's saved report (/v2/user-reports)." +) WindowFirstYear = Annotated[ int, Field(ge=1900, le=LAST_YEAR, description="The window's first year") ] @@ -203,7 +208,10 @@ def _omit_spm_default(schema: dict[str, Any]) -> None: class ReportDefinitionFields(StrictSubmissionModel): - """What every report names: a label, and a baseline and a reform simulation. + """What every report names: a baseline and a reform simulation. + + ``label`` and ``description`` are optional and stored nowhere (see + ``UNSTORED_FIELD_DESCRIPTION``). Both simulations use the report's year (a budget window's first year; each later year gets simulations of its own), region and dataset, as @@ -214,8 +222,12 @@ class ReportDefinitionFields(StrictSubmissionModel): """ kind: Literal["economy", "budget_window"] - label: ReportLabel - description: ReportDescription | None = None + label: ReportLabel | None = Field( + default=None, description=UNSTORED_FIELD_DESCRIPTION + ) + description: ReportDescription | None = Field( + default=None, description=UNSTORED_FIELD_DESCRIPTION + ) year: SimulationYear region_code: CatalogKey dataset_name: CatalogKey | None = None @@ -269,12 +281,18 @@ class HouseholdReportFields(StrictSubmissionModel): simulation calculates every yearly, monthly and eternal variable the household lacks, as ``/calculate-full`` does, so the report gives every value; there is no option to fill in only the values the household asks - for, as ``/v2/household-calculations`` has. + for, as ``/v2/household-calculations`` has. ``label`` and + ``description`` are optional and stored nowhere, as for an economy + report. """ kind: Literal["household"] - label: ReportLabel - description: ReportDescription | None = None + label: ReportLabel | None = Field( + default=None, description=UNSTORED_FIELD_DESCRIPTION + ) + description: ReportDescription | None = Field( + default=None, description=UNSTORED_FIELD_DESCRIPTION + ) household_id: ResourceId baseline: HouseholdPolicyChoice reform: HouseholdPolicyChoice | None = None @@ -351,6 +369,19 @@ class SubmittedBudgetWindow: budget_window_run_id: UUID +@dataclass(frozen=True) +class StartedReportRun: + """The run a re-run ended on, whether the service created it or joined it. + + ``report_run_id`` is an economy or household report's report run, or a + budget window's first year's; ``budget_window_run_id`` is a budget + window's budget-window run, None for any other report. + """ + + report_run_id: UUID + budget_window_run_id: UUID | None = None + + # Each kind of stored report a re-run names, as the report's row records it. ReportRerunKind = Literal["economy", "household", "budget_window"] @@ -445,8 +476,12 @@ def submit_budget_window( """ ... - def rerun_report(self, rerun: ReportRerunInput) -> None: - """Create the stored report's new run on the current versions, and start it.""" + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: + """Start the stored report's run on the current versions; return that run. + + The run is a new one, or one already pending, running or succeeded on + those versions that the request joined. + """ ... diff --git a/policyengine_api/services/v2/stored_reports/database_connectors/reads.py b/policyengine_api/services/v2/stored_reports/database_connectors/reads.py index 90a20910a..51929f32c 100644 --- a/policyengine_api/services/v2/stored_reports/database_connectors/reads.py +++ b/policyengine_api/services/v2/stored_reports/database_connectors/reads.py @@ -1,15 +1,17 @@ -"""Database reads of v2 reports, their newest runs and budget-window years. +"""Database reads of v2 reports, their default runs and budget-window years. A report's own runs are report runs that are not a year of a budget window; a budget-window report's are budget-window runs, each with a report run per -year. +year. A report's default run, the one a read answers without naming one, is +its newest succeeded run that wasn't forced (a window's: not recalculated), +else its newest such run of any status; a forced run is never the default. """ from __future__ import annotations from typing import TYPE_CHECKING -from sqlalchemy import and_, func, select +from sqlalchemy import and_, case, func, select from sqlalchemy.orm import aliased from sqlmodel import col @@ -19,6 +21,7 @@ Region, Report, ReportRun, + ReportRunStatus, Simulation, ) from policyengine_api.services.v2.stored_reports.types import StoredReportRows @@ -31,18 +34,26 @@ from sqlmodel import Session -def _newest_first(run_model: type[ReportRun | BudgetWindowRun]) -> Label[int]: - """Return a label numbering each report's runs from 1, newest first. +def _default_first(run_model: type[ReportRun | BudgetWindowRun]) -> Label[int]: + """Return a label numbering each report's runs from 1, its default run first. - The newest is the run created last, ties going to the larger ID. + Succeeded runs come before the others, then the newest: the run created + last, ties going to the larger ID. The caller leaves forced runs out. """ + not_succeeded = case( + (col(run_model.status) == ReportRunStatus.SUCCEEDED, 0), else_=1 + ) return ( func.row_number() .over( partition_by=col(run_model.report_id), - order_by=(col(run_model.created_at).desc(), col(run_model.id).desc()), + order_by=( + not_succeeded, + col(run_model.created_at).desc(), + col(run_model.id).desc(), + ), ) - .label("newest_first") + .label("default_first") ) @@ -52,30 +63,35 @@ def read_report_rows( """Return the country's reports among ``report_ids``, in two queries. Each comes with its region's code, its dataset's name, its two simulations - and its newest run: the run created last (ties broken by the larger ID), - whatever its status and versions. Runs of a budget window's years are not - the report's own runs, so they are left out. Each also comes with its - newest budget-window run, chosen the same way, and that run's years, - which a second query reads for every report at once. Reports of other + and its default run: the newest succeeded run that wasn't forced, else + the newest run that wasn't forced, whatever its status and versions + (newest is created last, ties broken by the larger ID). Runs of a budget + window's years are not the report's own runs, so they are left out. Each + also comes with its default budget-window run, chosen the same way among + runs that weren't recalculated, and that run's years, which a second + query reads for every report at once. Reports of other countries, and IDs no report has, are absent from the list, which has no set order. """ - run_rank = _newest_first(ReportRun) ranked_runs = ( - select(ReportRun, run_rank) + select(ReportRun, _default_first(ReportRun)) .where( col(ReportRun.report_id).in_(report_ids), col(ReportRun.budget_window_run_id).is_(None), + col(ReportRun.forced).is_(False), ) .subquery("ranked_report_runs") ) - newest_run = aliased(ReportRun, ranked_runs) + default_run = aliased(ReportRun, ranked_runs) ranked_window_runs = ( - select(BudgetWindowRun, _newest_first(BudgetWindowRun)) - .where(col(BudgetWindowRun.report_id).in_(report_ids)) + select(BudgetWindowRun, _default_first(BudgetWindowRun)) + .where( + col(BudgetWindowRun.report_id).in_(report_ids), + col(BudgetWindowRun.recalculate).is_(False), + ) .subquery("ranked_budget_window_runs") ) - newest_window_run = aliased(BudgetWindowRun, ranked_window_runs) + default_window_run = aliased(BudgetWindowRun, ranked_window_runs) baseline_simulation = aliased(Simulation) reform_simulation = aliased(Simulation) statement = ( @@ -85,8 +101,8 @@ def read_report_rows( col(Dataset.name), baseline_simulation, reform_simulation, - newest_run, - newest_window_run, + default_run, + default_window_run, ) .outerjoin(Region, col(Region.id) == Report.region_id) .outerjoin(Dataset, col(Dataset.id) == Report.dataset_id) @@ -98,17 +114,17 @@ def read_report_rows( reform_simulation, col(reform_simulation.id) == Report.reform_simulation_id ) .outerjoin( - newest_run, + default_run, and_( - col(newest_run.report_id) == Report.id, - ranked_runs.c.newest_first == 1, + col(default_run.report_id) == Report.id, + ranked_runs.c.default_first == 1, ), ) .outerjoin( - newest_window_run, + default_window_run, and_( - col(newest_window_run.report_id) == Report.id, - ranked_window_runs.c.newest_first == 1, + col(default_window_run.report_id) == Report.id, + ranked_window_runs.c.default_first == 1, ), ) .where(col(Report.country) == country_id, col(Report.id).in_(report_ids)) @@ -125,8 +141,8 @@ def read_report_rows( dataset_name=dataset_name, baseline_simulation=baseline, reform_simulation=reform, - newest_run=run, - newest_budget_window_run=window_run, + run=run, + budget_window_run=window_run, budget_window_years=( () if window_run is None else years.get(window_run.id, ()) ), diff --git a/policyengine_api/services/v2/stored_reports/database_connectors/reads_requests.py b/policyengine_api/services/v2/stored_reports/database_connectors/reads_requests.py new file mode 100644 index 000000000..e545aab16 --- /dev/null +++ b/policyengine_api/services/v2/stored_reports/database_connectors/reads_requests.py @@ -0,0 +1,117 @@ +"""Database reads of what a requesting user sees of reports. + +A user's saved report gives a report its label and description, and the +user's newest request for one of a report's runs +(``user_report_run_associations``) is the run a read answers by default. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import func, select +from sqlalchemy.orm import aliased +from sqlmodel import col + +from policyengine_api.data.v2.models import ( + BudgetWindowRun, + ReportRun, + UserReportAssociation, + UserReportRunAssociation, +) + +if TYPE_CHECKING: + from collections.abc import Collection + from uuid import UUID + + from sqlmodel import Session + + +def read_saved_report_labels( + session: Session, *, user_id: UUID, report_ids: Collection[UUID] +) -> dict[UUID, tuple[str | None, str | None]]: + """Return each report's name and description from the user's saved report. + + A user with several saved reports of one report gives the one saved last + (ties to the larger ID). Reports the user hasn't saved are absent. + """ + newest_saved_first = ( + func.row_number() + .over( + partition_by=col(UserReportAssociation.report_id), + order_by=( + col(UserReportAssociation.created_at).desc(), + col(UserReportAssociation.id).desc(), + ), + ) + .label("newest_saved_first") + ) + ranked = ( + select( + col(UserReportAssociation.report_id), + col(UserReportAssociation.name), + col(UserReportAssociation.description), + newest_saved_first, + ) + .where( + col(UserReportAssociation.user_id) == user_id, + col(UserReportAssociation.report_id).in_(report_ids), + ) + .subquery("ranked_saved_reports") + ) + statement = select(ranked.c.report_id, ranked.c.name, ranked.c.description).where( + ranked.c.newest_saved_first == 1 + ) + return { + report_id: (name, description) + for report_id, name, description in session.execute(statement).all() + } + + +def read_requested_report_runs( + session: Session, *, user_id: UUID, report_ids: Collection[UUID] +) -> dict[UUID, ReportRun]: + """Return, by report, the report run of the user's newest request for one. + + Newest is requested last, ties going to the run with the larger ID. For a + budget window the run is a year's, naming its budget-window run. Reports + the user never asked to run are absent. + """ + newest_requested_first = ( + func.row_number() + .over( + partition_by=col(ReportRun.report_id), + order_by=( + col(UserReportRunAssociation.requested_at).desc(), + col(ReportRun.id).desc(), + ), + ) + .label("newest_requested_first") + ) + ranked = ( + select(ReportRun, newest_requested_first) + .join( + UserReportRunAssociation, + col(UserReportRunAssociation.report_run_id) == ReportRun.id, + ) + .where( + col(UserReportRunAssociation.user_id) == user_id, + col(ReportRun.report_id).in_(report_ids), + ) + .subquery("ranked_requested_runs") + ) + requested_run = aliased(ReportRun, ranked) + statement = select(requested_run).where(ranked.c.newest_requested_first == 1) + return {run.report_id: run for run in session.scalars(statement).all()} + + +def read_budget_window_runs_by_id( + session: Session, *, budget_window_run_ids: Collection[UUID] +) -> dict[UUID, BudgetWindowRun]: + """Return the budget-window runs with these IDs, by ID.""" + if not budget_window_run_ids: + return {} + statement = select(BudgetWindowRun).where( + col(BudgetWindowRun.id).in_(budget_window_run_ids) + ) + return {window_run.id: window_run for window_run in session.scalars(statement)} diff --git a/policyengine_api/services/v2/stored_reports/database_connectors/reads_runs.py b/policyengine_api/services/v2/stored_reports/database_connectors/reads_runs.py index ab6e5eca6..22b4e1ac6 100644 --- a/policyengine_api/services/v2/stored_reports/database_connectors/reads_runs.py +++ b/policyengine_api/services/v2/stored_reports/database_connectors/reads_runs.py @@ -18,9 +18,8 @@ def read_report_runs(session: Session, *, report_id: UUID) -> list[ReportRun]: """Return every run of the report, newest first. - Newest is created last, ties broken by the larger ID, as for the report's - newest run. Runs of a budget window's years are not the report's own - runs, so they are left out. + Newest is created last, ties broken by the larger ID. Runs of a budget + window's years are not the report's own runs, so they are left out. """ statement = ( select(ReportRun) @@ -38,9 +37,8 @@ def read_budget_window_runs( ) -> list[BudgetWindowRun]: """Return every budget-window run of the report, newest first. - Newest is created last, ties broken by the larger ID, as for the - window's newest run. Each run's years are read separately - (``reads.read_budget_window_years``). + Newest is created last, ties broken by the larger ID. Each run's years + are read separately (``reads.read_budget_window_years``). """ statement = ( select(BudgetWindowRun) @@ -50,3 +48,33 @@ def read_budget_window_runs( ) ) return list(session.scalars(statement).all()) + + +def read_report_run( + session: Session, *, report_id: UUID, run_id: UUID +) -> ReportRun | None: + """Return the report's own run ``run_id``, or None when it has no such run. + + A run of another report, or of a budget window's year, is not one of its + own runs. + """ + statement = select(ReportRun).where( + col(ReportRun.id) == run_id, + col(ReportRun.report_id) == report_id, + col(ReportRun.budget_window_run_id).is_(None), + ) + return session.scalars(statement).one_or_none() + + +def read_budget_window_run( + session: Session, *, report_id: UUID, run_id: UUID +) -> BudgetWindowRun | None: + """Return the report's budget-window run ``run_id``, or None without one. + + Its years are read separately (``reads.read_budget_window_years``). + """ + statement = select(BudgetWindowRun).where( + col(BudgetWindowRun.id) == run_id, + col(BudgetWindowRun.report_id) == report_id, + ) + return session.scalars(statement).one_or_none() diff --git a/policyengine_api/services/v2/stored_reports/services.py b/policyengine_api/services/v2/stored_reports/services.py index 2b239e2c0..588af9d12 100644 --- a/policyengine_api/services/v2/stored_reports/services.py +++ b/policyengine_api/services/v2/stored_reports/services.py @@ -1,40 +1,50 @@ """Create reports through the simulation service, and read them. -Reading a report reads it within its country (``reports.country``), with its -newest run: the run created last, whatever its status and versions. That -differs from ``policyengine_api.data.v2.report_runs.select_current_report_run``, -which picks the newest succeeded run on the versions the Public API runs: a -report whose newest run used an older PolicyEngine.py version still answers -that run's result, beside the versions the Public API runs now, so the caller -can show the stored result and say a newer model version exists. Nothing is -calculated again on a read. A report with no run, as a report migrated from -v1 has none, answers its definition with no result. - -The result is rebuilt from the newest run's result rows once it has -succeeded. A budget-window report's newest run is its newest budget-window -run, read with its years; once that run has succeeded, its result is the -budget-window format, rebuilt from each year's budget rows. Reading a +Reading a report reads it within its country (``reports.country``), as the +requesting user sees it. Its run is the one the read names (one of the +report's own report runs, or a budget window's budget-window run), else the +requesting user's newest recorded run of it (``user_report_run_associations``), +else its default run: its newest succeeded run that wasn't forced, else its +newest run that wasn't forced, on whatever versions. A run on an older +PolicyEngine.py version still answers its result, beside the versions the +Public API runs now, so the caller can show the stored result and say a +newer model version exists; the newer run someone else started is offered +from the report's run list, never in its place. Its label and description +are the requesting user's saved report's name and description, and None +for anyone else. Nothing is calculated again on a read. A report with no +run, as a report migrated from v1 has none, answers its definition with no +result. + +The result is rebuilt from the run's result rows once it has succeeded. A +budget-window report's run is a budget-window run, read with its years; +once that run has succeeded, its result is the budget-window format, +rebuilt from each year's budget rows. Reading a window's definition and runs never reads those rows, so only a read of a succeeded window can find them unreadable. A household report's result is instead each of its simulation runs' calculated household -(``simulation_runs.household_result``), read once its newest report run has +(``simulation_runs.household_result``), read once its report run has succeeded, and each side's failure once it has failed. Lists name at most 25 -reports and read them, with their newest runs and any window's years, in two -queries, without results. +reports and read them as a read without a run does, with any window's +years, without results: two queries, and with a user three or four more for +their labels and their newest recorded runs. Creating a report or budget window checks the country and the policies it names (and a household report's household), asks the simulation service to -create and start it, then reads it back from the tables. Once the simulation -service has answered with the report's ID, the report exists: a failure to -read it back raises CreatedReportUnreadableError, which names it, so the -caller reads it rather than submitting it again. +create and start it, then reads it back from the tables with the run the +service's answer names: a new run, or, once the service finds runs by key, +an existing one the request joined, which needn't be the newest. Once the +simulation service has answered with the report's ID, the report exists: a +failure to read it back raises CreatedReportUnreadableError, which names it, +so the caller reads it rather than submitting it again. Running a report again reads it in its country, checks its country and the policies its simulations name as a new report of its kind is checked, has the -simulation service create and start its new run on the current versions, -then reads the report back with that run as its newest. The database session -is closed while the simulation service is asked. A failure to read it back -raises RerunReportUnreadableError, since the new run exists. A report's runs +simulation service start its run on the current versions, then reads the +report back with the run the service's answer names: a new run, or one +already pending, running or succeeded on those versions that the request +joined. The database session is closed while the simulation service is +asked. A failure to read it back raises RerunReportUnreadableError, since +the run exists. A report's runs are listed newest first, whatever their status and versions, beside the versions the Public API runs, so a caller can tell which are out of date: an economy or household report's report runs, or a budget window's @@ -44,11 +54,17 @@ from __future__ import annotations from contextlib import contextmanager +from dataclasses import dataclass, replace from typing import TYPE_CHECKING from sqlalchemy.exc import SQLAlchemyError -from policyengine_api.data.v2.models import ReportRunStatus +from policyengine_api.data.v2.models import ReportRunStatus, ReportRunTrigger +from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.identity.services import ( + find_principal_user_in_session, +) +from policyengine_api.services.v2.report_run_records.types import ReportRunRequest from policyengine_api.services.v2.simulation_service.types import ( ReportRerunInput, SimulationServiceUnavailableError, @@ -62,6 +78,9 @@ from policyengine_api.services.v2.stored_reports.database_connectors import ( reads_household_results as household_result_reads, ) +from policyengine_api.services.v2.stored_reports.database_connectors import ( + reads_requests as request_reads, +) from policyengine_api.services.v2.stored_reports.database_connectors import ( reads_results as result_reads, ) @@ -70,7 +89,9 @@ read_report_rows, ) from policyengine_api.services.v2.stored_reports.database_connectors.reads_runs import ( + read_budget_window_run, read_budget_window_runs, + read_report_run, read_report_runs, ) from policyengine_api.services.v2.stored_reports.economy_results import ( @@ -97,10 +118,14 @@ ) from policyengine_api.services.v2.stored_reports.validators import ( CreatedReportUnreadableError, + ReportRunNotRecordedError, + ReportRunRecordsNotConfiguredError, ReportStorageError, RerunReportUnreadableError, + require_admin_to_force, require_reform_run, require_report, + require_report_run, ) if TYPE_CHECKING: @@ -109,6 +134,11 @@ from sqlmodel import Session + from policyengine_api.services.v2.identity.types import AccountPrincipal + from policyengine_api.services.v2.report_run_records.types import ( + CheckedRequester, + ReportRunRecorder, + ) from policyengine_api.services.v2.simulation_service.types import ( PackageVersions, ReportCreationInput, @@ -144,7 +174,7 @@ def _storage_failures() -> Iterator[None]: def _stored_report( session: Session, *, country_id: str, report_id: UUID ) -> StoredReportRows: - """Return the country's report with its newest run. + """Return the country's report with its default run (see ``read_report_rows``). Raise StoredReportNotFoundError when the country has no such report. """ @@ -154,10 +184,148 @@ def _stored_report( ) +def _with_run( + session: Session, stored: StoredReportRows, run_id: UUID +) -> StoredReportRows: + """Return the report's rows with its run ``run_id`` in place of the one read. + + A budget window's run is a budget-window run, read with its years; any + other report's is one of its own report runs. Raise ReportRunNotFoundError + when the report has no such run. + """ + report_id = stored.report.id + if report_kind(stored.report) == "budget_window": + window_run = require_report_run( + read_budget_window_run(session, report_id=report_id, run_id=run_id), + report_id=report_id, + run_id=run_id, + ) + years = read_budget_window_years(session, [window_run.id]) + return replace( + stored, + budget_window_run=window_run, + budget_window_years=years.get(window_run.id, ()), + ) + run = require_report_run( + read_report_run(session, report_id=report_id, run_id=run_id), + report_id=report_id, + run_id=run_id, + ) + return replace(stored, run=run) + + +@dataclass(frozen=True) +class _RunRecording: + """The recorder of a request naming its user, and that user as checked.""" + + recorder: ReportRunRecorder + requester: CheckedRequester + + +def _record_request(recording: _RunRecording, request: ReportRunRequest) -> UUID: + """Record the requester's request for its run; return the user it names. + + Raise ReportRunNotRecordedError, caused by the record's own error, + whatever fails: the run exists, so the caller reads it. + """ + try: + return recording.recorder.record(recording.requester, request) + except Exception as error: + raise ReportRunNotRecordedError( + report_id=request.report_id, + run_id=request.run_id, + country_id=request.country_id, + ) from error + + +def _viewer_user_id( + session: Session, + *, + requester: AccountPrincipal | None, + recorded_user_id: UUID | None, +) -> UUID | None: + """Return the ID of the user a read is for. + + That is ``recorded_user_id`` when a request was just recorded for it, or + else the requester's stored user's, or None. + """ + if recorded_user_id is not None: + return recorded_user_id + if requester is None: + return None + user = find_principal_user_in_session(session, requester) + return None if user is None else user.user_id + + +def _requesters_view( + session: Session, + rows: Sequence[StoredReportRows], + *, + requester: AccountPrincipal | None, + recorded_user_id: UUID | None = None, + with_requested_runs: bool, +) -> list[StoredReportRows]: + """Return the reports as the requester sees them. + + The requester's stored user is looked up, unless ``recorded_user_id`` + names the user a request was just recorded for. Each report takes its + label and description from that user's saved report of it. With + ``with_requested_runs``, a report the user asked to run + is read with the run of their newest request in place of its default (a + budget window's with that year's budget-window run and its years). A + request naming no user, or a user not stored yet, sees each report's + default run and no label. Raise StandardUserRequiresSignInError for an + unauthenticated credential that names a signed-in user. + """ + user_id = _viewer_user_id( + session, requester=requester, recorded_user_id=recorded_user_id + ) + if user_id is None or not rows: + return list(rows) + report_ids = [stored.report.id for stored in rows] + labels = request_reads.read_saved_report_labels( + session, user_id=user_id, report_ids=report_ids + ) + requested_runs = ( + request_reads.read_requested_report_runs( + session, user_id=user_id, report_ids=report_ids + ) + if with_requested_runs + else {} + ) + window_run_ids = [ + run.budget_window_run_id + for run in requested_runs.values() + if run.budget_window_run_id is not None + ] + window_runs = request_reads.read_budget_window_runs_by_id( + session, budget_window_run_ids=window_run_ids + ) + window_years = read_budget_window_years(session, window_run_ids) + viewed = [] + for stored in rows: + name, description = labels.get(stored.report.id, (None, None)) + seen = replace( + stored, saved_report_name=name, saved_report_description=description + ) + requested = requested_runs.get(stored.report.id) + if requested is not None and requested.budget_window_run_id is None: + seen = replace(seen, run=requested) + elif requested is not None and requested.budget_window_run_id in window_runs: + window_run_id = requested.budget_window_run_id + seen = replace( + seen, + budget_window_run=window_runs[window_run_id], + budget_window_years=window_years.get(window_run_id, ()), + ) + viewed.append(seen) + return viewed + + def _summary_read( stored: StoredReportRows, *, current_versions: PackageVersions ) -> StoredReportSummaryRead: - """Return the report's definition and newest run, as its kind reads them.""" + """Return the report's definition and the run read, as its kind reads them.""" kind = report_kind(stored.report) if kind == "household": return household_report_summary_read(stored, current_versions=current_versions) @@ -175,6 +343,7 @@ def __init__( *, current_versions: Callable[[str], PackageVersions], submissions: SimulationServiceClient | None = None, + run_records: ReportRunRecorder | None = None, ) -> None: """Read through ``database_session``; create through ``submissions``. @@ -182,10 +351,33 @@ def __init__( versions the Public API runs for a country. ``submissions`` is the checking ``SimulationSubmissionService``; without it, creating a report or running one again raises SubmissionsNotConfiguredError. + ``run_records`` records each user's request for a run; without it, a + create or re-run naming its user raises + ReportRunRecordsNotConfiguredError. """ self._database_session = database_session self._current_versions = current_versions self._submissions = submissions + self._run_records = run_records + + def _run_recording( + self, requester: AccountPrincipal | None + ) -> _RunRecording | None: + """Check a request's user before anything starts; return how to record it. + + Return None for a request naming no user, whose run nobody records. + Raise ReportRunRecordsNotConfiguredError without a recorder, and the + recorder's check's errors. + """ + if requester is None: + return None + if self._run_records is None: + msg = "Users' report runs are not recorded here" + raise ReportRunRecordsNotConfiguredError(msg) + return _RunRecording( + recorder=self._run_records, + requester=self._run_records.check_requester(requester), + ) def _configured_submissions(self) -> SimulationServiceClient: """Return the submissions, or raise SubmissionsNotConfiguredError.""" @@ -194,26 +386,73 @@ def _configured_submissions(self) -> SimulationServiceClient: raise SubmissionsNotConfiguredError(msg) return self._submissions - def get_report(self, *, country_id: str, report_id: UUID) -> StoredReportRead: - """Return the country's report, its newest run and that run's result. - - The report is an economy report, a household report or a budget - window. Raise StoredReportNotFoundError when the country has no such - report, EconomyResultUnreadableError when a succeeded run's rows lack - a part every result has, HouseholdResultUnreadableError when a + def get_report( + self, + *, + country_id: str, + report_id: UUID, + run_id: UUID | None = None, + requester: AccountPrincipal | None = None, + ) -> StoredReportRead: + """Return the country's report, a run and that run's result, as a user sees it. + + The run is ``run_id``; without it, the requester's newest recorded + run of the report, else the report's default run (its newest + succeeded run that wasn't forced, else its newest run that wasn't + forced). The label and description are the requester's saved + report's, or None. The report is an economy report, a household + report or a budget window. Raise StoredReportNotFoundError when the + country has no such report, ReportRunNotFoundError when it has no run + ``run_id`` (a budget window's runs are its budget-window runs; any + other report's, its own report runs), + StandardUserRequiresSignInError for an unauthenticated credential + that names a signed-in user, EconomyResultUnreadableError when a + succeeded run's rows lack a part every result has, + HouseholdResultUnreadableError when a succeeded household report run's calculated households cannot be read, BudgetWindowResultUnreadableError when a succeeded window run's years cannot give its result, and ReportStorageError when the database cannot be read. """ + return self._read_report( + country_id=country_id, + report_id=report_id, + run_id=run_id, + requester=requester, + ) + + def _read_report( + self, + *, + country_id: str, + report_id: UUID, + run_id: UUID | None, + requester: AccountPrincipal | None, + recorded_user_id: UUID | None = None, + ) -> StoredReportRead: + """Read the report as ``get_report`` does, for a requester or a recorded user. + + ``recorded_user_id`` names the user a request was just recorded for, + whom a read back is for: after a sign-in converted a browser's user + in place, the browser's principal names a signed-in user. + """ with _storage_failures(), self._database_session.read() as session: stored = _stored_report(session, country_id=country_id, report_id=report_id) + (stored,) = _requesters_view( + session, + [stored], + requester=requester, + recorded_user_id=recorded_user_id, + with_requested_runs=run_id is None, + ) + if run_id is not None: + stored = _with_run(session, stored, run_id) kind = report_kind(stored.report) if kind == "budget_window": return self._budget_window(session, stored) if kind == "household": return self._household_report(session, stored) - run = stored.newest_run + run = stored.run result = None if run is not None and run.status is ReportRunStatus.SUCCEEDED: result = result_transformations.economy_result( @@ -233,8 +472,8 @@ def get_report(self, *, country_id: str, report_id: UUID) -> StoredReportRead: def _budget_window( self, session: Session, stored: StoredReportRows ) -> BudgetWindowReportRead: - """Return a budget window, with its result once its newest run succeeded.""" - window_run = stored.newest_budget_window_run + """Return a budget window, with its result once the run read succeeded.""" + window_run = stored.budget_window_run result = None if window_run is not None and window_run.status is ReportRunStatus.SUCCEEDED: result = window_result_transformations.budget_window_result( @@ -253,12 +492,12 @@ def _budget_window( def _household_report( self, session: Session, stored: StoredReportRows ) -> HouseholdReportRead: - """Return a household report with its newest run's outcome. + """Return a household report with the outcome of the run read. Its calculated households are read once that run has succeeded, and each side's failure once it has failed. """ - run = stored.newest_run + run = stored.run result = failure = None if run is not None and run.status in { ReportRunStatus.SUCCEEDED, @@ -281,17 +520,28 @@ def _household_report( ) def list_reports( - self, *, country_id: str, report_ids: Sequence[UUID] + self, + *, + country_id: str, + report_ids: Sequence[UUID], + requester: AccountPrincipal | None = None, ) -> tuple[StoredReportSummaryRead, ...]: """Return the country's reports among ``report_ids``, without results. - They are in the order named, each once; unknown reports and other - countries' are left out. Raise ReportStorageError when the database + They are in the order named, each once, as ``get_report`` reads them + without a run: with the requester's newest recorded run, else the + report's default run, and the requester's saved report's label. + Unknown reports and other countries' are left out. Raise + StandardUserRequiresSignInError for an unauthenticated credential + that names a signed-in user, and ReportStorageError when the database cannot be read. """ with _storage_failures(), self._database_session.read() as session: - rows = read_report_rows( - session, country_id=country_id, report_ids=report_ids + rows = _requesters_view( + session, + read_report_rows(session, country_id=country_id, report_ids=report_ids), + requester=requester, + with_requested_runs=True, ) current_versions = self._current_versions(country_id) return tuple( @@ -299,67 +549,159 @@ def list_reports( for stored in listed_reports(rows, report_ids=report_ids) ) - def create_report(self, report_input: ReportCreationInput) -> StoredReportRead: + def create_report( + self, + report_input: ReportCreationInput, + *, + requester: AccountPrincipal | None = None, + ) -> StoredReportRead: """Create the report through the simulation service and return it as stored. - Each kind of report goes through its own submission. Raise the - submission's errors (see ``SimulationSubmissionService``), and - CreatedReportUnreadableError, caused by the read's own error, when the - report the simulation service created cannot be read back in the - country: not stored there, the database unavailable, or any other - failure of the read. + Each kind of report goes through its own submission, and the report + is read back with the run the simulation service's answer names (a + budget window's budget-window run). With a ``requester``, it is + checked before the simulation service is asked, and the request is + recorded for its user (trigger ``initial``) once the service has + answered: that run, or each year's report run of a budget window's. + Raise the + recorder's check's errors, the submission's errors (see + ``SimulationSubmissionService``), ReportRunNotRecordedError when the + request could not be recorded, and CreatedReportUnreadableError, + caused by the read's own error, when the report the simulation + service created cannot be read back in the country: not stored there, + the database unavailable, or any other failure of the read. """ submissions = self._configured_submissions() + requested_at = utc_now() + recording = self._run_recording(requester) if report_input.kind == "budget_window": - report_id = submissions.submit_budget_window(report_input).report_id - elif report_input.kind == "household": - report_id = submissions.submit_household_report(report_input).report_id + window = submissions.submit_budget_window(report_input) + request = ReportRunRequest( + country_id=report_input.country_id, + report_id=window.report_id, + budget_window_run_id=window.budget_window_run_id, + requested_at=requested_at, + trigger=ReportRunTrigger.INITIAL, + ) else: - report_id = submissions.submit_report(report_input).report_id + submitted = ( + submissions.submit_household_report(report_input) + if report_input.kind == "household" + else submissions.submit_report(report_input) + ) + request = ReportRunRequest( + country_id=report_input.country_id, + report_id=submitted.report_id, + report_run_id=submitted.report_run_id, + requested_at=requested_at, + trigger=ReportRunTrigger.INITIAL, + ) + report_id = request.report_id + recorded_user_id = ( + None if recording is None else _record_request(recording, request) + ) try: - return self.get_report( - country_id=report_input.country_id, report_id=report_id + return self._read_report( + country_id=report_input.country_id, + report_id=report_id, + run_id=request.run_id, + requester=None, + recorded_user_id=recorded_user_id, ) except Exception as error: # The report exists whatever failed, so the caller gets its ID. raise CreatedReportUnreadableError( - report_id=report_id, country_id=report_input.country_id + report_id=report_id, + run_id=request.run_id, + country_id=report_input.country_id, ) from error def rerun_report( - self, *, country_id: str, report_id: UUID, force: bool + self, + *, + country_id: str, + report_id: UUID, + force: bool, + requester: AccountPrincipal | None = None, ) -> StoredReportRead: - """Run the country's report again; return it with its new run as its newest. - - Without ``force``, each side reuses a run of its own simulation that - succeeded on the current versions; with it, both are calculated again - (a budget window's new run then recalculates each year). A household - report's sides are always calculated again. Raise - StoredReportNotFoundError when the country has no such report, - ReportStorageError when it cannot be read, the submission's checks' - and the simulation service's errors (see - ``SimulationSubmissionService.rerun_report``), and + """Run the country's report again; return it with the run the request ended on. + + That is the run the simulation service's answer names: a new run, or + one already pending, running or succeeded on the current versions + that the request joined (a budget window's budget-window run). Without + ``force``, each side reuses a run of its own simulation that succeeded + on the current versions; with it, both are calculated again (a budget + window's new run then recalculates each year). A household report's + sides are always calculated again. Only an admin user may force a + re-run. With a ``requester``, it is + checked before the simulation service is asked, and the request is + recorded for its user (trigger ``manual``, forced or not) once the + service has answered: that run, or each year's report run of a budget + window's. Raise StoredReportNotFoundError when the country has no such + report, ForcedRerunRequiresSignInError for a forced re-run naming no + user and ForcedRerunRequiresAdminError for one by a user below admin, + both before the simulation service is asked, ReportStorageError when + it cannot be read, the recorder's check's + errors, the submission's checks' and the simulation service's errors + (see ``SimulationSubmissionService.rerun_report``), + ReportRunNotRecordedError when the request could not be recorded, and RerunReportUnreadableError, caused by the read's own error, when the - report cannot be read back once its new run was started. + report cannot be read back once its run was started, or without a + cause when a budget window's answer names no budget-window run. """ submissions = self._configured_submissions() + requested_at = utc_now() with _storage_failures(), self._database_session.read() as session: stored = _stored_report(session, country_id=country_id, report_id=report_id) - submissions.rerun_report( + kind = report_kind(stored.report) + recording = self._run_recording(requester) + if force: + require_admin_to_force(None if recording is None else recording.requester) + started = submissions.rerun_report( ReportRerunInput( country_id=country_id, report_id=report_id, - kind=report_kind(stored.report), + kind=kind, force=force, policy_ids=report_policy_ids(stored), ) ) + if kind != "budget_window": + request = ReportRunRequest( + country_id=country_id, + report_id=report_id, + report_run_id=started.report_run_id, + requested_at=requested_at, + trigger=ReportRunTrigger.MANUAL, + ) + elif started.budget_window_run_id is not None: + request = ReportRunRequest( + country_id=country_id, + report_id=report_id, + budget_window_run_id=started.budget_window_run_id, + requested_at=requested_at, + trigger=ReportRunTrigger.MANUAL, + ) + else: + # The run exists, but the answer doesn't say which window run. + raise RerunReportUnreadableError( + report_id=report_id, run_id=None, country_id=country_id + ) + recorded_user_id = ( + None if recording is None else _record_request(recording, request) + ) try: - return self.get_report(country_id=country_id, report_id=report_id) + return self._read_report( + country_id=country_id, + report_id=report_id, + run_id=request.run_id, + requester=None, + recorded_user_id=recorded_user_id, + ) except Exception as error: - # The new run exists whatever failed, so the caller reads it. + # The run exists whatever failed, so the caller reads it. raise RerunReportUnreadableError( - report_id=report_id, country_id=country_id + report_id=report_id, run_id=request.run_id, country_id=country_id ) from error def list_report_runs( diff --git a/policyengine_api/services/v2/stored_reports/transformations.py b/policyengine_api/services/v2/stored_reports/transformations.py index af8aecb7e..2dbcdafc4 100644 --- a/policyengine_api/services/v2/stored_reports/transformations.py +++ b/policyengine_api/services/v2/stored_reports/transformations.py @@ -95,8 +95,8 @@ def _report_definition( return ReportDefinitionRead( id=report.id, country_id=report.country, - label=report.label, - description=report.description, + label=rows.saved_report_name, + description=rows.saved_report_description, year=report.year, baseline_simulation=_simulation(rows.baseline_simulation), reform_simulation=_simulation(rows.reform_simulation), @@ -106,11 +106,11 @@ def _report_definition( ) -def _newest_run( +def _run_and_status( rows: StoredReportRows, ) -> tuple[ReportStatusName, ReportRunRead | None]: - """Return the report's status and newest run: ``not_run`` and None without one.""" - run = rows.newest_run + """Return the report's status and the run read: ``not_run`` and None without one.""" + run = rows.run if run is None: return "not_run", None return run.status.value, report_run_read(run) @@ -119,8 +119,8 @@ def _newest_run( def report_summary_read( rows: StoredReportRows, *, current_versions: PackageVersions ) -> ReportSummaryRead: - """Return an economy report's definition and newest run, without a result.""" - status, run = _newest_run(rows) + """Return an economy report's definition and the run read, without a result.""" + status, run = _run_and_status(rows) return ReportSummaryRead( **vars(_report_definition(rows, current_versions=current_versions)), kind="economy", @@ -137,7 +137,7 @@ def report_read( current_versions: PackageVersions, result: EconomyResult | None, ) -> ReportRead: - """Return an economy report with its newest run's result, if it has one.""" + """Return an economy report with the run read's result, if it has one.""" summary = report_summary_read(rows, current_versions=current_versions) return ReportRead(**vars(summary), result=result) @@ -145,7 +145,7 @@ def report_read( def household_report_summary_read( rows: StoredReportRows, *, current_versions: PackageVersions ) -> HouseholdReportSummaryRead: - """Return a household report's definition and newest run, without a result. + """Return a household report's definition and the run read, without a result. Raise ValueError for a report naming no household, which is not a household report. @@ -154,7 +154,7 @@ def household_report_summary_read( if report.household_id is None: msg = f"Report {report.id} is not a household report" raise ValueError(msg) - status, run = _newest_run(rows) + status, run = _run_and_status(rows) return HouseholdReportSummaryRead( **vars(_report_definition(rows, current_versions=current_versions)), kind="household", @@ -171,7 +171,7 @@ def household_report_read( result: HouseholdReportResult | None, failure: HouseholdReportFailure | None, ) -> HouseholdReportRead: - """Return a household report with its newest run's outcome, if it has one.""" + """Return a household report with the run read's outcome, if it has one.""" summary = household_report_summary_read(rows, current_versions=current_versions) return HouseholdReportRead(**vars(summary), result=result, failure=failure) @@ -212,7 +212,7 @@ def budget_window_run_read( def budget_window_summary_read( rows: StoredReportRows, *, current_versions: PackageVersions ) -> BudgetWindowReportSummaryRead: - """Return a budget-window report's definition and newest run, without a result. + """Return a budget-window report's definition and the run read, without a result. Raise ValueError for a report with no window size, which is not a budget window. @@ -221,7 +221,7 @@ def budget_window_summary_read( if report.budget_window_size is None: msg = f"Report {report.id} is not a budget window" raise ValueError(msg) - window_run = rows.newest_budget_window_run + window_run = rows.budget_window_run return BudgetWindowReportSummaryRead( **vars(_report_definition(rows, current_versions=current_versions)), kind="budget_window", @@ -243,7 +243,7 @@ def budget_window_read( current_versions: PackageVersions, result: window_result_types.BudgetWindowResult | None, ) -> BudgetWindowReportRead: - """Return a budget-window report with its newest run's result, if it has one.""" + """Return a budget-window report with the run read's result, if it has one.""" summary = budget_window_summary_read(rows, current_versions=current_versions) return BudgetWindowReportRead(**vars(summary), result=result) diff --git a/policyengine_api/services/v2/stored_reports/types.py b/policyengine_api/services/v2/stored_reports/types.py index 9c495ed64..175072fc0 100644 --- a/policyengine_api/services/v2/stored_reports/types.py +++ b/policyengine_api/services/v2/stored_reports/types.py @@ -42,8 +42,8 @@ HouseholdReportResult, ) -# A report's status is its newest run's, or not_run while it has none, as a -# report migrated from v1 has none. +# A report's status is its run's (the run a read answers), or not_run while it +# has none, as a report migrated from v1 has none. ReportStatusName = Literal["not_run", "pending", "running", "succeeded", "failed"] # What a report compares: two economy simulations, two household simulations, # or two economy simulations in each year of a budget window. @@ -91,7 +91,7 @@ class ReportDefinitionRead: """What every report's read gives: its definition and the current versions. ``current_versions`` are the country package and PolicyEngine.py versions - the Public API serves now; a newest run on other versions is out of date. + the Public API serves now; a run on other versions is out of date. ``options`` are the options the report was created with, as stored. """ @@ -109,7 +109,7 @@ class ReportDefinitionRead: @dataclass(frozen=True) class ReportSummaryRead(ReportDefinitionRead): - """An economy report's definition, its newest run and the current versions. + """An economy report's definition, the run read and the current versions. ``region_code`` and ``dataset_name`` are the region and dataset both simulations use, null when the report names none. @@ -124,19 +124,19 @@ class ReportSummaryRead(ReportDefinitionRead): @dataclass(frozen=True) class ReportRead(ReportSummaryRead): - """A report with its newest run's result, once that run has succeeded.""" + """A report with its run's result, once that run has succeeded.""" result: EconomyResult | None @dataclass(frozen=True) class HouseholdReportSummaryRead(ReportDefinitionRead): - """A household report's definition, its newest run and the current versions. + """A household report's definition, the run read and the current versions. Both simulations calculate the stored household ``household_id`` in its default year, ``year``; ``reform_simulation`` is None for a report of the - baseline alone. It has no region or dataset. ``run`` is the newest report - run, whatever its status and versions, naming each simulation's run. + baseline alone. It has no region or dataset. ``run`` is the report run + the read answers (see ``StoredReportRows``), naming each simulation's run. """ kind: Literal["household"] @@ -147,7 +147,7 @@ class HouseholdReportSummaryRead(ReportDefinitionRead): @dataclass(frozen=True) class HouseholdReportRead(HouseholdReportSummaryRead): - """A household report with its newest run's outcome. + """A household report with its run's outcome. ``result`` holds the calculated households once that run has succeeded; ``failure`` says why each side failed once it has failed. Both are None @@ -194,13 +194,14 @@ class BudgetWindowRunRead: @dataclass(frozen=True) class BudgetWindowReportSummaryRead(ReportDefinitionRead): - """A budget-window report's definition, its newest run and the current versions. + """A budget-window report's definition, the run read and the current versions. The window is ``budget_window_size`` years from ``year``, in the region ``region_code`` and dataset ``dataset_name``, null when the report names none. The report's two simulations are its first year's; each year's - report run names that year's own simulation runs. ``run`` is the newest - budget-window run, whatever its status and versions, with its years. + report run names that year's own simulation runs. ``run`` is the + budget-window run the read answers (see ``StoredReportRows``), with its + years. """ kind: Literal["budget_window"] @@ -213,7 +214,7 @@ class BudgetWindowReportSummaryRead(ReportDefinitionRead): @dataclass(frozen=True) class BudgetWindowReportRead(BudgetWindowReportSummaryRead): - """A budget-window report with its newest run's result, once that run succeeded.""" + """A budget-window report with its run's result, once that run succeeded.""" result: window_result_types.BudgetWindowResult | None @@ -260,14 +261,19 @@ class BudgetWindowRunsRead: @dataclass(frozen=True) class StoredReportRows: - """A report's row, its newest runs and the catalog and simulation rows it names. + """A report's row, the run a read answers and the rows it names. ``region_code`` and ``dataset_name`` are those of the report's region and dataset, and null when it names none; a simulation is None once deleted. - ``newest_run`` is the report run created last that is not a year of a - budget window, whatever its status and versions, or None. - ``newest_budget_window_run`` is the budget-window run created last, or - None, and ``budget_window_years`` its years' report runs in year order. + ``run`` is a report run that is not a year of a budget window, and + ``budget_window_run`` a budget-window run, with ``budget_window_years`` + its years' report runs in year order, or None. Read without naming a + run, each is the report's default: its newest succeeded run that wasn't + forced (for a window, not recalculated), else its newest such run of any + status, or None; the service may put the requesting user's newest + recorded run, or a run the caller names, in its place. + ``saved_report_name`` and ``saved_report_description`` are the requesting + user's saved report's, the report's label and description, or None. """ report: Report @@ -275,6 +281,8 @@ class StoredReportRows: dataset_name: str | None baseline_simulation: Simulation | None reform_simulation: Simulation | None - newest_run: ReportRun | None - newest_budget_window_run: BudgetWindowRun | None + run: ReportRun | None + budget_window_run: BudgetWindowRun | None budget_window_years: tuple[ReportRun, ...] + saved_report_name: str | None = None + saved_report_description: str | None = None diff --git a/policyengine_api/services/v2/stored_reports/validators.py b/policyengine_api/services/v2/stored_reports/validators.py index 2a9a26f88..683874a30 100644 --- a/policyengine_api/services/v2/stored_reports/validators.py +++ b/policyengine_api/services/v2/stored_reports/validators.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar +from policyengine_api.data.v2.models import UserType +from policyengine_api.services.v2.identity.validators import user_type_meets_minimum from policyengine_api.services.v2.stored_reports.economy_results.types import ( EconomyResultUnreadableError, ) @@ -12,13 +14,23 @@ from uuid import UUID from policyengine_api.data.v2.models import ReportRun + from policyengine_api.services.v2.report_run_records.types import ( + CheckedRequester, + ) from policyengine_api.services.v2.stored_reports.types import StoredReportRows +# A report run, or a budget window's budget-window run. +RunT = TypeVar("RunT") + class StoredReportNotFoundError(LookupError): """No report has this ID in the requested country.""" +class ReportRunNotFoundError(LookupError): + """The report has no run with the ID a read names.""" + + class ReportStorageError(RuntimeError): """The detailed-save database could not be read.""" @@ -27,36 +39,104 @@ class CreatedReportUnreadableError(RuntimeError): """The simulation service created the report, but it could not be read back. The simulation service answered with the report's ID, so the report - exists and submitting it again would create another. ``report_id`` and - ``country_id`` say where to read it; the error's cause says why reading - it back failed. + exists and submitting it again would create another. ``report_id``, + ``run_id`` (the run the answer named, which a read names to answer it) + and ``country_id`` say where to read it; the error's cause says why + reading it back failed. """ - def __init__(self, *, report_id: UUID, country_id: str) -> None: - """Name the created report and its country.""" + def __init__(self, *, report_id: UUID, run_id: UUID, country_id: str) -> None: + """Name the created report, its run and its country.""" super().__init__( f"The simulation service created report {report_id} in country " f"{country_id!r}, but it could not be read back" ) self.report_id = report_id + self.run_id = run_id + self.country_id = country_id + + +class ForcedRerunRequiresSignInError(PermissionError): + """A forced re-run sent no user credential; only an admin user may force one.""" + + +class ForcedRerunRequiresAdminError(PermissionError): + """A forced re-run's user is below admin; only an admin user may force one.""" + + +FORCED_RERUN_SIGN_IN_MESSAGE = ( + "Only admin users may force a re-run: sign in with a bearer access token" +) +FORCED_RERUN_ADMIN_MESSAGE = "Only admin users may force a re-run" + + +def require_admin_to_force(requester: CheckedRequester | None) -> None: + """Allow a forced re-run only for a requester whose stored type is admin. + + Raise ForcedRerunRequiresSignInError for a request naming no user, and + ForcedRerunRequiresAdminError for one below admin: a signed-out + browser, a signed-in user not stored yet, or a standard or author user. + """ + if requester is None: + raise ForcedRerunRequiresSignInError(FORCED_RERUN_SIGN_IN_MESSAGE) + if requester.user_type is None or not user_type_meets_minimum( + requester.user_type, UserType.ADMIN + ): + raise ForcedRerunRequiresAdminError(FORCED_RERUN_ADMIN_MESSAGE) + + +class ReportRunRecordsNotConfiguredError(ReportStorageError): + """A request named its user, but the service records no users' runs.""" + + +class ReportRunNotRecordedError(RuntimeError): + """The simulation service started the run, but the user's request wasn't recorded. + + The run exists, so the caller reads it rather than asking again; asking + again records it once the simulation service finds runs by key, since + the request then ends on the same run. ``report_id``, ``run_id`` (a + report run, or a budget window's budget-window run) and ``country_id`` + say where to read it; the error's cause says why the record failed. + """ + + def __init__(self, *, report_id: UUID, run_id: UUID, country_id: str) -> None: + """Name the report, its run and its country.""" + super().__init__( + f"The simulation service started run {run_id} of report " + f"{report_id}, but the request could not be recorded for its user. " + f"Read it with GET /v2/reports/{report_id}?country_id={country_id}" + f"&run_id={run_id}." + ) + self.report_id = report_id + self.run_id = run_id self.country_id = country_id class RerunReportUnreadableError(RuntimeError): - """The simulation service started the report's new run, but it could not be read. + """The simulation service started the report's run, but it can't be answered. - The new run exists, so the caller reads the report rather than running - it again. ``report_id`` and ``country_id`` say where to read it; the - error's cause says why reading it back failed. + Either the report could not be read back with the run the answer named + (``run_id``), and the error's cause says why, or a budget window's answer + named no budget-window run (``run_id`` None, and no cause). The run + exists, so the caller reads the report rather than running it again; + ``report_id`` and ``country_id`` say where. """ - def __init__(self, *, report_id: UUID, country_id: str) -> None: - """Name the report and its country.""" + def __init__( + self, *, report_id: UUID, run_id: UUID | None, country_id: str + ) -> None: + """Name the report, its run and its country.""" + failure = ( + "its answer named no budget-window run" + if run_id is None + else "the report could not be read back" + ) super().__init__( - f"The simulation service started a new run of report {report_id} in " - f"country {country_id!r}, but the report could not be read back" + f"The simulation service started a run of report {report_id} in " + f"country {country_id!r}, but {failure}" ) self.report_id = report_id + self.run_id = run_id self.country_id = country_id @@ -70,6 +150,14 @@ def require_report( return rows +def require_report_run(run: RunT | None, *, report_id: UUID, run_id: UUID) -> RunT: + """Return the report's run, or raise ReportRunNotFoundError for None.""" + if run is None: + msg = f"Report {report_id} has no run {run_id}" + raise ReportRunNotFoundError(msg) + return run + + def require_reform_run(report_run: ReportRun) -> UUID: """Return the reform simulation run an economy report run compares. diff --git a/policyengine_api/services/v2/user_reports/services.py b/policyengine_api/services/v2/user_reports/services.py index 853d4e086..2c356065a 100644 --- a/policyengine_api/services/v2/user_reports/services.py +++ b/policyengine_api/services/v2/user_reports/services.py @@ -8,7 +8,11 @@ refused by the ``RESTRICT`` foreign key, which is answered as not found too. A saved report naming one gives that report's own budget window as ``budget_window_years``, or none for a report without a window of two or -more years. +more years. Its ``last_run_at`` is the owner's newest request for one of +that report's runs (``user_report_run_associations``), or null without one, +set when it is saved or repointed and as the owner asks for runs, whatever +a caller sends; a saved report naming only a v1 report output keeps the +caller's ``last_run_at``. The owner always comes from the verified session principal (a signed-in user or a signed-out browser), and v1 references are checked in a short read-only v1 transaction before the v2 transaction opens. Reads never provision a @@ -36,6 +40,9 @@ resolve_principal_user_in_session, ) from policyengine_api.services.v2.legacy_refs.types import LegacyRecordKind +from policyengine_api.services.v2.report_run_records.database_connectors import ( + reads as record_reads, +) from policyengine_api.services.v2.user_reports.database_connectors.creates import ( create_user_report, ) @@ -283,7 +290,9 @@ def create_user_report( AssociationBudgetWindowConflictError if ``budget_window_years`` disagrees with its budget window. Each saved-reform link must name the owner's saved policy in the same country, or - AssociationLinkNotFoundError is raised. + AssociationLinkNotFoundError is raised. A saved detailed-save report's + ``last_run_at`` is the owner's newest request for one of its runs, or + null, whatever the caller sends. """ country_id = association_input.country_id require_one_reference( @@ -321,12 +330,20 @@ def create_user_report( country_id=association_input.country_id, user_policy_id=linked_user_policy_id, ) - return user_report_read( - create_user_report( - session, - owned_user_report_input(association_input, user_id=owner.user_id), - ) + creation_input = owned_user_report_input( + association_input, user_id=owner.user_id ) + if association_input.report_id is not None: + creation_input = creation_input.model_copy( + update={ + "last_run_at": record_reads.read_newest_report_request( + session, + user_id=owner.user_id, + report_id=association_input.report_id, + ) + } + ) + return user_report_read(create_user_report(session, creation_input)) def get_user_report( self, @@ -434,7 +451,10 @@ def patch_user_report( the saved report naming both, AssociationBudgetWindowConflictError when it would leave a ``budget_window_years`` that disagrees with the named report's budget window, and UserReportNotFoundError if the - principal has no such report in this country. + principal has no such report in this country. A saved report that + names a detailed-save report after the update, and whose ``report_id`` + or ``last_run_at`` the update supplies, takes the owner's newest + request for one of that report's runs as ``last_run_at``, or null. """ require_one_reference( "report", @@ -471,13 +491,20 @@ def patch_user_report( country_id=country_id, user_policy_id=getattr(association_input, link_field), ) - return user_report_read( - update_user_report( - session, - association, - user_report_update_values(association_input), + values = user_report_update_values(association_input) + report_id = values.get("report_id", association.report_id) + if ( + report_id is not None + and { + "report_id", + "last_run_at", + } + & association_input.model_fields_set + ): + values["last_run_at"] = record_reads.read_newest_report_request( + session, user_id=owner.user_id, report_id=report_id ) - ) + return user_report_read(update_user_report(session, association, values)) def delete_user_report( self, diff --git a/pyproject.toml b/pyproject.toml index 2a96b7045..25ca45f7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -409,6 +409,8 @@ banned-module-level-imports = [ "policyengine_api.services.v2.metadata.services", "policyengine_api.services.v2.policies.database_session", "policyengine_api.services.v2.policies.services", + "policyengine_api.services.v2.report_run_records.database_session", + "policyengine_api.services.v2.report_run_records.services", "policyengine_api.services.v2.simulation_service.services", "policyengine_api.services.v2.stored_reports.database_session", "policyengine_api.services.v2.stored_reports.services", diff --git a/tests/integration/test_v2_budget_window_report_reads.py b/tests/integration/test_v2_budget_window_report_reads.py index 178e34a8e..71548db1e 100644 --- a/tests/integration/test_v2_budget_window_report_reads.py +++ b/tests/integration/test_v2_budget_window_report_reads.py @@ -358,13 +358,13 @@ def test_a_budget_window_with_no_run_answers_its_definition( assert (item["status"], item["run"], item["result"]) == ("not_run", None, None) -def test_the_newest_window_run_is_read_even_when_an_older_one_succeeded( +def test_an_older_succeeded_window_run_is_read_before_a_newer_pending_one( owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient ) -> None: catalog = catalogs["us"] with owner_engine.begin() as connection: report = store_report(connection, catalog, budget_window_size=2) - store_window( + succeeded_run_id, _ = store_window( connection, catalog, report, @@ -373,7 +373,7 @@ def test_the_newest_window_run_is_read_even_when_an_older_one_succeeded( {2026: "succeeded", 2027: "succeeded"}, ), ) - newest_run_id, _ = store_window( + store_window( connection, catalog, report, @@ -389,10 +389,10 @@ def test_the_newest_window_run_is_read_even_when_an_older_one_succeeded( item = read_item(client, report.report_id, "us") - assert item["run"]["id"] == str(newest_run_id) - assert item["run"]["country_package_version"] == "1.5.0" - assert item["status"] == "pending" - assert item["result"] is None + assert item["run"]["id"] == str(succeeded_run_id) + assert item["run"]["country_package_version"] == "1.4.0" + assert item["status"] == "succeeded" + assert item["result"] is not None def test_of_two_window_runs_created_together_the_larger_id_is_the_newest( diff --git a/tests/integration/test_v2_forced_report_reruns.py b/tests/integration/test_v2_forced_report_reruns.py new file mode 100644 index 000000000..4506c1cca --- /dev/null +++ b/tests/integration/test_v2_forced_report_reruns.py @@ -0,0 +1,156 @@ +"""Only admin users may force a report's re-run, on migrated PostgreSQL. + +C1 · 24's answer: ``POST /v2/reports/{report_id}/runs`` takes ``force`` +from an admin user and refuses it, before the simulation service is asked, +from anyone else: 401 without a user credential, 403 for a signed-out +browser or a user below admin. An unforced re-run needs no credential. The +routes run as the Public API's runtime role. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from tests.integration.stored_report_rows import ( + delete_catalog, + store_catalog, + store_report, +) +from tests.integration.user_report_run_support import ( + JoiningSimulationService, + as_browser, + client_for, + store_signed_in_user, + stored_records, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + + from tests.integration.stored_report_rows import CatalogRows + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: + engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + yield sessionmaker(engine, class_=Session) + engine.dispose() + + +@pytest.fixture +def user_ids(owner_engine: Engine) -> Iterator[list[UUID]]: + """Collect the users a test makes, and delete them and their records after.""" + collected: list[UUID] = [] + yield collected + with owner_engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = ANY(:ids)"), {"ids": collected} + ) + + +def test_a_forced_rerun_without_a_user_credential_is_refused_with_401( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{report.report_id}/runs?country_id=us", json={"force": True} + ) + + assert response.status_code == 401, response.text + assert response.headers["WWW-Authenticate"] == "Bearer" + assert "admin" in response.json()["message"] + assert simulation_service.reruns == [] + + +@pytest.mark.parametrize("requester_type", ["unauthenticated", "standard", "author"]) +def test_a_forced_rerun_by_anyone_below_admin_is_refused_with_403( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + user_ids: list[UUID], + requester_type: str, +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + simulation_service = JoiningSimulationService(owner_engine, catalog) + if requester_type == "unauthenticated": + browser_id = uuid4() + user_ids.append(browser_id) + client = client_for(runtime_sessions, simulation_service) + headers = as_browser(browser_id) + else: + principal, user_id = store_signed_in_user( + runtime_sessions, owner_engine, user_type=requester_type + ) + user_ids.append(user_id) + client = client_for(runtime_sessions, simulation_service, requester=principal) + headers = {} + + response = client.post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={"force": True}, + headers=headers, + ) + + assert response.status_code == 403, response.text + assert "admin" in response.json()["message"] + assert simulation_service.reruns == [] + assert stored_records(owner_engine, user_ids) == [] + + +def test_an_admins_forced_rerun_is_sent_and_recorded( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + user_ids: list[UUID], +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + admin, admin_id = store_signed_in_user( + runtime_sessions, owner_engine, user_type="admin" + ) + user_ids.append(admin_id) + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service, requester=admin).post( + f"/v2/reports/{report.report_id}/runs?country_id=us", json={"force": True} + ) + + assert response.status_code == 201, response.text + (rerun,) = simulation_service.reruns + assert rerun.force is True + (record,) = stored_records(owner_engine, [admin_id]) + assert str(record.report_run_id) == response.json()["result"]["item"]["run"]["id"] + assert record.trigger == "manual" diff --git a/tests/integration/test_v2_report_creation.py b/tests/integration/test_v2_report_creation.py index 98efc910a..7ce1d32b7 100644 --- a/tests/integration/test_v2_report_creation.py +++ b/tests/integration/test_v2_report_creation.py @@ -92,7 +92,7 @@ class StoringSimulationService: """Store what the simulation service would create, and answer with its IDs. With ``answer_unstored``, it answers with IDs it stored nothing for, and - keeps them in ``unstored_ids``. + keeps them in ``unstored_ids`` (a report's run in ``unstored_run_ids``). """ engine: Engine @@ -101,13 +101,16 @@ class StoringSimulationService: reports: list[EconomyReportCreationInput] = field(default_factory=list) simulations: list[EconomySimulationCreationInput] = field(default_factory=list) unstored_ids: list[UUID] = field(default_factory=list) + unstored_run_ids: list[UUID] = field(default_factory=list) def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: self.reports.append(report) if self.answer_unstored: self.unstored_ids.append(uuid4()) + self.unstored_run_ids.append(uuid4()) return SubmittedReport( - report_id=self.unstored_ids[-1], report_run_id=uuid4() + report_id=self.unstored_ids[-1], + report_run_id=self.unstored_run_ids[-1], ) with self.engine.begin() as connection: stored = store_report(connection, self.catalog) @@ -271,14 +274,16 @@ def test_a_created_report_that_cannot_be_read_back_answers_its_id( assert response.status_code == 502 (report_id,) = simulation_service.unstored_ids + (run_id,) = simulation_service.unstored_run_ids assert response.json() == { "status": "error", "message": ( f"The simulation service created report {report_id}, but it could " f"not be read back. Read it with GET /v2/reports/{report_id}" - "?country_id=us rather than submitting it again." + f"?country_id=us&run_id={run_id} rather than submitting it again." ), "report_id": str(report_id), + "run_id": str(run_id), } diff --git a/tests/integration/test_v2_report_reads.py b/tests/integration/test_v2_report_reads.py index 945b36a3b..2b4e11817 100644 --- a/tests/integration/test_v2_report_reads.py +++ b/tests/integration/test_v2_report_reads.py @@ -134,7 +134,7 @@ def _as_json(value: Any) -> Any: [("us", us_economy_result()), ("uk", uk_economy_result())], ids=["us", "uk"], ) -def test_a_report_reads_its_newest_run_and_that_runs_result( +def test_a_report_reads_its_newest_unforced_success_and_that_runs_result( owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient, @@ -142,19 +142,20 @@ def test_a_report_reads_its_newest_run_and_that_runs_result( output: dict[str, Any], ) -> None: catalog = catalogs[country] - older_output = json.loads(json.dumps(output)) - older_output["budget"]["tax_revenue_impact"] = 1.0 + forced_output = json.loads(json.dumps(output)) + forced_output["budget"]["tax_revenue_impact"] = 1.0 with owner_engine.begin() as connection: report = store_report(connection, catalog) - store_report_run( + newest = store_report_run( connection, catalog, report, RunSettings(versions=OLDER_VERSIONS), - result=older_output, + result=output, ) - # On the same versions, the newer run is a forced re-run's. - newest = store_report_run( + # On the same versions, the newer run is a forced re-run's, which is + # never a read's default run. + store_report_run( connection, catalog, report, @@ -163,7 +164,7 @@ def test_a_report_reads_its_newest_run_and_that_runs_result( created_at=STARTED_AT + timedelta(days=1), forced=True, ), - result=output, + result=forced_output, ) response = client.get(f"/v2/reports/{report.report_id}?country_id={country}") @@ -193,7 +194,7 @@ def test_a_report_reads_its_newest_run_and_that_runs_result( assert item["kind"] == "economy" -def test_a_newest_run_still_running_has_no_result_even_after_an_older_success( +def test_an_older_success_is_read_before_a_newer_run_still_running( owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient ) -> None: catalog = catalogs["us"] @@ -221,9 +222,9 @@ def test_a_newest_run_still_running_has_no_result_even_after_an_older_success( "item" ] - assert item["status"] == "running" - assert item["run"]["completed_at"] is None - assert item["result"] is None + assert item["status"] == "succeeded" + assert item["run"]["policyengine_version"] == "4.1.0" + assert item["result"] is not None def test_a_failed_newest_run_says_why_without_a_result( @@ -268,7 +269,8 @@ def test_a_report_with_no_run_answers_its_definition( assert item["status"] == "not_run" assert item["run"] is None assert item["result"] is None - assert item["label"] == "uk reform" + # A label is the requesting user's saved report's; this read names no user. + assert item["label"] is None assert item["year"] == 2026 assert item["dataset_name"] == "enhanced_frs_2023_24" @@ -408,7 +410,7 @@ def test_a_simulation_with_no_run_answers_its_definition( assert item["year"] == 2026 -def test_of_two_report_runs_created_together_the_larger_id_is_the_newest( +def test_of_two_unsucceeded_runs_created_together_the_larger_id_is_read( owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient ) -> None: catalog = catalogs["us"] @@ -429,8 +431,9 @@ def test_of_two_report_runs_created_together_the_larger_id_is_the_newest( connection, catalog, report, - RunSettings(versions=CURRENT_VERSIONS, run_id=SMALLER_RUN_ID), - result=us_economy_result(), + RunSettings( + versions=CURRENT_VERSIONS, status="pending", run_id=SMALLER_RUN_ID + ), ) read = client.get(f"/v2/reports/{report.report_id}?country_id=us") diff --git a/tests/integration/test_v2_report_reads_by_run.py b/tests/integration/test_v2_report_reads_by_run.py new file mode 100644 index 000000000..729f743b1 --- /dev/null +++ b/tests/integration/test_v2_report_reads_by_run.py @@ -0,0 +1,288 @@ +"""Read a v2 report with the run a caller names, on migrated PostgreSQL. + +``GET /v2/reports/{report_id}?run_id=`` answers that run of the report, its +status, versions and result, whichever run is the newest: a report run of an +economy or household report, or a budget-window run of a budget window. The +routes run as the Public API's runtime role. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.constants import REPO +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.simulation_service.types import PackageVersions +from policyengine_api.services.v2.stored_reports.database_session import ( + StoredReportDatabaseSession, +) +from policyengine_api.services.v2.stored_reports.services import V2StoredReportService +from tests.fixtures.economy_results import us_economy_result +from tests.integration.simulation_rows import insert_row +from tests.integration.stored_report_rows import ( + STARTED_AT, + BudgetWindowYear, + RunSettings, + delete_catalog, + store_budget_window_run, + store_catalog, + store_report, + store_report_run, + store_year_simulations, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + + from tests.integration.stored_report_rows import CatalogRows + +CURRENT_VERSIONS = {"country_package_version": "1.5.0", "policyengine_version": "4.2.0"} +OLDER_VERSIONS = {"country_package_version": "1.4.0", "policyengine_version": "4.1.0"} +WINDOW_YEARS = (2026, 2027) + + +def current_versions(country_id: str) -> PackageVersions: + return PackageVersions(**CURRENT_VERSIONS) + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def client(managed_v2_roles, v2_engine_as_role) -> Iterator[TestClient]: + """Serve the report routes as the runtime role.""" + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + sessions = sessionmaker(runtime_engine, class_=Session) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_stored_report_service_factory=lambda: V2StoredReportService( + StoredReportDatabaseSession(sessions), current_versions=current_versions + ), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + yield TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + runtime_engine.dispose() + + +def test_an_economy_report_is_read_with_the_run_named_rather_than_its_newest( + owner_engine: Engine, catalog: CatalogRows, client: TestClient +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + named = store_report_run( + connection, + catalog, + report, + RunSettings(versions=OLDER_VERSIONS, status="succeeded"), + result=us_economy_result(), + ) + newest = store_report_run( + connection, + catalog, + report, + RunSettings( + versions=CURRENT_VERSIONS, + status="running", + created_at=STARTED_AT + timedelta(days=1), + ), + ) + + named_read = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={named.report_run_id}" + ) + newest_read = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={newest.report_run_id}" + ) + + assert named_read.status_code == 200, named_read.text + item = named_read.json()["result"]["item"] + assert item["status"] == "succeeded" + assert item["run"]["id"] == str(named.report_run_id) + assert item["run"]["policyengine_version"] == "4.1.0" + assert item["result"] is not None + assert newest_read.status_code == 200, newest_read.text + newest_item = newest_read.json()["result"]["item"] + assert newest_item["run"]["id"] == str(newest.report_run_id) + assert newest_item["status"] == "running" + assert newest_item["result"] is None + + +def test_a_run_that_is_not_one_of_the_reports_own_is_not_found( + owner_engine: Engine, catalog: CatalogRows, client: TestClient +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + store_report_run(connection, catalog, report, RunSettings(CURRENT_VERSIONS)) + other_report = store_report(connection, catalog) + other_run = store_report_run( + connection, catalog, other_report, RunSettings(CURRENT_VERSIONS) + ) + window = store_report(connection, catalog, budget_window_size=2) + window_run_id = store_budget_window_run( + connection, catalog, window, RunSettings(CURRENT_VERSIONS) + ) + year_run = store_report_run( + connection, + catalog, + store_year_simulations(connection, catalog, window, WINDOW_YEARS[0]), + RunSettings( + CURRENT_VERSIONS, + budget_window_year=BudgetWindowYear( + budget_window_run_id=window_run_id, year=WINDOW_YEARS[0] + ), + ), + ) + + another_reports_run = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={other_run.report_run_id}" + ) + unknown_run = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={uuid4()}" + ) + window_year_run = client.get( + f"/v2/reports/{window.report_id}?country_id=us&run_id={year_run.report_run_id}" + ) + window_run_of_an_economy_report = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={window_run_id}" + ) + + for response in ( + another_reports_run, + unknown_run, + window_year_run, + window_run_of_an_economy_report, + ): + assert response.status_code == 404, response.text + assert "run" in response.json()["message"] + + +def test_a_budget_window_is_read_with_the_window_run_named_and_its_years( + owner_engine: Engine, catalog: CatalogRows, client: TestClient +) -> None: + with owner_engine.begin() as connection: + window = store_report(connection, catalog, budget_window_size=2) + window_run_ids = [] + for days_later, status in ((0, "failed"), (1, "pending")): + settings = RunSettings( + versions=CURRENT_VERSIONS, + status=status, + created_at=STARTED_AT + timedelta(days=days_later), + ) + window_run_id = store_budget_window_run( + connection, catalog, window, settings + ) + window_run_ids.append(window_run_id) + for year in WINDOW_YEARS: + store_report_run( + connection, + catalog, + store_year_simulations(connection, catalog, window, year), + RunSettings( + versions=CURRENT_VERSIONS, + status=status, + created_at=settings.created_at, + budget_window_year=BudgetWindowYear( + budget_window_run_id=window_run_id, year=year + ), + ), + ) + older_window_run_id = window_run_ids[0] + + response = client.get( + f"/v2/reports/{window.report_id}?country_id=us&run_id={older_window_run_id}" + ) + + assert response.status_code == 200, response.text + item = response.json()["result"]["item"] + assert item["kind"] == "budget_window" + assert item["run"]["id"] == str(older_window_run_id) + assert item["status"] == "failed" + assert [year["year"] for year in item["run"]["years"]] == list(WINDOW_YEARS) + assert {year["status"] for year in item["run"]["years"]} == {"failed"} + + +def test_a_household_report_is_read_with_the_run_named( + owner_engine: Engine, catalog: CatalogRows, client: TestClient +) -> None: + with owner_engine.begin() as connection: + household_id = uuid4() + insert_row( + connection, + "households", + id=household_id, + country_id=catalog.country, + household_data="{}", + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + catalog.household_ids.append(household_id) + report = store_report(connection, catalog, household_id=household_id) + named = store_report_run( + connection, + catalog, + report, + RunSettings(versions=OLDER_VERSIONS, status="pending"), + ) + store_report_run( + connection, + catalog, + report, + RunSettings( + versions=CURRENT_VERSIONS, + status="running", + created_at=STARTED_AT + timedelta(days=1), + ), + ) + + response = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={named.report_run_id}" + ) + + assert response.status_code == 200, response.text + item = response.json()["result"]["item"] + assert item["kind"] == "household" + assert item["run"]["id"] == str(named.report_run_id) + assert item["status"] == "pending" diff --git a/tests/integration/test_v2_report_runs.py b/tests/integration/test_v2_report_runs.py index 08c7f2bb1..ca4612001 100644 --- a/tests/integration/test_v2_report_runs.py +++ b/tests/integration/test_v2_report_runs.py @@ -43,6 +43,12 @@ PolicyDatabaseSession, ) from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.report_run_records.database_session import ( + ReportRunRecordDatabaseSession, +) +from policyengine_api.services.v2.report_run_records.services import ( + ReportRunRecordService, +) from policyengine_api.services.v2.simulation_service.services import ( SimulationSubmissionService, ) @@ -51,6 +57,7 @@ ReportRerunInput, SimulationServiceConflictError, SimulationServiceUnavailableError, + StartedReportRun, ) from policyengine_api.services.v2.stored_reports.database_session import ( StoredReportDatabaseSession, @@ -70,12 +77,17 @@ store_report_run, store_year_simulations, ) +from tests.integration.user_report_run_support import store_signed_in_user if TYPE_CHECKING: from collections.abc import Iterator from sqlalchemy import Connection, Engine + from policyengine_api.services.v2.identity.types import ( + AccountPrincipal, + UserPrincipal, + ) from tests.integration.stored_report_rows import CatalogRows CURRENT_VERSIONS = {"country_package_version": "1.5.0", "policyengine_version": "4.2.0"} @@ -153,7 +165,7 @@ class RerunningSimulationService: new_run_ids: list[UUID] = field(default_factory=list) open_sessions_when_asked: list[int] = field(default_factory=list) - def rerun_report(self, rerun: ReportRerunInput) -> None: + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: self.reruns.append(rerun) if self.database_session is not None: self.open_sessions_when_asked.append(self.database_session.open_sessions) @@ -183,15 +195,27 @@ def rerun_report(self, rerun: ReportRerunInput) -> None: ) mark_window_manual(connection, window_run_id, recalculate=rerun.force) self.new_run_ids.append(window_run_id) + started = StartedReportRun( + report_run_id=connection.execute( + text( + "SELECT id FROM report_runs WHERE budget_window_run_id " + "= :id ORDER BY year LIMIT 1" + ), + {"id": window_run_id}, + ).scalar_one(), + budget_window_run_id=window_run_id, + ) else: run_ids = store_report_run(connection, self.catalog, report, settings) mark_manual(connection, run_ids.report_run_id) self.new_run_ids.append(run_ids.report_run_id) + started = StartedReportRun(report_run_id=run_ids.report_run_id) if self.delete_report: connection.execute( text("DELETE FROM reports WHERE id = :id"), {"id": rerun.report_id}, ) + return started @pytest.fixture @@ -218,6 +242,17 @@ def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: engine.dispose() +@pytest.fixture +def admin(owner_engine: Engine, runtime_sessions: Any) -> Iterator[UserPrincipal]: + """Provision an admin user, who may force a re-run; delete it after.""" + principal, user_id = store_signed_in_user( + runtime_sessions, owner_engine, user_type="admin" + ) + yield principal + with owner_engine.begin() as connection: + connection.execute(text("DELETE FROM users WHERE id = :id"), {"id": user_id}) + + class CountingDatabaseSession(StoredReportDatabaseSession): """Count the report service's read sessions open at each moment.""" @@ -239,8 +274,13 @@ def client_for( sessions: Any, simulation_service: RerunningSimulationService, database_session: StoredReportDatabaseSession | None = None, + requester: AccountPrincipal | None = None, ) -> TestClient: - """Serve the routes as the runtime role, running reports through the stand-in.""" + """Serve the routes as the runtime role, running reports through the stand-in. + + With ``requester``, every request acts for that principal, as a + signed-in user's verified bearer token does, and is recorded for it. + """ submissions = SimulationSubmissionService( countries=V2CountryService(CountryDatabaseSession(sessions)), households=V2HouseholdService(HouseholdDatabaseSession(sessions)), @@ -249,7 +289,9 @@ def client_for( ), client_factory=lambda: simulation_service, ) + signed_in = {} if requester is None else {"optional_user": lambda: requester} dependencies = NativeRouteDependencies( + **signed_in, readiness_probe=lambda: True, gateway_client_factory=lambda: None, metadata_reader_factory=lambda: None, @@ -257,6 +299,9 @@ def client_for( database_session or StoredReportDatabaseSession(sessions), current_versions=current_versions, submissions=submissions, + run_records=ReportRunRecordService( + ReportRunRecordDatabaseSession(sessions) + ), ), ) settings = RouteImplementationSettings( @@ -502,6 +547,7 @@ def test_running_a_report_again_answers_it_with_its_new_run( owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any, + admin: UserPrincipal, force: bool, # noqa: FBT001 # pytest passes parameters positionally ) -> None: with owner_engine.begin() as connection: @@ -510,7 +556,10 @@ def test_running_a_report_again_answers_it_with_its_new_run( connection, catalog, report, RunSettings(versions=OLDER_VERSIONS) ) simulation_service = RerunningSimulationService(owner_engine, catalog) - client = client_for(runtime_sessions, simulation_service) + # Only an admin user may force a re-run. + client = client_for( + runtime_sessions, simulation_service, requester=admin if force else None + ) response = client.post( f"/v2/reports/{report.report_id}/runs?country_id=us", json={"force": force} @@ -585,15 +634,17 @@ def test_a_new_run_whose_report_cannot_be_read_back_answers_the_reports_id( ) assert response.status_code == 502 + (new_run_id,) = simulation_service.new_run_ids assert response.json() == { "status": "error", "message": ( - f"The simulation service started a new run of report " + f"The simulation service started a run of report " f"{report.report_id}, but the report could not be read back. Read it " - f"with GET /v2/reports/{report.report_id}?country_id=us rather than " - "running it again." + f"with GET /v2/reports/{report.report_id}?country_id=us" + f"&run_id={new_run_id} rather than running it again." ), "report_id": str(report.report_id), + "run_id": str(new_run_id), } @@ -689,6 +740,7 @@ def test_running_a_budget_window_again_answers_it_with_its_new_run( owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any, + admin: UserPrincipal, force: bool, # noqa: FBT001 # pytest passes parameters positionally ) -> None: with owner_engine.begin() as connection: @@ -703,7 +755,10 @@ def test_running_a_budget_window_again_answers_it_with_its_new_run( simulation_service = RerunningSimulationService( owner_engine, catalog, window_years=(2026, 2027) ) - client = client_for(runtime_sessions, simulation_service) + # Only an admin user may force a re-run. + client = client_for( + runtime_sessions, simulation_service, requester=admin if force else None + ) response = client.post( f"/v2/reports/{window.report_id}/runs?country_id=us", json={"force": force} diff --git a/tests/integration/test_v2_user_budget_window_runs.py b/tests/integration/test_v2_user_budget_window_runs.py new file mode 100644 index 000000000..19f691bf6 --- /dev/null +++ b/tests/integration/test_v2_user_budget_window_runs.py @@ -0,0 +1,204 @@ +"""Record users' requests for budget-window runs, on migrated PostgreSQL. + +A budget window's run is a budget-window run with a report run per year; a +request for one records each year's report run for the user, with one +``requested_at``. The routes run as the Public API's runtime role, through +a stand-in for the simulation service that answers a window run it names or +one it stores. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from tests.integration.stored_report_rows import ( + STARTED_AT, + RunSettings, + delete_catalog, + store_catalog, +) +from tests.integration.user_report_run_support import ( + CURRENT_VERSIONS, + JoinedRun, + JoiningSimulationService, + as_browser, + client_for, + store_window, + stored_records, + window_body, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + + from tests.integration.stored_report_rows import CatalogRows + +WINDOW_YEARS = (2026, 2027, 2028) + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: + engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + yield sessionmaker(engine, class_=Session) + engine.dispose() + + +@pytest.fixture +def browser_id(owner_engine: Engine) -> Iterator[UUID]: + """Name a signed-out browser, and delete its user and records after.""" + browser = uuid4() + yield browser + with owner_engine.begin() as connection: + connection.execute(text("DELETE FROM users WHERE id = :id"), {"id": browser}) + + +def window_run_years(engine: Engine, window_run_id: UUID) -> set[UUID]: + with engine.begin() as connection: + return set( + connection.execute( + text("SELECT id FROM report_runs WHERE budget_window_run_id = :id"), + {"id": window_run_id}, + ).scalars() + ) + + +def test_a_budget_window_create_records_each_years_report_run( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browser_id: UUID, +) -> None: + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + response = client.post( + "/v2/reports?country_id=us", + json=window_body(catalog.policy_id, years=len(WINDOW_YEARS)), + headers=as_browser(browser_id), + ) + + assert response.status_code == 201, response.text + window_run_id = UUID(response.json()["result"]["item"]["run"]["id"]) + records = stored_records(owner_engine, [browser_id]) + assert {record.report_run_id for record in records} == window_run_years( + owner_engine, window_run_id + ) + assert len(records) == len(WINDOW_YEARS) + assert {record.trigger for record in records} == {"initial"} + assert len({record.requested_at for record in records}) == 1 + + +def test_a_budget_window_rerun_records_each_year_of_the_window_run_it_joined( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browser_id: UUID, +) -> None: + with owner_engine.begin() as connection: + joined = store_window( + connection, + catalog, + years=WINDOW_YEARS, + settings=RunSettings(versions=CURRENT_VERSIONS, status="running"), + ) + newer = store_window( + connection, + catalog, + years=WINDOW_YEARS, + settings=RunSettings( + versions={**CURRENT_VERSIONS, "policyengine_version": "4.3.0"}, + status="pending", + created_at=STARTED_AT + timedelta(days=1), + ), + report=joined.report, + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun( + report_id=joined.report.report_id, + report_run_id=joined.year_run_ids[0], + budget_window_run_id=joined.window_run_id, + ), + ) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{joined.report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(browser_id), + ) + + assert response.status_code == 201, response.text + assert response.json()["result"]["item"]["run"]["id"] == str(joined.window_run_id) + records = stored_records(owner_engine, [browser_id]) + assert {record.report_run_id for record in records} == set(joined.year_run_ids) + assert not {record.report_run_id for record in records} & set(newer.year_run_ids) + assert {record.trigger for record in records} == {"manual"} + + +def test_a_window_rerun_whose_answer_names_no_window_run_points_to_the_run_list( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browser_id: UUID, +) -> None: + with owner_engine.begin() as connection: + window = store_window( + connection, + catalog, + years=WINDOW_YEARS, + settings=RunSettings(versions=CURRENT_VERSIONS, status="pending"), + ) + # An answer naming the first year's report run but no budget-window run. + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun( + report_id=window.report.report_id, report_run_id=window.year_run_ids[0] + ), + ) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{window.report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(browser_id), + ) + + assert response.status_code == 502, response.text + body = response.json() + assert body["run_id"] is None + assert ( + f"/v2/reports/{window.report.report_id}/runs?country_id=us" in body["message"] + ) + assert stored_records(owner_engine, [browser_id]) == [] diff --git a/tests/integration/test_v2_user_report_run_requesters.py b/tests/integration/test_v2_user_report_run_requesters.py new file mode 100644 index 000000000..996ea480e --- /dev/null +++ b/tests/integration/test_v2_user_report_run_requesters.py @@ -0,0 +1,217 @@ +"""Whose request a report run is recorded for, on migrated PostgreSQL. + +A signed-in user's request provisions and records that user. A signed-out +browser's user may be merged into an account, or converted into one in +place, while the simulation service is asked: the run is then recorded for +the user the check found, or not at all, never for a new user re-created +with the browser's ID. The routes run as the Public API's runtime role. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from policyengine_api.services.v2.identity.services import ( + convert_unauthenticated_user_in_session, + resolve_principal_user_in_session, +) +from policyengine_api.services.v2.identity.types import ( + AccountConversionOutcome, + UserPrincipal, +) +from tests.integration.stored_report_rows import delete_catalog, store_catalog +from tests.integration.user_report_run_support import ( + JoiningSimulationService, + as_browser, + client_for, + report_body, + stored_records, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + + from tests.integration.stored_report_rows import CatalogRows + +ISSUER = "https://users.example.test/" + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: + engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + yield sessionmaker(engine, class_=Session) + engine.dispose() + + +@pytest.fixture +def user_ids(owner_engine: Engine) -> Iterator[list[UUID]]: + """Collect the users a test makes, and delete them and their records after.""" + collected: list[UUID] = [] + yield collected + with owner_engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = ANY(:ids)"), {"ids": collected} + ) + + +def store_browser_user(engine: Engine, browser_id: UUID) -> None: + with engine.begin() as connection: + connection.execute( + text("INSERT INTO users (id, user_type) VALUES (:id, 'unauthenticated')"), + {"id": browser_id}, + ) + + +def user_type(engine: Engine, user_id: UUID) -> str | None: + with engine.begin() as connection: + return connection.execute( + text("SELECT user_type FROM users WHERE id = :id"), {"id": user_id} + ).scalar_one_or_none() + + +def test_a_signed_in_users_request_provisions_and_records_that_user( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + user_ids: list[UUID], +) -> None: + signed_in = UserPrincipal(issuer=ISSUER, subject=f"auth0|{uuid4()}") + client = client_for( + runtime_sessions, + JoiningSimulationService(owner_engine, catalog), + requester=signed_in, + ) + + response = client.post( + "/v2/reports?country_id=us", json=report_body(catalog.policy_id) + ) + + assert response.status_code == 201, response.text + with owner_engine.begin() as connection: + signed_in_user_id = connection.execute( + text( + "SELECT user_id FROM user_identities WHERE issuer = :issuer AND " + "subject = :subject" + ), + {"issuer": ISSUER, "subject": signed_in.subject}, + ).scalar_one() + user_ids.append(signed_in_user_id) + (record,) = stored_records(owner_engine, [signed_in_user_id]) + assert str(record.report_run_id) == response.json()["result"]["item"]["run"]["id"] + assert user_type(owner_engine, signed_in_user_id) == "standard" + + +def test_a_browser_merged_while_the_service_is_asked_is_not_recorded_again( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + user_ids: list[UUID], +) -> None: + browser_id = uuid4() + store_browser_user(owner_engine, browser_id) + account = UserPrincipal(issuer=ISSUER, subject=f"auth0|{uuid4()}") + with runtime_sessions.begin() as session: + account_id = resolve_principal_user_in_session( + session, account, country_id="us" + ).user_id + user_ids.extend([browser_id, account_id]) + outcomes: list[AccountConversionOutcome] = [] + + def sign_in_and_merge() -> None: + with runtime_sessions.begin() as session: + outcomes.append( + convert_unauthenticated_user_in_session( + session, + unauthenticated_user_id=browser_id, + principal=account, + country_id="us", + ).outcome + ) + + simulation_service = JoiningSimulationService( + owner_engine, catalog, during_call=sign_in_and_merge + ) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browser_id), + ) + + assert outcomes == [AccountConversionOutcome.MERGED] + assert response.status_code == 503, response.text + body = response.json() + assert body["report_id"] == str(catalog.report_ids[-1]) + assert UUID(body["run_id"]) + # The browser's user stays merged away, and nobody holds the record. + assert user_type(owner_engine, browser_id) is None + assert stored_records(owner_engine, [browser_id, account_id]) == [] + + +def test_a_browser_converted_while_the_service_is_asked_is_recorded_as_itself( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + user_ids: list[UUID], +) -> None: + browser_id = uuid4() + store_browser_user(owner_engine, browser_id) + user_ids.append(browser_id) + new_account = UserPrincipal(issuer=ISSUER, subject=f"auth0|{uuid4()}") + outcomes: list[AccountConversionOutcome] = [] + + def sign_in_and_convert() -> None: + with runtime_sessions.begin() as session: + outcomes.append( + convert_unauthenticated_user_in_session( + session, + unauthenticated_user_id=browser_id, + principal=new_account, + country_id="us", + ).outcome + ) + + simulation_service = JoiningSimulationService( + owner_engine, catalog, during_call=sign_in_and_convert + ) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browser_id), + ) + + assert outcomes == [AccountConversionOutcome.CONVERTED] + assert response.status_code == 201, response.text + (record,) = stored_records(owner_engine, [browser_id]) + assert str(record.report_run_id) == response.json()["result"]["item"]["run"]["id"] + assert user_type(owner_engine, browser_id) == "standard" diff --git a/tests/integration/test_v2_user_report_runs.py b/tests/integration/test_v2_user_report_runs.py new file mode 100644 index 000000000..abbaa0759 --- /dev/null +++ b/tests/integration/test_v2_user_report_runs.py @@ -0,0 +1,683 @@ +"""Create and re-run v2 reports whose runs other requests share, on PostgreSQL. + +Once the simulation service finds or creates reports and runs by key, a +request may end on a run that already exists: it answers 201 naming that +run, and a re-run while a run is pending or running answers that run rather +than 409. A stand-in for the simulation service's client answers so: with a +run it names (``joined_run``), or else with a new pending run it stores. The +routes run as the Public API's runtime role and read the report back with +the run the answer names, whichever run is the report's newest. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER +from policyengine_api.services.v2.identity.services import ( + convert_unauthenticated_user_in_session, + resolve_principal_user_in_session, +) +from policyengine_api.services.v2.identity.types import ( + AccountConversionOutcome, + UnauthenticatedPrincipal, + UserPrincipal, +) +from policyengine_api.services.v2.user_reports.database_session import ( + UserReportDatabaseSession, +) +from policyengine_api.services.v2.user_reports.services import V2UserReportService +from policyengine_api.services.v2.user_reports.types import ( + UserReportCreateFields, + UserReportUpdateInput, +) +from tests.integration.stored_report_rows import ( + STARTED_AT, + BudgetWindowYear, + RunSettings, + delete_catalog, + store_budget_window_run, + store_catalog, + store_report, + store_report_run, + store_year_simulations, +) +from tests.integration.user_report_run_support import ( + CURRENT_VERSIONS, + WINDOW_YEARS, + JoinedRun, + JoiningSimulationService, + UnwritableRecordSession, + as_browser, + client_for, + newest_request, + report_body, + rerun_as_browser, + saved_report_last_run_at, + store_joined_and_newer_forced_run, + store_saved_report, + store_signed_in_user, + stored_records, +) +from tests.unit.v2.user_service_support import default_legacy_references + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + + from tests.integration.stored_report_rows import CatalogRows + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: + engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + yield sessionmaker(engine, class_=Session) + engine.dispose() + + +def test_a_create_that_joins_a_run_answers_that_run_not_the_reports_newest( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + report, joined_run_id, _forced_run_id = store_joined_and_newer_forced_run( + connection, catalog, joined_status="succeeded" + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun(report_id=report.report_id, report_run_id=joined_run_id), + ) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", json=report_body(catalog.policy_id) + ) + + assert response.status_code == 201, response.text + item = response.json()["result"]["item"] + assert item["id"] == str(report.report_id) + assert item["run"]["id"] == str(joined_run_id) + assert item["status"] == "succeeded" + + +def test_a_rerun_while_a_run_is_under_way_answers_201_with_that_run( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + report, joined_run_id, _forced_run_id = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun(report_id=report.report_id, report_run_id=joined_run_id), + ) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{report.report_id}/runs?country_id=us", json={} + ) + + assert response.status_code == 201, response.text + item = response.json()["result"]["item"] + assert item["run"]["id"] == str(joined_run_id) + assert item["status"] == "pending" + + +def test_a_budget_window_rerun_answers_the_window_run_it_joined( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + window = store_report(connection, catalog, budget_window_size=2) + window_run_ids = [] + first_year_run_ids = [] + for days_later in (0, 1): + settings = RunSettings( + versions=CURRENT_VERSIONS, + status="pending", + created_at=STARTED_AT + timedelta(days=days_later), + forced=days_later == 1, + ) + window_run_id = store_budget_window_run( + connection, catalog, window, settings + ) + window_run_ids.append(window_run_id) + for year in WINDOW_YEARS: + year_run = store_report_run( + connection, + catalog, + store_year_simulations(connection, catalog, window, year), + RunSettings( + versions=CURRENT_VERSIONS, + status="pending", + created_at=settings.created_at, + budget_window_year=BudgetWindowYear( + budget_window_run_id=window_run_id, year=year + ), + forced=settings.forced, + ), + ) + if year == WINDOW_YEARS[0]: + first_year_run_ids.append(year_run.report_run_id) + joined = JoinedRun( + report_id=window.report_id, + report_run_id=first_year_run_ids[0], + budget_window_run_id=window_run_ids[0], + ) + simulation_service = JoiningSimulationService( + owner_engine, catalog, joined_run=joined + ) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{window.report_id}/runs?country_id=us", json={} + ) + + assert response.status_code == 201, response.text + item = response.json()["result"]["item"] + assert item["kind"] == "budget_window" + assert item["run"]["id"] == str(window_run_ids[0]) + + +@pytest.fixture +def browsers(owner_engine: Engine) -> Iterator[list[UUID]]: + """Name signed-out browsers' users, and delete them and their records after.""" + browser_ids = [uuid4() for _ in range(2)] + yield browser_ids + with owner_engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = ANY(:ids)"), {"ids": browser_ids} + ) + + +def test_a_create_with_a_signed_out_browsers_credential_records_its_run( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + simulation_service = JoiningSimulationService(owner_engine, catalog) + asked_at = datetime.now(UTC) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browsers[0]), + ) + + assert response.status_code == 201, response.text + run_id = UUID(response.json()["result"]["item"]["run"]["id"]) + (record,) = stored_records(owner_engine, browsers) + assert (record.user_id, record.report_run_id, record.trigger) == ( + browsers[0], + run_id, + "initial", + ) + assert asked_at <= record.requested_at <= datetime.now(UTC) + + +def test_two_users_who_join_one_run_each_record_it( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + with owner_engine.begin() as connection: + report, joined_run_id, _forced_run_id = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun(report_id=report.report_id, report_run_id=joined_run_id), + ) + client = client_for(runtime_sessions, simulation_service) + + answers = [ + client.post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browser_id), + ) + for browser_id in browsers + ] + + assert [answer.status_code for answer in answers] == [201, 201] + records = stored_records(owner_engine, browsers) + assert {(record.user_id, record.report_run_id) for record in records} == { + (browsers[0], joined_run_id), + (browsers[1], joined_run_id), + } + + +def test_a_user_asking_again_for_a_run_they_have_keeps_their_first_record( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + with owner_engine.begin() as connection: + report, joined_run_id, _forced_run_id = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun(report_id=report.report_id, report_run_id=joined_run_id), + ) + client = client_for(runtime_sessions, simulation_service) + first = client.post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browsers[0]), + ) + (first_record,) = stored_records(owner_engine, browsers) + + again = client.post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(browsers[0]), + ) + + assert (first.status_code, again.status_code) == (201, 201) + assert stored_records(owner_engine, browsers) == [first_record] + assert first_record.trigger == "initial" + + +def test_a_rerun_records_a_manual_request_for_the_run_it_ended_on( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service).post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(browsers[0]), + ) + + assert response.status_code == 201, response.text + run_id = UUID(response.json()["result"]["item"]["run"]["id"]) + (record,) = stored_records(owner_engine, browsers) + assert (record.report_run_id, record.trigger) == (run_id, "manual") + (rerun,) = simulation_service.reruns + assert rerun.force is False + + +def test_a_request_without_a_user_credential_records_nothing( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", json=report_body(catalog.policy_id) + ) + + assert response.status_code == 201, response.text + run_id = response.json()["result"]["item"]["run"]["id"] + with owner_engine.begin() as connection: + recorded = connection.execute( + text( + "SELECT count(*) FROM user_report_run_associations WHERE " + "report_run_id = :id" + ), + {"id": run_id}, + ).scalar_one() + assert recorded == 0 + + +def test_a_browser_credential_naming_a_signed_in_user_is_refused_before_asking( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + signed_in_user_id = browsers[0] + with owner_engine.begin() as connection: + connection.execute( + text( + "INSERT INTO users (id, user_type, primary_country) VALUES " + "(:id, 'standard', 'us')" + ), + {"id": signed_in_user_id}, + ) + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(signed_in_user_id), + ) + + assert response.status_code == 401, response.text + assert response.headers["WWW-Authenticate"] == "Bearer" + assert simulation_service.reports == [] + assert stored_records(owner_engine, browsers) == [] + + +def test_a_malformed_browser_credential_is_refused_before_asking( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for(runtime_sessions, simulation_service).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers={UNAUTHENTICATED_USER_HEADER: "not-a-uuid"}, + ) + + assert response.status_code == 401, response.text + assert simulation_service.reports == [] + + +def test_a_saved_reports_last_run_at_is_its_owners_newest_request( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + # An admin, who may force a re-run, so each re-run is a new run. + owner, owner_id = store_signed_in_user( + runtime_sessions, owner_engine, user_type="admin" + ) + browsers.append(owner_id) + other_user_id = browsers[0] + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + saved_report_id = store_saved_report( + owner_engine, user_id=owner_id, report_id=report.report_id + ) + others_saved_report_id = store_saved_report( + owner_engine, user_id=other_user_id, report_id=report.report_id + ) + client = client_for( + runtime_sessions, + JoiningSimulationService(owner_engine, catalog), + requester=owner, + ) + + for _ in range(2): + response = client.post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={"force": True}, + ) + assert response.status_code == 201, response.text + + newest_request = max( + record.requested_at for record in stored_records(owner_engine, [owner_id]) + ) + assert saved_report_last_run_at(owner_engine, saved_report_id) == newest_request + assert saved_report_last_run_at(owner_engine, others_saved_report_id) is None + + +def test_a_run_whose_record_cannot_be_written_answers_503_naming_it( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + simulation_service = JoiningSimulationService(owner_engine, catalog) + + response = client_for( + runtime_sessions, + simulation_service, + record_session=UnwritableRecordSession(runtime_sessions), + ).post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browsers[0]), + ) + + assert response.status_code == 503, response.text + body = response.json() + report_id = catalog.report_ids[-1] + with owner_engine.begin() as connection: + run_id = connection.execute( + text("SELECT id FROM report_runs WHERE report_id = :id"), + {"id": report_id}, + ).scalar_one() + assert body["report_id"] == str(report_id) + assert body["run_id"] == str(run_id) + assert f"run_id={run_id}" in body["message"] + assert stored_records(owner_engine, browsers) == [] + + +def test_a_sign_in_merge_moves_the_browsers_run_records_into_the_account( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + browser_id = browsers[0] + with owner_engine.begin() as connection: + alone_report, alone_run_id, _ = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + shared_report, shared_run_id, _ = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + accounts_own_report, accounts_own_run_id, _ = store_joined_and_newer_forced_run( + connection, catalog, joined_status="pending" + ) + account = UserPrincipal(issuer="https://users.example.test/", subject="auth0|merge") + with runtime_sessions.begin() as session: + account_id = resolve_principal_user_in_session( + session, account, country_id="us" + ).user_id + browsers.append(account_id) + account_asked_at = datetime(2026, 1, 1, tzinfo=UTC) + # Later than any request the browser makes in this test. + account_asked_for_its_own_at = datetime(2030, 1, 1, tzinfo=UTC) + with owner_engine.begin() as connection: + for run_id, requested_at in ( + (shared_run_id, account_asked_at), + (accounts_own_run_id, account_asked_for_its_own_at), + ): + connection.execute( + text( + "INSERT INTO user_report_run_associations (id, user_id, " + "report_run_id, requested_at, trigger) VALUES (:id, " + ":user_id, :run_id, :requested_at, 'initial')" + ), + { + "id": uuid4(), + "user_id": account_id, + "run_id": run_id, + "requested_at": requested_at, + }, + ) + account_saved_report_id = store_saved_report( + owner_engine, user_id=account_id, report_id=alone_report.report_id + ) + browser_saved_report_id = store_saved_report( + owner_engine, user_id=browser_id, report_id=shared_report.report_id + ) + # The browser saved a report it never asked to run, but the account did. + browsers_unrun_saved_report_id = store_saved_report( + owner_engine, user_id=browser_id, report_id=accounts_own_report.report_id + ) + simulation_service = JoiningSimulationService(owner_engine, catalog) + client = client_for(runtime_sessions, simulation_service) + for report, run_id in ( + (alone_report, alone_run_id), + (shared_report, shared_run_id), + ): + simulation_service.joined_run = JoinedRun( + report_id=report.report_id, report_run_id=run_id + ) + response = client.post( + "/v2/reports?country_id=us", + json=report_body(catalog.policy_id), + headers=as_browser(browser_id), + ) + assert response.status_code == 201, response.text + browser_asked_at = { + record.report_run_id: record.requested_at + for record in stored_records(owner_engine, [browser_id]) + } + + with runtime_sessions.begin() as session: + conversion = convert_unauthenticated_user_in_session( + session, + unauthenticated_user_id=browser_id, + principal=account, + country_id="us", + ) + + assert conversion.outcome is AccountConversionOutcome.MERGED + records = stored_records(owner_engine, [account_id, browser_id]) + # A run the account had already asked for keeps the account's record. + assert { + (record.user_id, record.report_run_id, record.requested_at) + for record in records + } == { + (account_id, alone_run_id, browser_asked_at[alone_run_id]), + (account_id, shared_run_id, account_asked_at), + (account_id, accounts_own_run_id, account_asked_for_its_own_at), + } + assert {record.trigger for record in records} == {"initial"} + # Every saved report the account now holds shows the account's newest + # request for its report: its own saved report takes the request that + # moved in, and the browser's moved saved reports take the account's + # records, whatever time they held before. + assert ( + saved_report_last_run_at(owner_engine, account_saved_report_id) + == browser_asked_at[alone_run_id] + ) + assert ( + saved_report_last_run_at(owner_engine, browser_saved_report_id) + == account_asked_at + ) + assert ( + saved_report_last_run_at(owner_engine, browsers_unrun_saved_report_id) + == account_asked_for_its_own_at + ) + + +def test_saving_or_repointing_a_report_takes_its_owners_newest_request( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + owner_id = browsers[0] + owner = UnauthenticatedPrincipal(user_id=owner_id) + with owner_engine.begin() as connection: + first_report = store_report(connection, catalog) + second_report = store_report(connection, catalog) + never_run_report = store_report(connection, catalog) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + saved_reports = V2UserReportService( + UserReportDatabaseSession(runtime_sessions), default_legacy_references() + ) + callers_time = datetime(2020, 1, 1, tzinfo=UTC) + rerun_as_browser(client, first_report.report_id, owner_id) + first_request = newest_request(owner_engine, owner_id) + + saved = saved_reports.create_user_report( + principal=owner, + association_input=UserReportCreateFields( + country_id="us", + report_id=first_report.report_id, + last_run_at=callers_time, + ), + ) + never_run = saved_reports.create_user_report( + principal=owner, + association_input=UserReportCreateFields( + country_id="us", + report_id=never_run_report.report_id, + last_run_at=callers_time, + ), + ) + rerun_as_browser(client, second_report.report_id, owner_id) + second_request = newest_request(owner_engine, owner_id) + repointed = saved_reports.patch_user_report( + principal=owner, + country_id="us", + association_id=saved.id, + association_input=UserReportUpdateInput(report_id=second_report.report_id), + ) + given_a_time = saved_reports.patch_user_report( + principal=owner, + country_id="us", + association_id=saved.id, + association_input=UserReportUpdateInput(last_run_at=callers_time), + ) + renamed = saved_reports.patch_user_report( + principal=owner, + country_id="us", + association_id=saved.id, + association_input=UserReportUpdateInput(name="Renamed"), + ) + + assert saved.last_run_at == first_request + assert never_run.last_run_at is None + assert repointed.last_run_at == second_request + assert given_a_time.last_run_at == second_request + assert renamed.last_run_at == second_request + + +@pytest.mark.parametrize("request_kind", ["create", "rerun"]) +def test_a_run_that_cannot_be_read_back_is_named_in_the_502( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + request_kind: str, +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + # The answer names a run the report doesn't have, so reading it back fails. + unreadable_run_id = uuid4() + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun( + report_id=report.report_id, report_run_id=unreadable_run_id + ), + ) + client = client_for(runtime_sessions, simulation_service) + + response = ( + client.post("/v2/reports?country_id=us", json=report_body(catalog.policy_id)) + if request_kind == "create" + else client.post(f"/v2/reports/{report.report_id}/runs?country_id=us", json={}) + ) + + assert response.status_code == 502, response.text + body = response.json() + assert body["report_id"] == str(report.report_id) + assert body["run_id"] == str(unreadable_run_id) + assert f"run_id={unreadable_run_id}" in body["message"] diff --git a/tests/integration/test_v2_user_report_views.py b/tests/integration/test_v2_user_report_views.py new file mode 100644 index 000000000..1a146f5f3 --- /dev/null +++ b/tests/integration/test_v2_user_report_views.py @@ -0,0 +1,451 @@ +"""What each user reads of a shared report, on migrated PostgreSQL. + +C1 · 24's answers 7 and 6. A read without ``run_id`` answers the requesting +user's newest recorded run of the report, else its newest succeeded run that +wasn't forced, else its newest run that wasn't forced; a forced run is never +the default. A report's label and description are the requesting user's +saved report's name and description, and null without one. The routes run as +the Public API's runtime role. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER +from tests.fixtures.economy_results import us_economy_result +from tests.integration.stored_report_rows import ( + STARTED_AT, + RunSettings, + delete_catalog, + store_catalog, + store_report, + store_report_run, +) +from tests.integration.user_report_run_support import ( + CURRENT_VERSIONS, + JoinedRun, + JoiningSimulationService, + as_browser, + client_for, + store_saved_report, + store_window, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection, Engine + + from tests.integration.stored_report_rows import CatalogRows, StoredReport + +NEWER_VERSIONS = {**CURRENT_VERSIONS, "policyengine_version": "4.3.0"} + + +@pytest.fixture +def owner_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + yield engine + engine.dispose() + + +@pytest.fixture +def catalog(owner_engine: Engine) -> Iterator[CatalogRows]: + with owner_engine.begin() as connection: + stored = store_catalog(connection, "us") + yield stored + with owner_engine.begin() as connection: + delete_catalog(connection, stored) + + +@pytest.fixture +def runtime_sessions(managed_v2_roles, v2_engine_as_role) -> Iterator[Any]: + engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + yield sessionmaker(engine, class_=Session) + engine.dispose() + + +@pytest.fixture +def browsers(owner_engine: Engine) -> Iterator[list[UUID]]: + """Name signed-out browsers' users, and delete them and their records after.""" + browser_ids = [uuid4() for _ in range(2)] + yield browser_ids + with owner_engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = ANY(:ids)"), {"ids": browser_ids} + ) + + +def store_run( # noqa: PLR0913 # each run differs in a keyword-only setting + connection: Connection, + catalog: CatalogRows, + report: StoredReport, + *, + days_later: int, + status: str, + forced: bool = False, + versions: dict[str, str] = CURRENT_VERSIONS, +) -> UUID: + """Store a run of the report, with its result once it succeeded.""" + return store_report_run( + connection, + catalog, + report, + RunSettings( + versions=versions, + status=status, + created_at=STARTED_AT + timedelta(days=days_later), + forced=forced, + ), + result=us_economy_result() if status == "succeeded" else None, + ).report_run_id + + +def read_item(client: Any, report_id: UUID, headers: dict[str, str] | None = None): + response = client.get(f"/v2/reports/{report_id}?country_id=us", headers=headers) + assert response.status_code == 200, response.text + return response.json()["result"]["item"] + + +def test_without_a_record_the_newest_succeeded_unforced_run_is_read( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + store_run(connection, catalog, report, days_later=0, status="failed") + succeeded = store_run( + connection, catalog, report, days_later=1, status="succeeded" + ) + store_run( + connection, catalog, report, days_later=2, status="succeeded", forced=True + ) + store_run( + connection, + catalog, + report, + days_later=3, + status="pending", + versions=NEWER_VERSIONS, + ) + only_forced = store_report(connection, catalog) + store_run( + connection, + catalog, + only_forced, + days_later=0, + status="succeeded", + forced=True, + ) + unsucceeded = store_report(connection, catalog) + store_run(connection, catalog, unsucceeded, days_later=0, status="failed") + newest_unforced = store_run( + connection, catalog, unsucceeded, days_later=1, status="pending" + ) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + item = read_item(client, report.report_id) + forced_only_item = read_item(client, only_forced.report_id) + unsucceeded_item = read_item(client, unsucceeded.report_id) + listed = client.get( + f"/v2/reports?country_id=us&report_id={report.report_id}" + f"&report_id={unsucceeded.report_id}" + ).json()["result"]["items"] + + assert (item["run"]["id"], item["status"]) == (str(succeeded), "succeeded") + assert item["result"] is not None + assert forced_only_item["run"] is None + assert forced_only_item["status"] == "not_run" + assert unsucceeded_item["run"]["id"] == str(newest_unforced) + assert [summary["run"]["id"] for summary in listed] == [ + str(succeeded), + str(newest_unforced), + ] + + +def test_a_user_reads_their_own_newest_recorded_run( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + asker, other_user = browsers + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + first_asked = store_run( + connection, catalog, report, days_later=0, status="succeeded" + ) + newest_asked = store_run( + connection, + catalog, + report, + days_later=1, + status="running", + versions=NEWER_VERSIONS, + ) + newest_succeeded = store_run( + connection, + catalog, + report, + days_later=2, + status="succeeded", + versions={**CURRENT_VERSIONS, "policyengine_version": "4.4.0"}, + ) + simulation_service = JoiningSimulationService(owner_engine, catalog) + client = client_for(runtime_sessions, simulation_service) + for run_id in (first_asked, newest_asked): + simulation_service.joined_run = JoinedRun( + report_id=report.report_id, report_run_id=run_id + ) + response = client.post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(asker), + ) + assert response.status_code == 201, response.text + + askers_item = read_item(client, report.report_id, as_browser(asker)) + askers_listed = client.get( + f"/v2/reports?country_id=us&report_id={report.report_id}", + headers=as_browser(asker), + ).json()["result"]["items"] + others_item = read_item(client, report.report_id, as_browser(other_user)) + named_item = client.get( + f"/v2/reports/{report.report_id}?country_id=us&run_id={first_asked}", + headers=as_browser(asker), + ).json()["result"]["item"] + + assert askers_item["run"]["id"] == str(newest_asked) + assert askers_item["status"] == "running" + assert [summary["run"]["id"] for summary in askers_listed] == [str(newest_asked)] + assert others_item["run"]["id"] == str(newest_succeeded) + assert named_item["run"]["id"] == str(first_asked) + + +def test_a_user_reads_the_window_run_of_their_newest_recorded_year( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + asker = browsers[0] + with owner_engine.begin() as connection: + asked = store_window( + connection, + catalog, + years=(2026, 2027), + settings=RunSettings(versions=CURRENT_VERSIONS, status="running"), + ) + store_window( + connection, + catalog, + years=(2026, 2027), + settings=RunSettings( + versions=NEWER_VERSIONS, + status="pending", + created_at=STARTED_AT + timedelta(days=1), + ), + report=asked.report, + ) + simulation_service = JoiningSimulationService( + owner_engine, + catalog, + joined_run=JoinedRun( + report_id=asked.report.report_id, + report_run_id=asked.year_run_ids[0], + budget_window_run_id=asked.window_run_id, + ), + ) + client = client_for(runtime_sessions, simulation_service) + response = client.post( + f"/v2/reports/{asked.report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(asker), + ) + assert response.status_code == 201, response.text + + item = read_item(client, asked.report.report_id, as_browser(asker)) + + assert item["kind"] == "budget_window" + assert item["run"]["id"] == str(asked.window_run_id) + assert [year["year"] for year in item["run"]["years"]] == [2026, 2027] + + +def test_a_reports_label_and_description_are_the_users_saved_reports( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + saver, other_user = browsers + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + store_saved_report(owner_engine, user_id=saver, report_id=report.report_id) + with owner_engine.begin() as connection: + connection.execute( + text( + "UPDATE user_report_associations SET description = 'Mine' " + "WHERE user_id = :user_id" + ), + {"user_id": saver}, + ) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + savers_item = read_item(client, report.report_id, as_browser(saver)) + savers_listed = client.get( + f"/v2/reports?country_id=us&report_id={report.report_id}", + headers=as_browser(saver), + ).json()["result"]["items"] + others_item = read_item(client, report.report_id, as_browser(other_user)) + anonymous_item = read_item(client, report.report_id) + + assert (savers_item["label"], savers_item["description"]) == ("My report", "Mine") + assert [summary["label"] for summary in savers_listed] == ["My report"] + assert (others_item["label"], others_item["description"]) == (None, None) + assert (anonymous_item["label"], anonymous_item["description"]) == (None, None) + + +def test_a_malformed_browser_credential_on_a_read_is_refused( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + read = client.get( + f"/v2/reports/{report.report_id}?country_id=us", + headers={UNAUTHENTICATED_USER_HEADER: "not-a-uuid"}, + ) + listed = client.get( + f"/v2/reports?country_id=us&report_id={report.report_id}", + headers={UNAUTHENTICATED_USER_HEADER: "not-a-uuid"}, + ) + + assert (read.status_code, listed.status_code) == (401, 401) + + +def test_the_newest_request_wins_whichever_run_was_created_last( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + asker = browsers[0] + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + # Not succeeded, so neither the default run nor the newest run. + created_first = store_run( + connection, catalog, report, days_later=0, status="pending" + ) + created_last = store_run( + connection, + catalog, + report, + days_later=1, + status="succeeded", + versions=NEWER_VERSIONS, + ) + simulation_service = JoiningSimulationService(owner_engine, catalog) + client = client_for(runtime_sessions, simulation_service) + # Asked for the run created last first, then the one created first. + for run_id in (created_last, created_first): + simulation_service.joined_run = JoinedRun( + report_id=report.report_id, report_run_id=run_id + ) + response = client.post( + f"/v2/reports/{report.report_id}/runs?country_id=us", + json={}, + headers=as_browser(asker), + ) + assert response.status_code == 201, response.text + + item = read_item(client, report.report_id, as_browser(asker)) + + assert item["run"]["id"] == str(created_first) + + +def test_a_recalculated_window_run_is_never_the_default( + owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any +) -> None: + with owner_engine.begin() as connection: + unrecalculated = store_window( + connection, + catalog, + years=(2026, 2027), + settings=RunSettings(versions=CURRENT_VERSIONS, status="pending"), + ) + recalculated = store_window( + connection, + catalog, + years=(2026, 2027), + settings=RunSettings( + versions=NEWER_VERSIONS, + status="pending", + created_at=STARTED_AT + timedelta(days=1), + ), + report=unrecalculated.report, + ) + connection.execute( + text("UPDATE budget_window_runs SET recalculate = true WHERE id = :id"), + {"id": recalculated.window_run_id}, + ) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + item = read_item(client, unrecalculated.report.report_id) + + assert item["run"]["id"] == str(unrecalculated.window_run_id) + + +def test_of_two_saved_reports_of_one_report_the_one_saved_last_names_it( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + saver = browsers[0] + with owner_engine.begin() as connection: + report = store_report(connection, catalog) + first_saved = store_saved_report( + owner_engine, user_id=saver, report_id=report.report_id + ) + last_saved = store_saved_report( + owner_engine, user_id=saver, report_id=report.report_id + ) + with owner_engine.begin() as connection: + for saved_report_id, name, saved_at in ( + (first_saved, "First name", STARTED_AT), + (last_saved, "Last name", STARTED_AT + timedelta(days=1)), + ): + connection.execute( + text( + "UPDATE user_report_associations SET name = :name, " + "created_at = :saved_at WHERE id = :id" + ), + {"name": name, "saved_at": saved_at, "id": saved_report_id}, + ) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + + item = read_item(client, report.report_id, as_browser(saver)) + + assert item["label"] == "Last name" diff --git a/tests/integration/user_report_run_support.py b/tests/integration/user_report_run_support.py new file mode 100644 index 000000000..9aac21c8a --- /dev/null +++ b/tests/integration/user_report_run_support.py @@ -0,0 +1,467 @@ +"""Stand-ins and row helpers for the tests of users' report-run records. + +``JoiningSimulationService`` answers creates and re-runs as the simulation +service does once identical requests share runs: with a run it names, or +with a new pending run it stores. The helpers serve the report routes as the +Public API's runtime role and read users' records and saved reports back. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy import text +from sqlalchemy.exc import OperationalError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.households.database_session import ( + HouseholdDatabaseSession, +) +from policyengine_api.services.v2.households.services import V2HouseholdService +from policyengine_api.services.v2.identity.services import ( + resolve_principal_user_in_session, +) +from policyengine_api.services.v2.identity.types import UserPrincipal +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.report_run_records.database_session import ( + ReportRunRecordDatabaseSession, +) +from policyengine_api.services.v2.report_run_records.services import ( + ReportRunRecordService, +) +from policyengine_api.services.v2.simulation_service.services import ( + SimulationSubmissionService, +) +from policyengine_api.services.v2.simulation_service.types import ( + PackageVersions, + ReportRerunInput, + StartedReportRun, + SubmittedBudgetWindow, + SubmittedReport, +) +from policyengine_api.services.v2.stored_reports.database_session import ( + StoredReportDatabaseSession, +) +from policyengine_api.services.v2.stored_reports.services import V2StoredReportService +from tests.fixtures.economy_results import us_economy_result +from tests.integration.stored_report_rows import ( + STARTED_AT, + BudgetWindowYear, + RunSettings, + StoredReport, + store_budget_window_run, + store_report, + store_report_run, + store_year_simulations, +) + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + + from sqlalchemy import Connection, Engine + from sqlmodel import Session + + from policyengine_api.services.v2.identity.types import AccountPrincipal + from policyengine_api.services.v2.simulation_service.types import ( + BudgetWindowReportCreationInput, + EconomyReportCreationInput, + ) + from tests.integration.stored_report_rows import CatalogRows + + +CURRENT_VERSIONS = {"country_package_version": "1.5.0", "policyengine_version": "4.2.0"} +WINDOW_YEARS = (2026, 2027) + + +def current_versions(country_id: str) -> PackageVersions: + return PackageVersions(**CURRENT_VERSIONS) + + +@dataclass(frozen=True) +class JoinedRun: + """A run an answer names that the request didn't create. + + ``budget_window_run_id`` is set for a budget window's run, whose first + year's report run is ``report_run_id``. + """ + + report_id: UUID + report_run_id: UUID + budget_window_run_id: UUID | None = None + + +@dataclass +class JoiningSimulationService: + """Answer as the simulation service does once identical requests share runs. + + With ``joined_run``, every create and re-run answers that run and stores + nothing, as a request that finds an existing run does. Without it, a + create stores a new report with a pending run, and a re-run a new pending + run of the report, as before, marked forced for a forced re-run. With + ``during_call``, each create and re-run first runs it, as something + another request does while the simulation service is asked. + """ + + engine: Engine + catalog: CatalogRows + joined_run: JoinedRun | None = None + during_call: Callable[[], None] | None = None + reports: list[EconomyReportCreationInput] = field(default_factory=list) + reruns: list[ReportRerunInput] = field(default_factory=list) + + def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: + self.reports.append(report) + if self.during_call is not None: + self.during_call() + if self.joined_run is not None: + return SubmittedReport( + report_id=self.joined_run.report_id, + report_run_id=self.joined_run.report_run_id, + ) + with self.engine.begin() as connection: + stored = store_report(connection, self.catalog) + run_ids = store_report_run( + connection, + self.catalog, + stored, + RunSettings(versions=CURRENT_VERSIONS, status="pending"), + ) + return SubmittedReport( + report_id=stored.report_id, report_run_id=run_ids.report_run_id + ) + + def submit_budget_window( + self, report: BudgetWindowReportCreationInput + ) -> SubmittedBudgetWindow: + if self.during_call is not None: + self.during_call() + if self.joined_run is not None: + if self.joined_run.budget_window_run_id is None: + msg = "a budget window joins a budget-window run" + raise AssertionError(msg) + return SubmittedBudgetWindow( + report_id=self.joined_run.report_id, + budget_window_run_id=self.joined_run.budget_window_run_id, + ) + with self.engine.begin() as connection: + window = store_window( + connection, + self.catalog, + years=tuple( + range(report.year, report.year + report.budget_window_size) + ), + settings=RunSettings(versions=CURRENT_VERSIONS, status="pending"), + ) + return SubmittedBudgetWindow( + report_id=window.report.report_id, + budget_window_run_id=window.window_run_id, + ) + + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: + self.reruns.append(rerun) + if self.during_call is not None: + self.during_call() + if self.joined_run is not None: + return StartedReportRun( + report_run_id=self.joined_run.report_run_id, + budget_window_run_id=self.joined_run.budget_window_run_id, + ) + with self.engine.begin() as connection: + simulations = connection.execute( + text( + "SELECT baseline_simulation_id, reform_simulation_id FROM " + "reports WHERE id = :id" + ), + {"id": rerun.report_id}, + ).one() + run_ids = store_report_run( + connection, + self.catalog, + StoredReport( + report_id=rerun.report_id, + baseline_simulation_id=simulations.baseline_simulation_id, + reform_simulation_id=simulations.reform_simulation_id, + ), + RunSettings( + versions=CURRENT_VERSIONS, + status="pending", + created_at=STARTED_AT + timedelta(days=30), + forced=rerun.force, + ), + ) + return StartedReportRun(report_run_id=run_ids.report_run_id) + + +class UnwritableRecordSession(ReportRunRecordDatabaseSession): + """Read as usual, but fail every transaction, as an unavailable database does.""" + + def transaction(self) -> AbstractContextManager[Session]: + statement = "INSERT INTO user_report_run_associations" + raise OperationalError(statement, {}, Exception("connection refused")) + + +def client_for( + sessions: Any, + simulation_service: JoiningSimulationService, + *, + record_session: ReportRunRecordDatabaseSession | None = None, + requester: AccountPrincipal | None = None, +) -> TestClient: + """Serve the report routes as the runtime role, through the stand-in. + + Users' report runs are recorded through ``record_session``, by default + the runtime role's. With ``requester``, every request acts for that + principal, as a signed-in user's verified bearer token does; without it, + the request's own credential decides. + """ + submissions = SimulationSubmissionService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + households=V2HouseholdService(HouseholdDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), + client_factory=lambda: simulation_service, + ) + signed_in = {} if requester is None else {"optional_user": lambda: requester} + dependencies = NativeRouteDependencies( + **signed_in, + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_stored_report_service_factory=lambda: V2StoredReportService( + StoredReportDatabaseSession(sessions), + current_versions=current_versions, + submissions=submissions, + run_records=ReportRunRecordService( + record_session or ReportRunRecordDatabaseSession(sessions) + ), + ), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + + +def report_body(policy_id: UUID) -> dict[str, Any]: + return { + "country_id": "us", + "kind": "economy", + "label": "Expanded child tax credit", + "year": 2026, + "region_code": "state/ca", + "baseline": {}, + "reform": {"policy_id": str(policy_id)}, + } + + +def store_joined_and_newer_forced_run( + connection: Any, catalog: CatalogRows, *, joined_status: str +) -> tuple[StoredReport, UUID, UUID]: + """Store a report with a run to join and a newer forced run beside it. + + A succeeded run to join has its result stored. Return the report, the + run to join and the forced run, which is the report's newest. + """ + report = store_report(connection, catalog) + joined = store_report_run( + connection, + catalog, + report, + RunSettings(versions=CURRENT_VERSIONS, status=joined_status), + result=us_economy_result() if joined_status == "succeeded" else None, + ) + forced = store_report_run( + connection, + catalog, + report, + RunSettings( + versions=CURRENT_VERSIONS, + status="running", + created_at=STARTED_AT + timedelta(days=1), + forced=True, + ), + ) + return report, joined.report_run_id, forced.report_run_id + + +@dataclass(frozen=True) +class RunRecord: + """A user's stored request for a report run.""" + + user_id: UUID + report_run_id: UUID + trigger: str + requested_at: datetime + + +def as_browser(browser_id: UUID) -> dict[str, str]: + return {UNAUTHENTICATED_USER_HEADER: str(browser_id)} + + +def stored_records(engine: Engine, user_ids: list[UUID]) -> list[RunRecord]: + with engine.begin() as connection: + rows = connection.execute( + text( + "SELECT user_id, report_run_id, trigger, requested_at FROM " + "user_report_run_associations WHERE user_id = ANY(:ids) " + "ORDER BY requested_at, user_id" + ), + {"ids": user_ids}, + ).all() + return [ + RunRecord( + user_id=row.user_id, + report_run_id=row.report_run_id, + trigger=row.trigger, + requested_at=row.requested_at, + ) + for row in rows + ] + + +def store_saved_report(engine: Engine, *, user_id: UUID, report_id: UUID) -> UUID: + """Save the report for the user, with no last run time, and return its ID.""" + saved_report_id = uuid4() + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO users (id, user_type) VALUES (:id, 'unauthenticated') " + "ON CONFLICT (id) DO NOTHING" + ), + {"id": user_id}, + ) + connection.execute( + text( + "INSERT INTO user_report_associations (id, user_id, report_id, " + "country_id, name) VALUES (:id, :user_id, :report_id, 'us', " + "'My report')" + ), + {"id": saved_report_id, "user_id": user_id, "report_id": report_id}, + ) + return saved_report_id + + +def saved_report_last_run_at(engine: Engine, saved_report_id: UUID) -> datetime | None: + with engine.begin() as connection: + return connection.execute( + text("SELECT last_run_at FROM user_report_associations WHERE id = :id"), + {"id": saved_report_id}, + ).scalar_one() + + +def rerun_as_browser(client: TestClient, report_id: UUID, browser_id: UUID) -> None: + """Run the report again as the browser; a report with no run gets a new one.""" + response = client.post( + f"/v2/reports/{report_id}/runs?country_id=us", + json={}, + headers=as_browser(browser_id), + ) + assert response.status_code == 201, response.text + + +def newest_request(engine: Engine, user_id: UUID) -> datetime: + return max(record.requested_at for record in stored_records(engine, [user_id])) + + +@dataclass(frozen=True) +class StoredWindow: + """A budget-window report, one of its window runs and that run's year runs.""" + + report: StoredReport + window_run_id: UUID + year_run_ids: tuple[UUID, ...] + + +def store_window( + connection: Connection, + catalog: CatalogRows, + *, + years: tuple[int, ...], + settings: RunSettings, + report: StoredReport | None = None, +) -> StoredWindow: + """Store a window run of ``report`` (a new window if None) with a run per year.""" + window = ( + store_report(connection, catalog, budget_window_size=len(years), year=years[0]) + if report is None + else report + ) + window_run_id = store_budget_window_run(connection, catalog, window, settings) + year_run_ids = tuple( + store_report_run( + connection, + catalog, + store_year_simulations(connection, catalog, window, year), + RunSettings( + versions=settings.versions, + status=settings.status, + created_at=settings.created_at, + budget_window_year=BudgetWindowYear( + budget_window_run_id=window_run_id, year=year + ), + ), + ).report_run_id + for year in years + ) + return StoredWindow( + report=window, window_run_id=window_run_id, year_run_ids=year_run_ids + ) + + +def window_body(policy_id: UUID, *, years: int) -> dict[str, Any]: + return { + "country_id": "us", + "kind": "budget_window", + "label": "Expanded child tax credit, over years", + "year": 2026, + "budget_window_size": years, + "region_code": "state/ca", + "baseline": {}, + "reform": {"policy_id": str(policy_id)}, + } + + +ADMIN_ISSUER = "https://users.example.test/" + + +def store_signed_in_user( + sessions: Any, engine: Engine, *, user_type: str = "standard" +) -> tuple[UserPrincipal, UUID]: + """Provision a signed-in user of ``user_type``; return its principal and ID.""" + principal = UserPrincipal(issuer=ADMIN_ISSUER, subject=f"auth0|{uuid4()}") + with sessions.begin() as session: + user_id = resolve_principal_user_in_session( + session, principal, country_id="us" + ).user_id + with engine.begin() as connection: + connection.execute( + text("UPDATE users SET user_type = :user_type WHERE id = :id"), + {"user_type": user_type, "id": user_id}, + ) + return principal, user_id diff --git a/tests/unit/libs/test_simulation_budget_window_reports.py b/tests/unit/libs/test_simulation_budget_window_reports.py index 61b6d0f02..adbd248c6 100644 --- a/tests/unit/libs/test_simulation_budget_window_reports.py +++ b/tests/unit/libs/test_simulation_budget_window_reports.py @@ -112,8 +112,6 @@ def test_a_budget_window_is_posted_to_reports_and_its_window_run_returned( assert service.sent_body() == { "country": "us", "report_type": "budget_window", - "label": "Expanded child tax credit, ten years", - "description": None, "year": 2026, "budget_window_size": 10, "region_code": "us", @@ -135,7 +133,7 @@ def test_a_budget_window_sends_its_spm_settings_only_when_given() -> None: body = service.sent_body() assert body["dataset_name"] == "enhanced_cps_2024" - assert body["description"] == "Doubles the credit" + assert "description" not in body assert body["reform"] == { "policy_id": str(POLICY_ID), "dynamic_id": str(DYNAMIC_ID), diff --git a/tests/unit/libs/test_simulation_household_report_requests.py b/tests/unit/libs/test_simulation_household_report_requests.py index 500151c0f..1fba85fb7 100644 --- a/tests/unit/libs/test_simulation_household_report_requests.py +++ b/tests/unit/libs/test_simulation_household_report_requests.py @@ -72,8 +72,6 @@ def test_a_household_report_is_posted_and_the_created_report_returned() -> None: assert service.sent_body() == { "country": "uk", "report_type": "household", - "label": "My household under the reform", - "description": "Doubles the credit", "household_id": str(HOUSEHOLD_ID), "baseline": {"policy_id": None}, "reform": {"policy_id": str(POLICY_ID)}, @@ -89,8 +87,6 @@ def test_a_report_of_the_baseline_alone_sends_a_null_reform() -> None: assert household_report_request(body) == { "country": "uk", "report_type": "household", - "label": "My household under the reform", - "description": None, "household_id": str(HOUSEHOLD_ID), "baseline": {"policy_id": str(POLICY_ID)}, "reform": None, diff --git a/tests/unit/libs/test_simulation_report_reruns.py b/tests/unit/libs/test_simulation_report_reruns.py index d221d0742..fdefc74fa 100644 --- a/tests/unit/libs/test_simulation_report_reruns.py +++ b/tests/unit/libs/test_simulation_report_reruns.py @@ -20,6 +20,7 @@ SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, + StartedReportRun, ) SIMULATION_URL = "https://simulation.example.test" @@ -40,6 +41,7 @@ RUNNING_RUN_ID = UUID("00000000-0000-0000-0000-00000000d004") POLICY_ID = UUID("00000000-0000-0000-0000-00000000d005") SIMULATION_ID = UUID("00000000-0000-0000-0000-00000000d006") +WINDOW_RUN_ID = UUID("00000000-0000-0000-0000-00000000d00b") # What the simulation service answers once it has started the report's new run. RERUN_STARTED = { "report_id": str(REPORT_ID), @@ -101,11 +103,11 @@ def answer(self, request: httpx.Request) -> httpx.Response: return httpx.Response(self.status_code, content=self.body) return httpx.Response(self.status_code, json=self.body) - def rerun(self, **changes: Any) -> None: + def rerun(self, **changes: Any) -> StartedReportRun: client = SimulationServiceReportClient.from_environment( transport=httpx.MockTransport(self.answer) ) - client.rerun_report(rerun_input(**changes)) + return client.rerun_report(rerun_input(**changes)) def possibly_started_message(status_and_code: str) -> str: @@ -133,7 +135,8 @@ def test_a_rerun_posts_force_to_the_reports_runs_route( @pytest.mark.parametrize("status_code", [201, 202]) def test_a_started_rerun_naming_the_report_is_accepted(status_code: int) -> None: service = ScriptedService( - status_code=status_code, body={"report_id": str(REPORT_ID)} + status_code=status_code, + body={"report_id": str(REPORT_ID), "report_run_id": str(NEW_RUN_ID)}, ) service.rerun() @@ -141,14 +144,37 @@ def test_a_started_rerun_naming_the_report_is_accepted(status_code: int) -> None assert len(service.requests) == 1 +def test_a_started_rerun_answers_the_run_it_names_created_or_joined() -> None: + started = ScriptedService(body={**RERUN_STARTED, "status": "succeeded"}).rerun() + + assert started == StartedReportRun(report_run_id=NEW_RUN_ID) + + +def test_a_started_budget_window_rerun_answers_its_window_run() -> None: + window_started = { + "report_id": str(REPORT_ID), + "report_run_id": str(NEW_RUN_ID), + "budget_window_run_id": str(WINDOW_RUN_ID), + "status": "pending", + "years": [], + } + + started = ScriptedService(body=window_started).rerun(kind="budget_window") + + assert started == StartedReportRun( + report_run_id=NEW_RUN_ID, budget_window_run_id=WINDOW_RUN_ID + ) + + @pytest.mark.parametrize( "body", [ {**RERUN_STARTED, "report_id": str(OTHER_REPORT_ID)}, {"report_run_id": str(NEW_RUN_ID)}, + {"report_id": str(REPORT_ID)}, b"not json", ], - ids=["another-report", "no-report", "not-json"], + ids=["another-report", "no-report", "no-run", "not-json"], ) def test_a_started_answer_that_cannot_be_read_may_have_started_a_run( body: object, diff --git a/tests/unit/libs/test_simulation_reports.py b/tests/unit/libs/test_simulation_reports.py index 6a1ff57ab..3da4fe89a 100644 --- a/tests/unit/libs/test_simulation_reports.py +++ b/tests/unit/libs/test_simulation_reports.py @@ -185,11 +185,11 @@ def test_a_report_is_posted_and_the_created_report_returned() -> None: (request,) = service.requests assert request.method == "POST" assert str(request.url) == f"{SIMULATION_URL}{REPORTS_PATH}" + # A report's name and description belong to users' saved reports, so + # neither is sent: the simulation service stores none. assert service.sent_body() == { "country": "us", "report_type": "economy", - "label": "Expanded child tax credit", - "description": None, "year": 2026, "region_code": "state/ca", "dataset_name": None, @@ -215,7 +215,7 @@ def test_a_report_sends_its_options_and_spm_settings_only_when_given() -> None: body = service.sent_body() assert body["include_cliffs"] is True assert body["dataset_name"] == "enhanced_cps_2024" - assert body["description"] == "Doubles the credit" + assert "description" not in body assert body["reform"] == { "policy_id": str(POLICY_ID), "dynamic_id": str(DYNAMIC_ID), diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 2b88932a6..b1e97956b 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -233,6 +233,22 @@ def test_reusable_v2_integration_check_runs_shared_report_rows_as_managed_roles( ) +def test_reusable_v2_integration_check_runs_the_users_report_run_tests(): + """Reading a report by run and users' run records run as the runtime role.""" + workflow = _workflow("v2-integration-check.yml") + + privileges_applied = workflow.index("reconcile_v2_privileges.py apply") + for module in ( + "test_v2_report_reads_by_run.py", + "test_v2_user_report_runs.py", + "test_v2_user_report_run_requesters.py", + "test_v2_user_budget_window_runs.py", + "test_v2_forced_report_reruns.py", + "test_v2_user_report_views.py", + ): + assert privileges_applied < workflow.index(module), module + + def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): workflow = _workflow("v2-integration-check.yml") diff --git a/tests/unit/v2/report_run_support.py b/tests/unit/v2/report_run_support.py index 2988ef581..d64e26a83 100644 --- a/tests/unit/v2/report_run_support.py +++ b/tests/unit/v2/report_run_support.py @@ -25,6 +25,7 @@ ) if TYPE_CHECKING: + from policyengine_api.services.v2.identity.types import AccountPrincipal from policyengine_api.services.v2.stored_reports.types import ( StoredReportRead, StoredReportRunsRead, @@ -113,11 +114,17 @@ class FakeReportRunService(FakeReportService): rerun: StoredReportRead | None = None def rerun_report( - self, *, country_id: str, report_id: UUID, force: bool + self, + *, + country_id: str, + report_id: UUID, + force: bool, + requester: AccountPrincipal | None = None, ) -> StoredReportRead: self._answer( "rerun_report", country_id=country_id, report_id=report_id, force=force ) + self.requesters.append(requester) return self.report if self.rerun is None else self.rerun def list_report_runs( diff --git a/tests/unit/v2/stored_report_support.py b/tests/unit/v2/stored_report_support.py index f182529fe..fc73076bd 100644 --- a/tests/unit/v2/stored_report_support.py +++ b/tests/unit/v2/stored_report_support.py @@ -39,6 +39,7 @@ V2StoredReportResourceService, V2StoredSimulationResourceService, ) + from policyengine_api.services.v2.identity.types import AccountPrincipal from policyengine_api.services.v2.simulation_service.types import ( EconomyReportCreationInput, EconomySimulationCreationInput, @@ -150,24 +151,47 @@ class FakeReportService: listed: tuple[ReportSummaryRead, ...] = () error: Exception | None = None calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + # The user each create or re-run named, None for a request naming none. + requesters: list[AccountPrincipal | None] = field(default_factory=list) def _answer(self, name: str, **arguments: Any) -> None: self.calls.append((name, arguments)) if self.error is not None: raise self.error - def get_report(self, *, country_id: str, report_id: UUID) -> ReportRead: - self._answer("get_report", country_id=country_id, report_id=report_id) + def get_report( + self, + *, + country_id: str, + report_id: UUID, + run_id: UUID | None = None, + requester: AccountPrincipal | None = None, + ) -> ReportRead: + self.requesters.append(requester) + self._answer( + "get_report", country_id=country_id, report_id=report_id, run_id=run_id + ) return self.report def list_reports( - self, *, country_id: str, report_ids: Sequence[UUID] + self, + *, + country_id: str, + report_ids: Sequence[UUID], + requester: AccountPrincipal | None = None, ) -> tuple[ReportSummaryRead, ...]: + self.requesters.append(requester) self._answer("list_reports", country_id=country_id, report_ids=report_ids) return self.listed - def create_report(self, report_input: EconomyReportCreationInput) -> ReportRead: + def create_report( + self, + report_input: EconomyReportCreationInput, + *, + requester: AccountPrincipal | None = None, + ) -> ReportRead: self._answer("create_report", report_input=report_input) + self.requesters.append(requester) return self.report diff --git a/tests/unit/v2/test_budget_window_report_create_routes.py b/tests/unit/v2/test_budget_window_report_create_routes.py index e051a6627..8659d9ab3 100644 --- a/tests/unit/v2/test_budget_window_report_create_routes.py +++ b/tests/unit/v2/test_budget_window_report_create_routes.py @@ -140,8 +140,11 @@ def test_a_budget_window_the_service_failed_answers_503_with_its_id() -> None: def test_a_created_budget_window_that_cannot_be_read_back_answers_502() -> None: report_id = uuid4() + window_run_id = uuid4() service = FakeReportService( - error=CreatedReportUnreadableError(report_id=report_id, country_id="us") + error=CreatedReportUnreadableError( + report_id=report_id, run_id=window_run_id, country_id="us" + ) ) response = report_route_client(reports=service).post( @@ -150,6 +153,7 @@ def test_a_created_budget_window_that_cannot_be_read_back_answers_502() -> None: assert response.status_code == 502 assert response.json()["report_id"] == str(report_id) + assert response.json()["run_id"] == str(window_run_id) def test_the_openapi_document_tells_the_report_bodies_apart_by_kind() -> None: diff --git a/tests/unit/v2/test_budget_window_report_read_routes.py b/tests/unit/v2/test_budget_window_report_read_routes.py index be68119ef..e27cc49ba 100644 --- a/tests/unit/v2/test_budget_window_report_read_routes.py +++ b/tests/unit/v2/test_budget_window_report_read_routes.py @@ -74,7 +74,10 @@ def test_a_budget_window_answers_its_window_newest_run_years_and_result() -> Non } assert item["result"] == BUDGET_WINDOW_RESULT assert service.calls == [ - ("get_report", {"country_id": "us", "report_id": BUDGET_WINDOW_REPORT_ID}) + ( + "get_report", + {"country_id": "us", "report_id": BUDGET_WINDOW_REPORT_ID, "run_id": None}, + ) ] diff --git a/tests/unit/v2/test_household_report_reads.py b/tests/unit/v2/test_household_report_reads.py index 1647d81e0..14ec4b7ef 100644 --- a/tests/unit/v2/test_household_report_reads.py +++ b/tests/unit/v2/test_household_report_reads.py @@ -327,11 +327,8 @@ def test_a_succeeded_household_report_reads_both_calculated_households( # A household report has no region or dataset. assert not hasattr(report, "region_code") assert not hasattr(report, "dataset_name") - assert (report.label, report.description, report.year) == ( - "My household under the reform", - "Doubles the credit", - 2026, - ) + # A label is the requesting user's saved report's; this read names no user. + assert (report.label, report.description, report.year) == (None, None, 2026) assert report.options == {} assert report.baseline_simulation is not None assert report.baseline_simulation.id == stored.baseline_simulation_id diff --git a/tests/unit/v2/test_household_report_routes.py b/tests/unit/v2/test_household_report_routes.py index e569e0898..6b6af2afb 100644 --- a/tests/unit/v2/test_household_report_routes.py +++ b/tests/unit/v2/test_household_report_routes.py @@ -155,7 +155,8 @@ def test_a_household_report_of_the_baseline_alone_can_be_created() -> None: @pytest.mark.parametrize( "changes", [ - pytest.param({"label": None}, id="no-label"), + # A label is optional and unused, but one that is sent is checked. + pytest.param({"label": ""}, id="empty-label"), # The report is for the household's default year, so it names none. pytest.param({"year": 2026}, id="year"), pytest.param({"household_id": None}, id="no-household"), @@ -357,10 +358,11 @@ def test_the_create_routes_schema_offers_the_household_kind() -> None: assert set(household_request["required"]) >= { "country_id", "kind", - "label", "household_id", "baseline", } + # A report's name is each user's saved report's. + assert "label" not in household_request["required"] assert "year" not in household_request["properties"] # Each side's failure is read from the household function's failure # document, which always names its code. diff --git a/tests/unit/v2/test_household_report_submissions.py b/tests/unit/v2/test_household_report_submissions.py index f54ead890..9d9cd1620 100644 --- a/tests/unit/v2/test_household_report_submissions.py +++ b/tests/unit/v2/test_household_report_submissions.py @@ -338,8 +338,8 @@ def test_a_household_report_outside_its_schema_is_refused( household_report_input(**changes) -@pytest.mark.parametrize("missing", ["label", "household_id", "baseline"]) -def test_a_household_report_needs_its_label_household_and_baseline( +@pytest.mark.parametrize("missing", ["household_id", "baseline"]) +def test_a_household_report_needs_its_household_and_baseline( missing: str, ) -> None: values: dict[str, Any] = { diff --git a/tests/unit/v2/test_report_create_routes.py b/tests/unit/v2/test_report_create_routes.py index 6e255382a..ebb8c6f52 100644 --- a/tests/unit/v2/test_report_create_routes.py +++ b/tests/unit/v2/test_report_create_routes.py @@ -48,6 +48,7 @@ CURRENT_VERSIONS, POLICY_ID, REPORT_ID, + REPORT_RUN_ID, FakeReportService, FakeSimulationService, report_route_client, @@ -138,7 +139,8 @@ def test_a_report_body_naming_another_country_is_refused() -> None: @pytest.mark.parametrize( "body", [ - {key: value for key, value in report_body().items() if key != "label"}, + # A label is optional and unused, but one that is sent is checked. + report_body(label=""), report_body(kind="household"), report_body(year=2201), report_body(region_code="two words"), @@ -148,7 +150,7 @@ def test_a_report_body_naming_another_country_is_refused() -> None: report_body(reform={"policy_id": "not-a-uuid"}), ], ids=[ - "no-label", + "empty-label", "household-kind", "year-too-late", "region-with-space", @@ -315,7 +317,9 @@ def test_simulation_creation_failures_answer_typed_errors( def test_a_created_report_that_cannot_be_read_back_answers_its_id() -> None: - error = CreatedReportUnreadableError(report_id=REPORT_ID, country_id="us") + error = CreatedReportUnreadableError( + report_id=REPORT_ID, run_id=REPORT_RUN_ID, country_id="us" + ) client = report_route_client(reports=FakeReportService(error=error)) response = client.post("/v2/reports?country_id=us", json=report_body()) @@ -326,9 +330,11 @@ def test_a_created_report_that_cannot_be_read_back_answers_its_id() -> None: "message": ( f"The simulation service created report {REPORT_ID}, but it could " f"not be read back. Read it with GET /v2/reports/{REPORT_ID}" - "?country_id=us rather than submitting it again." + f"?country_id=us&run_id={REPORT_RUN_ID} rather than submitting it " + "again." ), "report_id": str(REPORT_ID), + "run_id": str(REPORT_RUN_ID), } @@ -460,40 +466,40 @@ def test_the_openapi_document_gives_the_missing_parameters_refusal(path: str) -> @pytest.mark.parametrize( - ("path", "status_code", "schema", "id_field"), + ("path", "status_code", "schema", "id_fields"), [ pytest.param( "/v2/reports", "502", "CreatedReportUnreadableResponse", - "report_id", + ("report_id", "run_id"), id="report-502", ), pytest.param( "/v2/simulations", "502", "CreatedSimulationUnreadableResponse", - "simulation_id", + ("simulation_id",), id="simulation-502", ), pytest.param( "/v2/reports", "503", "UnavailableAfterReportCreatedResponse", - "report_id", + ("report_id",), id="report-503", ), pytest.param( "/v2/simulations", "503", "UnavailableAfterSimulationCreatedResponse", - "simulation_id", + ("simulation_id",), id="simulation-503", ), ], ) def test_the_openapi_document_gives_the_created_records_id( - path: str, status_code: str, schema: str, id_field: str + path: str, status_code: str, schema: str, id_fields: tuple[str, ...] ) -> None: document = report_route_client().get("/v2/openapi.json").json() @@ -503,7 +509,7 @@ def test_the_openapi_document_gives_the_created_records_id( ]["schema"]["anyOf"] assert document["components"]["schemas"][schema]["required"] == [ "message", - id_field, + *id_fields, ] diff --git a/tests/unit/v2/test_report_read_by_run_routes.py b/tests/unit/v2/test_report_read_by_run_routes.py new file mode 100644 index 000000000..32955ef3d --- /dev/null +++ b/tests/unit/v2/test_report_read_by_run_routes.py @@ -0,0 +1,100 @@ +"""The query contract of reading a v2 report by run, over a stand-in service.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from policyengine_api.services.v2.stored_reports.validators import ( + ReportRunNotFoundError, +) +from tests.unit.v2.stored_report_support import ( + REPORT_ID, + REPORT_RUN_ID, + FakeReportService, + report_route_client, +) + + +def test_the_run_named_is_passed_to_the_service() -> None: + service = FakeReportService() + + response = report_route_client(reports=service).get( + f"/v2/reports/{REPORT_ID}?country_id=us&run_id={REPORT_RUN_ID}" + ) + + assert response.status_code == 200, response.text + assert service.calls == [ + ( + "get_report", + {"country_id": "us", "report_id": REPORT_ID, "run_id": REPORT_RUN_ID}, + ) + ] + + +def test_a_read_without_a_run_passes_none() -> None: + service = FakeReportService() + + response = report_route_client(reports=service).get( + f"/v2/reports/{REPORT_ID}?country_id=us" + ) + + assert response.status_code == 200, response.text + assert service.calls[0][1]["run_id"] is None + + +@pytest.mark.parametrize( + "query", + [ + "country_id=us&run_id=not-a-uuid", + f"country_id=us&run_id={uuid4()}&run_id={uuid4()}", + f"country_id=us&run={uuid4()}", + ], + ids=["not-a-uuid", "repeated", "misspelled"], +) +def test_a_malformed_run_query_is_refused_before_the_service(query: str) -> None: + service = FakeReportService() + + response = report_route_client(reports=service).get( + f"/v2/reports/{REPORT_ID}?{query}" + ) + + assert response.status_code == 422, response.text + assert service.calls == [] + + +def test_a_run_the_report_does_not_have_answers_404() -> None: + missing_run_id = uuid4() + service = FakeReportService( + error=ReportRunNotFoundError(f"Report {REPORT_ID} has no run {missing_run_id}") + ) + + response = report_route_client(reports=service).get( + f"/v2/reports/{REPORT_ID}?country_id=us&run_id={missing_run_id}" + ) + + assert response.status_code == 404 + assert response.json() == { + "status": "error", + "message": f"Report {REPORT_ID} has no run {missing_run_id}", + } + + +def test_openapi_declares_the_optional_run_query() -> None: + document = report_route_client(reports=FakeReportService()).get("/v2/openapi.json") + + operation = document.json()["paths"]["/v2/reports/{report_id}"]["get"] + run_parameter = next( + parameter + for parameter in operation["parameters"] + if parameter["name"] == "run_id" + ) + assert run_parameter["in"] == "query" + assert run_parameter["required"] is False + run_schemas = run_parameter["schema"]["anyOf"] + assert {"type": "null"} in run_schemas + assert any( + schema.get("type") == "string" and schema.get("format") == "uuid" + for schema in run_schemas + ) diff --git a/tests/unit/v2/test_report_read_routes.py b/tests/unit/v2/test_report_read_routes.py index 79ee88a33..1d220199a 100644 --- a/tests/unit/v2/test_report_read_routes.py +++ b/tests/unit/v2/test_report_read_routes.py @@ -50,7 +50,7 @@ def test_a_report_answers_its_definition_newest_run_versions_and_result() -> Non } assert item["reform_simulation"]["policy_id"] is not None assert service.calls == [ - ("get_report", {"country_id": "us", "report_id": REPORT_ID}) + ("get_report", {"country_id": "us", "report_id": REPORT_ID, "run_id": None}) ] diff --git a/tests/unit/v2/test_report_requester_routes.py b/tests/unit/v2/test_report_requester_routes.py new file mode 100644 index 000000000..3b49c4cde --- /dev/null +++ b/tests/unit/v2/test_report_requester_routes.py @@ -0,0 +1,222 @@ +"""Who asks for a report's run: the optional user credential, over stand-ins.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER +from policyengine_api.services.v2.identity.types import UnauthenticatedPrincipal +from policyengine_api.services.v2.identity.validators import ( + StandardUserRequiresSignInError, +) +from policyengine_api.services.v2.stored_reports.validators import ( + ReportRunNotRecordedError, + RerunReportUnreadableError, +) +from tests.unit.v2.report_run_support import FakeReportRunService +from tests.unit.v2.stored_report_support import ( + POLICY_ID, + REPORT_ID, + REPORT_RUN_ID, + report_route_client, +) + +REPORT_BODY = { + "country_id": "us", + "kind": "economy", + "label": "Expanded child tax credit", + "year": 2026, + "region_code": "state/ca", + "baseline": {}, + "reform": {"policy_id": str(POLICY_ID)}, +} + + +def test_a_signed_out_browsers_create_and_rerun_name_its_user() -> None: + browser_id = uuid4() + service = FakeReportRunService() + client = report_route_client(reports=service) + headers = {UNAUTHENTICATED_USER_HEADER: str(browser_id)} + + created = client.post( + "/v2/reports?country_id=us", json=REPORT_BODY, headers=headers + ) + rerun = client.post( + f"/v2/reports/{REPORT_ID}/runs?country_id=us", json={}, headers=headers + ) + + assert (created.status_code, rerun.status_code) == (201, 201) + assert service.requesters == [ + UnauthenticatedPrincipal(user_id=browser_id), + UnauthenticatedPrincipal(user_id=browser_id), + ] + + +def test_a_request_without_a_user_credential_names_no_user() -> None: + service = FakeReportRunService() + client = report_route_client(reports=service) + + created = client.post("/v2/reports?country_id=us", json=REPORT_BODY) + rerun = client.post(f"/v2/reports/{REPORT_ID}/runs?country_id=us", json={}) + + assert (created.status_code, rerun.status_code) == (201, 201) + assert service.requesters == [None, None] + + +def test_a_malformed_browser_credential_is_refused_before_the_service() -> None: + service = FakeReportRunService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=us", + json=REPORT_BODY, + headers={UNAUTHENTICATED_USER_HEADER: "not-a-uuid"}, + ) + + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Bearer") + assert service.calls == [] + + +def test_a_browser_credential_naming_a_signed_in_user_answers_401() -> None: + service = FakeReportRunService( + error=StandardUserRequiresSignInError("Sign in to act on this account") + ) + + response = report_route_client(reports=service).post( + f"/v2/reports/{REPORT_ID}/runs?country_id=us", + json={}, + headers={UNAUTHENTICATED_USER_HEADER: str(uuid4())}, + ) + + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"] == "Bearer" + assert response.json()["message"] == "Sign in to act on this account" + + +def test_a_run_whose_request_was_not_recorded_answers_503_naming_it() -> None: + not_recorded = ReportRunNotRecordedError( + report_id=REPORT_ID, run_id=REPORT_RUN_ID, country_id="us" + ) + service = FakeReportRunService(error=not_recorded) + client = report_route_client(reports=service) + + for response in ( + client.post("/v2/reports?country_id=us", json=REPORT_BODY), + client.post(f"/v2/reports/{REPORT_ID}/runs?country_id=us", json={}), + ): + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": str(not_recorded), + "report_id": str(REPORT_ID), + "run_id": str(REPORT_RUN_ID), + } + + +@pytest.mark.parametrize( + ("path", "method"), + [("/v2/reports", "post"), ("/v2/reports/{report_id}/runs", "post")], +) +def test_openapi_says_the_user_credential_is_optional(path: str, method: str) -> None: + document = report_route_client(reports=FakeReportRunService()).get( + "/v2/openapi.json" + ) + + security = document.json()["paths"][path][method]["security"] + assert {} in security + assert {"UserAccessToken": []} in security + assert {"UnauthenticatedUser": []} in security + + +def test_a_window_rerun_whose_answer_named_no_window_run_answers_a_null_run() -> None: + service = FakeReportRunService( + error=RerunReportUnreadableError( + report_id=REPORT_ID, run_id=None, country_id="us" + ) + ) + + response = report_route_client(reports=service).post( + f"/v2/reports/{REPORT_ID}/runs?country_id=us", json={} + ) + + assert response.status_code == 502 + body = response.json() + assert body["run_id"] is None + assert f"/v2/reports/{REPORT_ID}/runs?country_id=us" in body["message"] + # Nothing was read back: the answer named no window run to read. + assert "named no budget-window run" in body["message"] + + +def test_openapi_documents_the_reruns_nullable_run_and_the_reads_403() -> None: + document = ( + report_route_client(reports=FakeReportRunService()) + .get("/v2/openapi.json") + .json() + ) + + rerun_unreadable = document["components"]["schemas"][ + "RerunReportUnreadableResponse" + ] + assert {"type": "null"} in rerun_unreadable["properties"]["run_id"]["anyOf"] + assert "run_id" in rerun_unreadable["required"] + for path in ("/v2/reports", "/v2/reports/{report_id}"): + assert "403" in document["paths"][path]["get"]["responses"], path + + +HOUSEHOLD_REPORT_BODY = { + "country_id": "us", + "kind": "household", + "household_id": str(uuid4()), + "baseline": {}, + "reform": None, +} +BUDGET_WINDOW_BODY = {**REPORT_BODY, "kind": "budget_window", "budget_window_size": 3} + + +@pytest.mark.parametrize( + "unnamed_body", + [ + {key: value for key, value in REPORT_BODY.items() if key != "label"}, + {**REPORT_BODY, "label": None, "description": None}, + HOUSEHOLD_REPORT_BODY, + {key: value for key, value in BUDGET_WINDOW_BODY.items() if key != "label"}, + ], + ids=["economy-omitted", "economy-null", "household-omitted", "window-omitted"], +) +def test_a_report_is_created_without_a_label_or_description( + unnamed_body: dict[str, object], +) -> None: + service = FakeReportRunService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=us", json=unnamed_body + ) + + assert response.status_code == 201, response.text + ((_, arguments),) = service.calls + assert arguments["report_input"].label is None + assert arguments["report_input"].description is None + + +@pytest.mark.parametrize( + "schema_name", + [ + "EconomyReportCreateRequest", + "HouseholdReportCreateRequest", + "BudgetWindowReportCreateRequest", + ], +) +def test_openapi_says_a_creates_label_and_description_are_stored_nowhere( + schema_name: str, +) -> None: + schemas = ( + report_route_client(reports=FakeReportRunService()) + .get("/v2/openapi.json") + .json()["components"]["schemas"] + ) + + properties = schemas[schema_name]["properties"] + for field_name in ("label", "description"): + assert "stored nowhere" in properties[field_name]["description"], field_name diff --git a/tests/unit/v2/test_report_rerun_service.py b/tests/unit/v2/test_report_rerun_service.py index f63f87912..ea14728d7 100644 --- a/tests/unit/v2/test_report_rerun_service.py +++ b/tests/unit/v2/test_report_rerun_service.py @@ -16,7 +16,10 @@ ReportRunStatus, ReportRunTrigger, ) -from policyengine_api.services.v2.simulation_service.types import PackageVersions +from policyengine_api.services.v2.simulation_service.types import ( + PackageVersions, + StartedReportRun, +) from policyengine_api.services.v2.stored_reports.services import ( SubmissionsNotConfiguredError, V2StoredReportService, @@ -63,8 +66,9 @@ def read(self) -> AbstractContextManager[Session]: class RecordingSimulationService: reruns: list[ReportRerunInput] = field(default_factory=list) - def rerun_report(self, rerun: ReportRerunInput) -> None: + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: self.reruns.append(rerun) + return StartedReportRun(report_run_id=uuid4()) def stored_rows(**report_columns: object) -> StoredReportRows: @@ -83,8 +87,8 @@ def stored_rows(**report_columns: object) -> StoredReportRows: dataset_name=None, baseline_simulation=None, reform_simulation=None, - newest_run=None, - newest_budget_window_run=None, + run=None, + budget_window_run=None, budget_window_years=(), ) diff --git a/tests/unit/v2/test_report_rerun_submissions.py b/tests/unit/v2/test_report_rerun_submissions.py index 8f707b63f..03eda8a86 100644 --- a/tests/unit/v2/test_report_rerun_submissions.py +++ b/tests/unit/v2/test_report_rerun_submissions.py @@ -24,6 +24,7 @@ from policyengine_api.services.v2.simulation_service.types import ( ReportRerunInput, ReportRerunKind, + StartedReportRun, ) from policyengine_api.services.v2.simulation_service.validators import ( EconomyCalculationsUnsupportedError, @@ -97,8 +98,9 @@ def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: class RecordingSimulationService: reruns: list[ReportRerunInput] = field(default_factory=list) - def rerun_report(self, rerun: ReportRerunInput) -> None: + def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: self.reruns.append(rerun) + return StartedReportRun(report_run_id=uuid4()) class UnreadHouseholds: diff --git a/tests/unit/v2/test_report_run_record_merges.py b/tests/unit/v2/test_report_run_record_merges.py new file mode 100644 index 000000000..6f4eaaa93 --- /dev/null +++ b/tests/unit/v2/test_report_run_record_merges.py @@ -0,0 +1,51 @@ +"""A sign-in merge with no report-run records still completes in SQLite. + +The user service tests merge accounts in SQLite, where the merge's search +for a browser's records must run too. PostgreSQL tests +(``tests/integration/test_v2_user_report_runs.py``) check which records +move and the saved reports' ``last_run_at``. +""" + +from __future__ import annotations + +from uuid import uuid4 + +from sqlmodel import select + +from policyengine_api.data.v2.models import User +from policyengine_api.services.v2.identity.services import ( + convert_unauthenticated_user_in_session, + resolve_principal_user_in_session, +) +from policyengine_api.services.v2.identity.types import ( + AccountConversionOutcome, + UnauthenticatedPrincipal, + UserPrincipal, +) +from tests.unit.v2.user_service_support import v2_store + +ACCOUNT = UserPrincipal(issuer="https://users.example.test/", subject="auth0|merge") + + +def test_a_browser_without_run_records_merges_into_an_account() -> None: + browser = UnauthenticatedPrincipal(user_id=uuid4()) + with v2_store() as store: + with store.sessions.begin() as session: + account_id = resolve_principal_user_in_session( + session, ACCOUNT, country_id="us" + ).user_id + resolve_principal_user_in_session(session, browser, country_id="us") + + with store.sessions.begin() as session: + conversion = convert_unauthenticated_user_in_session( + session, + unauthenticated_user_id=browser.user_id, + principal=ACCOUNT, + country_id="us", + ) + + with store.sessions() as session: + user_ids = set(session.exec(select(User.id)).all()) + + assert conversion.outcome is AccountConversionOutcome.MERGED + assert user_ids == {account_id} diff --git a/tests/unit/v2/test_report_run_routes.py b/tests/unit/v2/test_report_run_routes.py index f3a3fc3ef..1f906988c 100644 --- a/tests/unit/v2/test_report_run_routes.py +++ b/tests/unit/v2/test_report_run_routes.py @@ -323,7 +323,9 @@ def test_a_report_the_service_will_not_run_again_now_answers_409( def test_a_new_run_whose_report_cannot_be_read_back_answers_the_reports_id() -> None: service = FakeReportRunService( - error=RerunReportUnreadableError(report_id=REPORT_ID, country_id="us") + error=RerunReportUnreadableError( + report_id=REPORT_ID, run_id=NEW_RUN_ID, country_id="us" + ) ) response = report_route_client(reports=service).post(RUNS_PATH, json={}) @@ -332,11 +334,13 @@ def test_a_new_run_whose_report_cannot_be_read_back_answers_the_reports_id() -> assert response.json() == { "status": "error", "message": ( - f"The simulation service started a new run of report {REPORT_ID}, " + f"The simulation service started a run of report {REPORT_ID}, " "but the report could not be read back. Read it with GET " - f"/v2/reports/{REPORT_ID}?country_id=us rather than running it again." + f"/v2/reports/{REPORT_ID}?country_id=us&run_id={NEW_RUN_ID} " + "rather than running it again." ), "report_id": str(REPORT_ID), + "run_id": str(NEW_RUN_ID), }