diff --git a/pyrit/models/parameter.py b/pyrit/models/parameter.py index 251136882a..3af091d609 100644 --- a/pyrit/models/parameter.py +++ b/pyrit/models/parameter.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from collections.abc import Collection, Sequence from dataclasses import dataclass -from enum import Enum +from enum import Enum, StrEnum from pathlib import Path from typing import Any, Literal, Union, get_args, get_origin @@ -39,7 +39,7 @@ } -class ComponentType(str, Enum): +class ComponentType(StrEnum): """ The component family a registry reference resolves to. diff --git a/pyrit/models/results/scenario_result.py b/pyrit/models/results/scenario_result.py index 7d2d68d10c..575f11f7eb 100644 --- a/pyrit/models/results/scenario_result.py +++ b/pyrit/models/results/scenario_result.py @@ -6,7 +6,7 @@ import logging import uuid from datetime import UTC, datetime -from enum import Enum +from enum import StrEnum from typing import Any from pydantic import BaseModel, ConfigDict, Field, computed_field, model_validator @@ -39,12 +39,13 @@ ) -class ScenarioRunState(str, Enum): +class ScenarioRunState(StrEnum): """ Lifecycle state of a scenario run. - Inherits from ``str`` so values serialize naturally in Pydantic models and - REST responses, and compare equal to their string form. + A ``StrEnum``, so values serialize naturally in Pydantic models and REST + responses, compare equal to their string form, and format as their value + (``COMPLETED``, not ``ScenarioRunState.COMPLETED``) in messages and logs. """ CREATED = "CREATED" diff --git a/pyrit/setup/initializers/scorers.py b/pyrit/setup/initializers/scorers.py index caff82e85f..797479a7ad 100644 --- a/pyrit/setup/initializers/scorers.py +++ b/pyrit/setup/initializers/scorers.py @@ -18,7 +18,7 @@ import logging from collections.abc import Callable, Sequence -from enum import Enum +from enum import StrEnum from typing import TYPE_CHECKING, ClassVar, TypeVar, cast from azure.ai.contentsafety.models import TextCategory @@ -55,7 +55,7 @@ RequiredDependencyT = TypeVar("RequiredDependencyT") -class ScorerInitializerTags(str, Enum): +class ScorerInitializerTags(StrEnum): """Tags applied to scorer registry entries by ScorerInitializer.""" # Category tags — every scorer in the category gets this tag diff --git a/tests/unit/backend/test_scenario_run_routes.py b/tests/unit/backend/test_scenario_run_routes.py index ff536b81d4..a4853c2a7a 100644 --- a/tests/unit/backend/test_scenario_run_routes.py +++ b/tests/unit/backend/test_scenario_run_routes.py @@ -19,6 +19,7 @@ from pyrit.backend.main import app from pyrit.backend.models import PaginationInfo, ScenarioRunListResponse from pyrit.backend.routes.scenarios import get_scenario_run_progress, list_scenario_runs +from pyrit.backend.services import ScenarioRunService from pyrit.backend.services.scenario_run_service import ScenarioRunConflictError, ScenarioRunNotFoundError from pyrit.memory import MemoryInterface from pyrit.models import ( @@ -30,6 +31,7 @@ ScenarioProgressSummary, ScenarioQueueEntry, ScenarioQueueSnapshot, + ScenarioResult, ScenarioRunListItem, ScenarioRunPlan, ScenarioRunProgress, @@ -605,7 +607,7 @@ def test_cancel_completed_run_returns_409(self, client: TestClient) -> None: """Test that cancelling a completed run returns 409 Conflict.""" with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as mock_get: mock_service = MagicMock() - mock_service.cancel_run_async = AsyncMock(side_effect=ValueError("Cannot cancel run in 'completed' state.")) + mock_service.cancel_run_async = AsyncMock(side_effect=ValueError("Cannot cancel run in 'COMPLETED' state.")) mock_get.return_value = mock_service response = client.post("/api/scenarios/runs/test-run-id/cancel") @@ -613,6 +615,20 @@ def test_cancel_completed_run_returns_409(self, client: TestClient) -> None: assert response.status_code == status.HTTP_409_CONFLICT assert "Cannot cancel" in response.json()["detail"] + def test_cancel_completed_run_409_detail_names_state_by_value(self, client: TestClient) -> None: + """The 409 detail from the real service names the state by its value, not as ScenarioRunState.X.""" + header = MagicMock(spec=ScenarioResult) + header.scenario_run_state = ScenarioRunState.COMPLETED + memory = MagicMock(spec=MemoryInterface) + memory.get_scenario_result_header_async = AsyncMock(return_value=header) + with patch("pyrit.memory.CentralMemory.get_memory_instance", return_value=memory): + service = ScenarioRunService() + with patch("pyrit.backend.routes.scenarios.get_scenario_run_service", return_value=service): + response = client.post("/api/scenarios/runs/sr-done/cancel") + + assert response.status_code == status.HTTP_409_CONFLICT + assert response.json()["detail"] == "Cannot cancel run in 'COMPLETED' state." + class TestGetScenarioRunResultsRoute: """Tests for GET /api/scenarios/runs/{id}/results.""" @@ -668,7 +684,7 @@ def test_get_results_not_completed_returns_409(self, client: TestClient) -> None with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as mock_get: mock_service = MagicMock() mock_service.get_run_results_async = AsyncMock( - side_effect=ValueError("Results are only available for completed runs. Current status: 'running'.") + side_effect=ValueError("Results are only available for completed runs. Current status: 'IN_PROGRESS'.") ) mock_get.return_value = mock_service diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 1445502d19..f0f3114fcb 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -3658,7 +3658,7 @@ async def test_cancel_rechecks_terminal_state_with_headers_async( mock_memory.get_scenario_result_header_async.side_effect = [initial, latest] service = ScenarioRunService() - with pytest.raises(ValueError, match="Cannot cancel run"): + with pytest.raises(ValueError, match=rf"^Cannot cancel run in '{latest_state.value}' state\.$"): await service.cancel_run_async(scenario_result_id="sr-uuid-1") assert mock_memory.get_scenario_result_header_async.await_count == 2 @@ -4493,7 +4493,7 @@ async def test_cancel_completed_run_raises_value_error(self, mock_memory) -> Non mock_memory.get_scenario_result_header_async.return_value = db_result service = ScenarioRunService() - with pytest.raises(ValueError, match="Cannot cancel run"): + with pytest.raises(ValueError, match=r"^Cannot cancel run in 'COMPLETED' state\.$"): await service.cancel_run_async(scenario_result_id="sr-done") mock_memory.get_scenario_results_async.assert_not_awaited() @@ -4503,7 +4503,7 @@ async def test_cancel_already_cancelled_run_raises_value_error(self, mock_memory mock_memory.get_scenario_result_header_async.return_value = db_result service = ScenarioRunService() - with pytest.raises(ValueError, match="Cannot cancel run"): + with pytest.raises(ValueError, match=r"^Cannot cancel run in 'CANCELLED' state\.$"): await service.cancel_run_async(scenario_result_id="sr-cancelled") mock_memory.get_scenario_results_async.assert_not_awaited() @@ -4854,7 +4854,7 @@ async def test_get_results_raises_if_not_completed(self, mock_memory) -> None: mock_memory.get_scenario_results_async = AsyncMock(return_value=[db_result]) service = ScenarioRunService() - with pytest.raises(ValueError, match="only available for completed runs"): + with pytest.raises(ValueError, match=r"only available for completed runs\. Current status: 'IN_PROGRESS'\.$"): (await service.get_run_results_async(scenario_result_id="sr-running")) async def test_get_results_returns_details_for_completed_run(self, mock_memory) -> None: diff --git a/tests/unit/models/test_parameter.py b/tests/unit/models/test_parameter.py index 798b3e9c5d..663dd6ff08 100644 --- a/tests/unit/models/test_parameter.py +++ b/tests/unit/models/test_parameter.py @@ -3,8 +3,9 @@ """Unit tests for the unified Parameter model and its coercion methods.""" +import json from collections.abc import Callable, Collection, Sequence -from enum import Enum +from enum import Enum, StrEnum from pathlib import Path from typing import Any, Literal, Union, get_args @@ -1021,3 +1022,16 @@ def test_coercion_keeps_every_alternative(self) -> None: assert parameter.for_external_catalog().required assert parameter.coerce_value(["a", "b"]) == ["a", "b"] + + +@pytest.mark.parametrize("component_type", list(ComponentType)) +def test_component_type_formats_as_its_value(component_type: ComponentType) -> None: + value = component_type.value + assert isinstance(component_type, StrEnum) + assert str(component_type) == value + assert f"{component_type}" == value + assert format(component_type) == value + assert component_type == value + assert hash(component_type) == hash(value) + assert ComponentType(value) is component_type + assert json.dumps(component_type) == json.dumps(value) diff --git a/tests/unit/models/test_scenario_result.py b/tests/unit/models/test_scenario_result.py index 1f5e527af1..9ceb66869d 100644 --- a/tests/unit/models/test_scenario_result.py +++ b/tests/unit/models/test_scenario_result.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json import uuid from datetime import UTC, datetime +from enum import StrEnum import pytest from unit.mocks import make_scenario_result @@ -15,6 +17,7 @@ ConversationType, RetryEvent, ScenarioResult, + ScenarioRunState, ) @@ -307,3 +310,16 @@ def test_scenario_result_display_group_map_is_public_field(): # Mutable and writable (used by benchmark merge logic). result.display_group_map["foundry"] = "Foundry" assert result.display_group_map["foundry"] == "Foundry" + + +@pytest.mark.parametrize("state", list(ScenarioRunState)) +def test_scenario_run_state_formats_as_its_value(state: ScenarioRunState) -> None: + value = state.value + assert isinstance(state, StrEnum) + assert str(state) == value + assert f"{state}" == value + assert format(state) == value + assert state == value + assert hash(state) == hash(value) + assert ScenarioRunState(value) is state + assert json.dumps(state) == json.dumps(value) diff --git a/tests/unit/registry/test_resolution.py b/tests/unit/registry/test_resolution.py index 029b2d5374..b117abc03c 100644 --- a/tests/unit/registry/test_resolution.py +++ b/tests/unit/registry/test_resolution.py @@ -33,6 +33,7 @@ reject_non_external_params, resolve_constructor_args, resolve_declared_params, + resolve_reference_value, validate_declared_params, ) @@ -830,3 +831,11 @@ def test_resolvable_component_type_maps_to_its_registry(component_type: Componen @pytest.mark.parametrize("component_type", sorted(_NON_RESOLVABLE_COMPONENT_TYPES)) def test_non_resolvable_component_type_has_no_registry(component_type: ComponentType) -> None: assert _registry_getter_for_component_type(component_type) is None + + +@pytest.mark.parametrize("component_type", sorted(_NON_RESOLVABLE_COMPONENT_TYPES)) +def test_non_resolvable_component_type_error_names_type_by_value(component_type: ComponentType) -> None: + with pytest.raises(ValueError) as exc_info: + resolve_reference_value(component_type=component_type, value="some_name", owner="Owner", name="param") + + assert str(exc_info.value) == f"Owner.param: no registry is wired for component type '{component_type.value}'." diff --git a/tests/unit/setup/test_scorer_initializer.py b/tests/unit/setup/test_scorer_initializer.py index ce759aea55..b7c6f652c0 100644 --- a/tests/unit/setup/test_scorer_initializer.py +++ b/tests/unit/setup/test_scorer_initializer.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json import logging import os +from enum import StrEnum from threading import get_ident from unittest.mock import MagicMock, patch @@ -453,7 +455,8 @@ def test_best_objective_picks_highest_f1(self, *, names: tuple[str, str], caplog for tag in (ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER): assert [entry.name for entry in registry.instances.get_by_tag(tag=tag)] == ["higher"] - assert "F1=0.9000" in caplog.text + assert "Tagged higher as best_objective with F1=0.9000" in caplog.text + assert "ScorerInitializerTags." not in caplog.text assert "configured fallback" not in caplog.text @pytest.mark.parametrize( @@ -521,8 +524,9 @@ def test_best_objective_without_complete_metrics_uses_configured_fallback( [fallback] if fallback is not None else [] ) assert "No complete-coverage metrics qualified" in caplog.text + assert "ScorerInitializerTags." not in caplog.text if fallback is not None: - assert fallback in caplog.text + assert f"tagged {fallback} as best_objective using the configured fallback" in caplog.text assert "configured fallback, not an F1-backed winner" in caplog.text else: assert "no composite scorers available" in caplog.text @@ -844,3 +848,16 @@ async def test_likert_scorers_use_round_robin_when_available(self) -> None: assert len(likert_entries) > 0 for entry in likert_entries: assert entry.instance._prompt_target is rr_mock + + +@pytest.mark.parametrize("tag", list(ScorerInitializerTags)) +def test_scorer_initializer_tag_formats_as_its_value(tag: ScorerInitializerTags) -> None: + value = tag.value + assert isinstance(tag, StrEnum) + assert str(tag) == value + assert f"{tag}" == value + assert format(tag) == value + assert tag == value + assert hash(tag) == hash(value) + assert ScorerInitializerTags(value) is tag + assert json.dumps(tag) == json.dumps(value)