Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyrit/models/parameter.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from abc import ABC, abstractmethod
from collections.abc import Collection, Sequence
from dataclasses import dataclass
from enum import Enum
from enum import Enum, StrEnum
from pathlib import Path
from typing import Any, Literal, Union, get_args, get_origin

Expand Down Expand Up @@ -39,7 +39,7 @@
}


class ComponentType(str, Enum):
class ComponentType(StrEnum):
"""
The component family a registry reference resolves to.

Expand Down
9 changes: 5 additions & 4 deletions pyrit/models/results/scenario_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import logging
import uuid
from datetime import UTC, datetime
from enum import Enum
from enum import StrEnum
from typing import Any

from pydantic import BaseModel, ConfigDict, Field, computed_field, model_validator
Expand Down Expand Up @@ -39,12 +39,13 @@
)


class ScenarioRunState(str, Enum):
class ScenarioRunState(StrEnum):
"""
Lifecycle state of a scenario run.

Inherits from ``str`` so values serialize naturally in Pydantic models and
REST responses, and compare equal to their string form.
A ``StrEnum``, so values serialize naturally in Pydantic models and REST
responses, compare equal to their string form, and format as their value
(``COMPLETED``, not ``ScenarioRunState.COMPLETED``) in messages and logs.
"""

CREATED = "CREATED"
Expand Down
4 changes: 2 additions & 2 deletions pyrit/setup/initializers/scorers.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import logging
from collections.abc import Callable, Sequence
from enum import Enum
from enum import StrEnum
from typing import TYPE_CHECKING, ClassVar, TypeVar, cast

from azure.ai.contentsafety.models import TextCategory
Expand Down Expand Up @@ -55,7 +55,7 @@
RequiredDependencyT = TypeVar("RequiredDependencyT")


class ScorerInitializerTags(str, Enum):
class ScorerInitializerTags(StrEnum):
"""Tags applied to scorer registry entries by ScorerInitializer."""

# Category tags — every scorer in the category gets this tag
Expand Down
20 changes: 18 additions & 2 deletions tests/unit/backend/test_scenario_run_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from pyrit.backend.main import app
from pyrit.backend.models import PaginationInfo, ScenarioRunListResponse
from pyrit.backend.routes.scenarios import get_scenario_run_progress, list_scenario_runs
from pyrit.backend.services import ScenarioRunService
from pyrit.backend.services.scenario_run_service import ScenarioRunConflictError, ScenarioRunNotFoundError
from pyrit.memory import MemoryInterface
from pyrit.models import (
Expand All @@ -30,6 +31,7 @@
ScenarioProgressSummary,
ScenarioQueueEntry,
ScenarioQueueSnapshot,
ScenarioResult,
ScenarioRunListItem,
ScenarioRunPlan,
ScenarioRunProgress,
Expand Down Expand Up @@ -605,14 +607,28 @@ def test_cancel_completed_run_returns_409(self, client: TestClient) -> None:
"""Test that cancelling a completed run returns 409 Conflict."""
with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as mock_get:
mock_service = MagicMock()
mock_service.cancel_run_async = AsyncMock(side_effect=ValueError("Cannot cancel run in 'completed' state."))
mock_service.cancel_run_async = AsyncMock(side_effect=ValueError("Cannot cancel run in 'COMPLETED' state."))
mock_get.return_value = mock_service

response = client.post("/api/scenarios/runs/test-run-id/cancel")

assert response.status_code == status.HTTP_409_CONFLICT
assert "Cannot cancel" in response.json()["detail"]

def test_cancel_completed_run_409_detail_names_state_by_value(self, client: TestClient) -> None:
"""The 409 detail from the real service names the state by its value, not as ScenarioRunState.X."""
header = MagicMock(spec=ScenarioResult)
header.scenario_run_state = ScenarioRunState.COMPLETED
memory = MagicMock(spec=MemoryInterface)
memory.get_scenario_result_header_async = AsyncMock(return_value=header)
with patch("pyrit.memory.CentralMemory.get_memory_instance", return_value=memory):
service = ScenarioRunService()
with patch("pyrit.backend.routes.scenarios.get_scenario_run_service", return_value=service):
response = client.post("/api/scenarios/runs/sr-done/cancel")

assert response.status_code == status.HTTP_409_CONFLICT
assert response.json()["detail"] == "Cannot cancel run in 'COMPLETED' state."


class TestGetScenarioRunResultsRoute:
"""Tests for GET /api/scenarios/runs/{id}/results."""
Expand Down Expand Up @@ -668,7 +684,7 @@ def test_get_results_not_completed_returns_409(self, client: TestClient) -> None
with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as mock_get:
mock_service = MagicMock()
mock_service.get_run_results_async = AsyncMock(
side_effect=ValueError("Results are only available for completed runs. Current status: 'running'.")
side_effect=ValueError("Results are only available for completed runs. Current status: 'IN_PROGRESS'.")
)
mock_get.return_value = mock_service

Expand Down
8 changes: 4 additions & 4 deletions tests/unit/backend/test_scenario_run_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -3658,7 +3658,7 @@ async def test_cancel_rechecks_terminal_state_with_headers_async(
mock_memory.get_scenario_result_header_async.side_effect = [initial, latest]
service = ScenarioRunService()

with pytest.raises(ValueError, match="Cannot cancel run"):
with pytest.raises(ValueError, match=rf"^Cannot cancel run in '{latest_state.value}' state\.$"):
await service.cancel_run_async(scenario_result_id="sr-uuid-1")

assert mock_memory.get_scenario_result_header_async.await_count == 2
Expand Down Expand Up @@ -4493,7 +4493,7 @@ async def test_cancel_completed_run_raises_value_error(self, mock_memory) -> Non
mock_memory.get_scenario_result_header_async.return_value = db_result

service = ScenarioRunService()
with pytest.raises(ValueError, match="Cannot cancel run"):
with pytest.raises(ValueError, match=r"^Cannot cancel run in 'COMPLETED' state\.$"):
await service.cancel_run_async(scenario_result_id="sr-done")
mock_memory.get_scenario_results_async.assert_not_awaited()

Expand All @@ -4503,7 +4503,7 @@ async def test_cancel_already_cancelled_run_raises_value_error(self, mock_memory
mock_memory.get_scenario_result_header_async.return_value = db_result

service = ScenarioRunService()
with pytest.raises(ValueError, match="Cannot cancel run"):
with pytest.raises(ValueError, match=r"^Cannot cancel run in 'CANCELLED' state\.$"):
await service.cancel_run_async(scenario_result_id="sr-cancelled")
mock_memory.get_scenario_results_async.assert_not_awaited()

Expand Down Expand Up @@ -4854,7 +4854,7 @@ async def test_get_results_raises_if_not_completed(self, mock_memory) -> None:
mock_memory.get_scenario_results_async = AsyncMock(return_value=[db_result])

service = ScenarioRunService()
with pytest.raises(ValueError, match="only available for completed runs"):
with pytest.raises(ValueError, match=r"only available for completed runs\. Current status: 'IN_PROGRESS'\.$"):
(await service.get_run_results_async(scenario_result_id="sr-running"))

async def test_get_results_returns_details_for_completed_run(self, mock_memory) -> None:
Expand Down
16 changes: 15 additions & 1 deletion tests/unit/models/test_parameter.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@

"""Unit tests for the unified Parameter model and its coercion methods."""

import json
from collections.abc import Callable, Collection, Sequence
from enum import Enum
from enum import Enum, StrEnum
from pathlib import Path
from typing import Any, Literal, Union, get_args

Expand Down Expand Up @@ -1021,3 +1022,16 @@ def test_coercion_keeps_every_alternative(self) -> None:

assert parameter.for_external_catalog().required
assert parameter.coerce_value(["a", "b"]) == ["a", "b"]


@pytest.mark.parametrize("component_type", list(ComponentType))
def test_component_type_formats_as_its_value(component_type: ComponentType) -> None:
value = component_type.value
assert isinstance(component_type, StrEnum)
assert str(component_type) == value
assert f"{component_type}" == value
assert format(component_type) == value
assert component_type == value
assert hash(component_type) == hash(value)
assert ComponentType(value) is component_type
assert json.dumps(component_type) == json.dumps(value)
16 changes: 16 additions & 0 deletions tests/unit/models/test_scenario_result.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import json
import uuid
from datetime import UTC, datetime
from enum import StrEnum

import pytest
from unit.mocks import make_scenario_result
Expand All @@ -15,6 +17,7 @@
ConversationType,
RetryEvent,
ScenarioResult,
ScenarioRunState,
)


Expand Down Expand Up @@ -307,3 +310,16 @@ def test_scenario_result_display_group_map_is_public_field():
# Mutable and writable (used by benchmark merge logic).
result.display_group_map["foundry"] = "Foundry"
assert result.display_group_map["foundry"] == "Foundry"


@pytest.mark.parametrize("state", list(ScenarioRunState))
def test_scenario_run_state_formats_as_its_value(state: ScenarioRunState) -> None:
value = state.value
assert isinstance(state, StrEnum)
assert str(state) == value
assert f"{state}" == value
assert format(state) == value
assert state == value
assert hash(state) == hash(value)
assert ScenarioRunState(value) is state
assert json.dumps(state) == json.dumps(value)
9 changes: 9 additions & 0 deletions tests/unit/registry/test_resolution.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
reject_non_external_params,
resolve_constructor_args,
resolve_declared_params,
resolve_reference_value,
validate_declared_params,
)

Expand Down Expand Up @@ -830,3 +831,11 @@ def test_resolvable_component_type_maps_to_its_registry(component_type: Componen
@pytest.mark.parametrize("component_type", sorted(_NON_RESOLVABLE_COMPONENT_TYPES))
def test_non_resolvable_component_type_has_no_registry(component_type: ComponentType) -> None:
assert _registry_getter_for_component_type(component_type) is None


@pytest.mark.parametrize("component_type", sorted(_NON_RESOLVABLE_COMPONENT_TYPES))
def test_non_resolvable_component_type_error_names_type_by_value(component_type: ComponentType) -> None:
with pytest.raises(ValueError) as exc_info:
resolve_reference_value(component_type=component_type, value="some_name", owner="Owner", name="param")

assert str(exc_info.value) == f"Owner.param: no registry is wired for component type '{component_type.value}'."
21 changes: 19 additions & 2 deletions tests/unit/setup/test_scorer_initializer.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import json
import logging
import os
from enum import StrEnum
from threading import get_ident
from unittest.mock import MagicMock, patch

Expand Down Expand Up @@ -453,7 +455,8 @@ def test_best_objective_picks_highest_f1(self, *, names: tuple[str, str], caplog

for tag in (ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER):
assert [entry.name for entry in registry.instances.get_by_tag(tag=tag)] == ["higher"]
assert "F1=0.9000" in caplog.text
assert "Tagged higher as best_objective with F1=0.9000" in caplog.text
assert "ScorerInitializerTags." not in caplog.text
assert "configured fallback" not in caplog.text

@pytest.mark.parametrize(
Expand Down Expand Up @@ -521,8 +524,9 @@ def test_best_objective_without_complete_metrics_uses_configured_fallback(
[fallback] if fallback is not None else []
)
assert "No complete-coverage metrics qualified" in caplog.text
assert "ScorerInitializerTags." not in caplog.text
if fallback is not None:
assert fallback in caplog.text
assert f"tagged {fallback} as best_objective using the configured fallback" in caplog.text
assert "configured fallback, not an F1-backed winner" in caplog.text
else:
assert "no composite scorers available" in caplog.text
Expand Down Expand Up @@ -844,3 +848,16 @@ async def test_likert_scorers_use_round_robin_when_available(self) -> None:
assert len(likert_entries) > 0
for entry in likert_entries:
assert entry.instance._prompt_target is rr_mock


@pytest.mark.parametrize("tag", list(ScorerInitializerTags))
def test_scorer_initializer_tag_formats_as_its_value(tag: ScorerInitializerTags) -> None:
value = tag.value
assert isinstance(tag, StrEnum)
assert str(tag) == value
assert f"{tag}" == value
assert format(tag) == value
assert tag == value
assert hash(tag) == hash(value)
assert ScorerInitializerTags(value) is tag
assert json.dumps(tag) == json.dumps(value)
Loading