# Copyright 2019, 2021-2024 The Debusine Developers
# See the AUTHORS file at the top-level directory of this distribution
#
# This file is part of Debusine. It is subject to the license terms
# in the LICENSE file found in the top-level directory of this
# distribution. No part of Debusine, including this file, may be copied,
# modified, propagated, or distributed except according to the terms
# contained in the LICENSE file.

"""Unit tests for the work request models."""

import io
import logging
from datetime import datetime, timedelta
from typing import Any, ClassVar
from unittest import mock

from asgiref.sync import sync_to_async
from django.core.exceptions import ValidationError
from django.core.serializers.json import DjangoJSONEncoder
from django.db import transaction
from django.test import override_settings
from django.utils import timezone

try:
    import pydantic.v1 as pydantic
except ImportError:
    import pydantic as pydantic  # type: ignore

from debusine.artifacts.models import (
    ArtifactCategory,
    BareDataCategory,
    CollectionCategory,
)
from debusine.db.models import (
    Collection,
    CollectionItem,
    Token,
    User,
    WorkRequest,
    Worker,
    WorkflowTemplate,
    default_workspace,
)
from debusine.db.models.work_requests import (
    CannotRetry,
    InternalTaskError,
    WorkersPendingWorkRequests,
)
from debusine.server.collections import CollectionManagerInterface
from debusine.server.management.commands.delete_expired import (
    DeleteExpiredWorkRequests,
    DeleteOperation,
)
from debusine.server.tasks import ServerNoop
from debusine.server.tasks.wait.delay import Delay
from debusine.server.workflows import Workflow
from debusine.server.workflows.models import (
    BaseWorkflowData,
    WorkRequestWorkflowData,
)
from debusine.server.workflows.noop import NoopWorkflow
from debusine.signing.tasks import SigningNoop
from debusine.tasks import TaskConfigError
from debusine.tasks.models import (
    ActionRetryWithDelays,
    ActionUpdateCollectionWithArtifacts,
    ActionUpdateCollectionWithData,
    BaseDynamicTaskData,
    EventReactions,
    TaskTypes,
)
from debusine.tasks.noop import Noop
from debusine.tasks.server import TaskDatabaseInterface
from debusine.tasks.tests.helper_mixin import TestTaskMixin
from debusine.tasks.tests.test_simplesystemimagebuild import (
    SimpleSystemImageBuildTests,
)
from debusine.test.django import ChannelsHelpersMixin, TestCase
from debusine.test.utils import preserve_task_registry


class TestWorkflowData(BaseWorkflowData):
    """Workflow data to test instantiation."""

    ivar: int = 1
    dvar: dict[str, Any] = pydantic.Field(default_factory=dict)


class WorkRequestManagerTestsTestWorkflow(
    TestTaskMixin, Workflow[TestWorkflowData, BaseDynamicTaskData]
):
    """Workflow used to test instantiation of data."""

    # This needs a unique name to avoid conflicting with other tests, because
    # it gets registered as a Workflow subclass

    def populate(self) -> None:
        """Unused abstract method from Workflow."""
        raise NotImplementedError()


class WorkRequestManagerTests(TestCase):
    """Tests for WorkRequestManager."""

    def test_create_workflow(self) -> None:
        """Workflows are created correctly."""
        user = self.get_test_user()

        workflow_name = "workrequestmanagerteststestworkflow"

        template = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name=workflow_name,
            task_data={},
            priority=10,
        )

        wr = WorkRequest.objects.create_workflow(
            template=template, created_by=user, data={}
        )
        self.assertEqual(wr.workspace, template.workspace)
        self.assertIsNone(wr.parent)
        self.assertEqual(wr.created_by, user)
        self.assertEqual(wr.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr.task_type, TaskTypes.WORKFLOW)
        self.assertEqual(wr.task_name, workflow_name)
        self.assertEqual(wr.task_data, {})
        self.assertEqual(wr.priority_base, 10)
        self.assertQuerysetEqual(wr.dependencies.all(), [])
        assert wr.internal_collection is not None
        self.assertEqual(wr.internal_collection.name, f"workflow-{wr.id}")
        self.assertEqual(
            wr.internal_collection.category,
            CollectionCategory.WORKFLOW_INTERNAL,
        )
        self.assertEqual(wr.internal_collection.workspace, template.workspace)
        self.assertEqual(
            wr.internal_collection.retains_artifacts,
            Collection.RetainsArtifacts.WORKFLOW,
        )
        self.assertEqual(wr.internal_collection.workflow, wr)

    def test_create_workflow_with_parent(self) -> None:
        """A sub-workflow does not have its own internal collection."""
        user = self.get_test_user()
        template = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name="workrequestmanagerteststestworkflow",
        )
        root = WorkRequest.objects.create_workflow(
            template=template, created_by=user, data={}
        )
        sub_template = WorkflowTemplate.objects.create(
            name="subtest", workspace=default_workspace(), task_name="noop"
        )
        sub = WorkRequest.objects.create_workflow(
            template=sub_template, created_by=user, data={}, parent=root
        )

        self.assertEqual(sub.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(sub.parent, root)
        self.assertQuerysetEqual(sub.dependencies.all(), [])
        self.assertIsNone(sub.internal_collection)

    def test_create_workflow_with_parms(self) -> None:
        """Check that WorkflowTemplate parameters have precedence."""
        user = self.get_test_user()

        workflow_name = "workrequestmanagerteststestworkflow"

        template = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name=workflow_name,
            task_data={"ivar": 2, "dvar": {"a": 1}},
        )

        wr = WorkRequest.objects.create_workflow(
            template=template, created_by=user, data={}
        )
        self.assertEqual(wr.task_data, {"ivar": 2, "dvar": {"a": 1}})

        wr = WorkRequest.objects.create_workflow(
            template=template,
            created_by=user,
            data={"ivar": 3, "dvar": {"b": 2}},
        )
        self.assertEqual(wr.task_data, {"ivar": 2, "dvar": {"a": 1}})

        template.task_data = {"ivar": 1}
        wr = WorkRequest.objects.create_workflow(
            template=template,
            created_by=user,
            data={"ivar": 3, "dvar": {"b": 2}},
        )
        self.assertEqual(wr.task_data, {"ivar": 1, "dvar": {"b": 2}})

    def test_create_workflow_validation(self) -> None:
        """Check validation of parameters on workflow instantiation."""
        user = self.get_test_user()

        workflow_name = "workrequestmanagerteststestworkflow"

        template = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name=workflow_name,
            task_data={},
        )

        with self.assertLogsContains(
            "Cannot create a workflow",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            wr = WorkRequest.objects.create_workflow(
                template=template,
                created_by=user,
                data={"noise": 1},
            )
        self.assertEqual(wr.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(wr.result, WorkRequest.Results.ERROR)

    def test_create_synchronization_point(self) -> None:
        """Synchronization points are created correctly."""
        user = self.get_test_user()

        parent = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=user,
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
            task_data={},
            priority_base=5,
            priority_adjustment=1,
        )

        wr = WorkRequest.objects.create_synchronization_point(
            parent=parent, step="test"
        )
        self.assertEqual(wr.workspace, parent.workspace)
        self.assertEqual(wr.parent, parent)
        self.assertIsNotNone(wr.created_at)
        self.assertIsNone(wr.started_at)
        self.assertIsNone(wr.completed_at)
        self.assertEqual(wr.created_by, user)
        self.assertEqual(wr.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr.result, WorkRequest.Results.NONE)
        self.assertIsNone(wr.worker)
        self.assertEqual(wr.task_type, TaskTypes.INTERNAL)
        self.assertEqual(wr.task_name, "synchronization_point")
        self.assertEqual(wr.task_data, {})
        self.assertEqual(wr.priority_base, 6)
        self.assertQuerysetEqual(wr.dependencies.all(), [])
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(step="test", display_name="test"),
        )
        wr.full_clean()

        workspace = self.playground.create_workspace(name="test")

        parent = WorkRequest.objects.create(
            workspace=workspace,
            created_by=user,
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
            task_data={},
        )
        wr = WorkRequest.objects.create_synchronization_point(
            parent=parent,
            step="test",
            display_name="Test",
            status=WorkRequest.Statuses.PENDING,
        )
        self.assertEqual(wr.workspace, workspace)
        self.assertEqual(wr.parent, parent)
        self.assertIsNotNone(wr.created_at)
        self.assertIsNone(wr.started_at)
        self.assertIsNone(wr.completed_at)
        self.assertEqual(wr.created_by, user)
        self.assertEqual(wr.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr.result, WorkRequest.Results.NONE)
        self.assertIsNone(wr.worker)
        self.assertEqual(wr.task_type, TaskTypes.INTERNAL)
        self.assertEqual(wr.task_name, "synchronization_point")
        self.assertEqual(wr.task_data, {})
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(step="test", display_name="Test"),
        )
        wr.full_clean()

    def test_create_workflow_callback(self) -> None:
        """Workflow callbacks are created correctly."""
        user = self.get_test_user()

        parent = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=user,
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
            task_data={},
        )

        wr = WorkRequest.objects.create_workflow_callback(
            parent=parent, step="test"
        )
        self.assertEqual(wr.workspace, parent.workspace)
        self.assertEqual(wr.parent, parent)
        self.assertIsNotNone(wr.created_at)
        self.assertIsNone(wr.started_at)
        self.assertIsNone(wr.completed_at)
        self.assertEqual(wr.created_by, user)
        self.assertEqual(wr.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr.result, WorkRequest.Results.NONE)
        self.assertIsNone(wr.worker)
        self.assertEqual(wr.task_type, TaskTypes.INTERNAL)
        self.assertEqual(wr.task_name, "workflow")
        self.assertEqual(wr.task_data, {})
        self.assertEqual(wr.priority_base, 0)
        self.assertQuerysetEqual(wr.dependencies.all(), [])
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(step="test", display_name="test"),
        )
        wr.full_clean()

        workspace = self.playground.create_workspace(name="test")

        parent = WorkRequest.objects.create(
            workspace=workspace,
            created_by=user,
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
            task_data={},
            priority_base=20,
            priority_adjustment=10,
        )
        wr = WorkRequest.objects.create_workflow_callback(
            parent=parent,
            status=WorkRequest.Statuses.PENDING,
            step="test1",
            display_name="Test One",
        )
        self.assertEqual(wr.workspace, workspace)
        self.assertEqual(wr.parent, parent)
        self.assertIsNotNone(wr.created_at)
        self.assertIsNone(wr.started_at)
        self.assertIsNone(wr.completed_at)
        self.assertEqual(wr.created_by, user)
        self.assertEqual(wr.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr.result, WorkRequest.Results.NONE)
        self.assertIsNone(wr.worker)
        self.assertEqual(wr.task_type, TaskTypes.INTERNAL)
        self.assertEqual(wr.task_name, "workflow")
        self.assertEqual(wr.task_data, {})
        self.assertEqual(wr.priority_base, 30)
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(step="test1", display_name="Test One"),
        )
        wr.full_clean()

    def test_pending(self) -> None:
        """WorkRequestManager.pending() returns pending WorkRequests."""
        work_request_1 = self.create_work_request(
            status=WorkRequest.Statuses.PENDING
        )

        work_request_2 = self.create_work_request(
            status=WorkRequest.Statuses.PENDING
        )

        self.create_work_request(status=WorkRequest.Statuses.ABORTED)

        self.assertQuerysetEqual(
            WorkRequest.objects.pending(), [work_request_1, work_request_2]
        )

        work_request_1.created_at = timezone.now()
        work_request_1.save()

        self.assertQuerysetEqual(
            WorkRequest.objects.pending(), [work_request_2, work_request_1]
        )

        work_request_1.priority_adjustment = 1
        work_request_1.save()

        self.assertQuerysetEqual(
            WorkRequest.objects.pending(), [work_request_1, work_request_2]
        )

        work_request_2.priority_base = 2
        work_request_2.save()

        self.assertQuerysetEqual(
            WorkRequest.objects.pending(), [work_request_2, work_request_1]
        )

    def test_pending_filter_by_worker(self) -> None:
        """WorkRequestManager.pending() returns WorkRequest for the worker."""
        self.create_work_request(
            status=WorkRequest.Statuses.PENDING, task_name="sbuild"
        )

        worker = Worker.objects.create_with_fqdn(
            "computer.lan", Token.objects.create()
        )

        work_request_2 = self.create_work_request(
            status=WorkRequest.Statuses.PENDING, worker=worker
        )

        self.assertQuerysetEqual(
            WorkRequest.objects.pending(worker=worker), [work_request_2]
        )

    def test_raise_value_error_exclude_assigned_and_worker(self) -> None:
        """WorkRequestManager.pending() raises ValueError."""
        worker = Worker.objects.create_with_fqdn(
            "computer.lan", Token.objects.create()
        )

        with self.assertRaisesRegex(
            ValueError, "Cannot exclude_assigned and filter by worker"
        ):
            WorkRequest.objects.pending(exclude_assigned=True, worker=worker)

    def test_pending_exclude_assigned(self) -> None:
        """
        Test WorkRequestManager.pending(exclude_assigned=True).

        It excludes work requests that are assigned to a worker.
        """
        # Pending, not assigned to a worker WorkRequest
        work_request = self.create_work_request(
            status=WorkRequest.Statuses.PENDING, task_name="sbuild"
        )

        # Is returned as expected
        self.assertQuerysetEqual(
            WorkRequest.objects.pending(exclude_assigned=True), [work_request]
        )

        # Creates a worker
        worker = Worker.objects.create_with_fqdn(
            'test', token=Token.objects.create()
        )

        # Assigns the worker to the work_request
        work_request.worker = worker
        work_request.save()

        # pending(exclude_assigned=True) doesn't return it anymore
        self.assertQuerysetEqual(
            WorkRequest.objects.pending(exclude_assigned=True), []
        )

        # without the exclude_assigned it returns it
        self.assertQuerysetEqual(
            WorkRequest.objects.pending(exclude_assigned=False), [work_request]
        )

    def test_running(self) -> None:
        """WorkRequestManager.running() returns running WorkRequests."""
        work_request = self.create_work_request(
            status=WorkRequest.Statuses.RUNNING
        )
        self.create_work_request(status=WorkRequest.Statuses.ABORTED)

        self.assertQuerysetEqual(WorkRequest.objects.running(), [work_request])

    def test_completed(self) -> None:
        """WorkRequestManager.completed() returns completed WorkRequests."""
        work_request = self.create_work_request(
            status=WorkRequest.Statuses.COMPLETED
        )
        self.create_work_request(status=WorkRequest.Statuses.RUNNING)

        self.assertQuerysetEqual(
            WorkRequest.objects.completed(), [work_request]
        )

    def test_aborted(self) -> None:
        """WorkRequestManager.aborted() returns aborted WorkRequests."""
        work_request = self.create_work_request(
            status=WorkRequest.Statuses.ABORTED
        )
        self.create_work_request(status=WorkRequest.Statuses.RUNNING)

        self.assertQuerysetEqual(WorkRequest.objects.aborted(), [work_request])

    def test_aborted_or_failed(self) -> None:
        """WorkRequestManager.aborted() returns aborted WorkRequests."""
        wr_aborted = self.create_work_request(
            status=WorkRequest.Statuses.ABORTED
        )
        wr_failed = self.create_work_request(
            result=WorkRequest.Results.FAILURE,
        )
        self.create_work_request(status=WorkRequest.Statuses.RUNNING)

        self.assertQuerysetEqual(
            WorkRequest.objects.aborted_or_failed(), [wr_aborted, wr_failed]
        )

    def test_expired(self) -> None:
        """Test expired() method."""
        wr = self.create_work_request(task_name="noop")

        wr.created_at = timezone.now() - timedelta(days=2)
        wr.expiration_delay = timedelta(days=1)
        wr.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), [wr]
        )

        wr.expiration_delay = timedelta(0)
        wr.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), []
        )

        wr.created_at = timezone.now()
        wr.expiration_delay = timedelta(days=1)
        wr.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), []
        )

        workspace = self.playground.create_workspace(name="test")
        workspace.default_expiration_delay = timedelta(days=2)
        workspace.save()
        wr.workspace = workspace
        wr.created_at = timezone.now() - timedelta(days=3)
        wr.expiration_delay = None
        wr.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), [wr]
        )

        workspace.default_expiration_delay = timedelta(days=5)
        workspace.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), []
        )

        workspace.default_expiration_delay = timedelta(days=0)
        workspace.save()
        self.assertQuerysetEqual(
            WorkRequest.objects.expired(timezone.now()), []
        )

    @override_settings(DISABLE_AUTOMATIC_SCHEDULING=True)
    # no scheduling: created work requests stay in "pending" instead of
    # being configured (and potentially failed depending on the task_data)
    def test_get_architecture_status_host_architecture_in_task_data(self):
        """
        get_architecture_status with work requests with "host_architecture".

        Test that connected workers are counted as available.
        """
        # Create three work requests of type sbuild
        self.playground.create_work_request(
            task_name="sbuild",
            task_data={"host_architecture": "amd64"},
            status=WorkRequest.Statuses.PENDING,
        )
        self.playground.create_work_request(
            task_name="sbuild",
            task_data={"host_architecture": "amd64"},
            status=WorkRequest.Statuses.PENDING,
        )
        self.playground.create_work_request(
            task_name="sbuild",
            task_data={"host_architecture": "arm64"},
            status=WorkRequest.Statuses.PENDING,
        )

        # Create two connected workers
        worker_amd64_1 = self.playground.create_worker(
            extra_dynamic_metadata={
                "system:architecture": "amd64",
                "system:architectures": ["amd64", "i386"],
            }
        )
        worker_amd64_1.mark_connected()

        worker_arm64_1 = self.playground.create_worker(
            extra_dynamic_metadata={
                "system:architecture": "arm64",
                "system:architectures": ["arm64"],
            }
        )
        worker_arm64_1.mark_connected()

        worker_arm64_2 = self.playground.create_worker(
            extra_dynamic_metadata={
                "system:architecture": "arm64",
                "system:architectures": ["arm64"],
            }
        )
        worker_arm64_2.mark_connected()

        # Create non-connected worker (will not appear as available
        # worker in the count for arm64)
        self.playground.create_worker(
            extra_dynamic_metadata={
                "system:architecture": "arm64",
                "system:architectures": ["arm64"],
            }
        )

        actual = WorkRequest.objects.get_architecture_status()

        expected = {
            "amd64": WorkersPendingWorkRequests(
                workers=1, work_requests_pending=2
            ),
            "arm64": WorkersPendingWorkRequests(
                workers=2, work_requests_pending=1
            ),
            "i386": WorkersPendingWorkRequests(
                workers=1, work_requests_pending=0
            ),
        }

        self.assertEqual(actual, expected)

    @override_settings(DISABLE_AUTOMATIC_SCHEDULING=True)
    # no scheduling: created work requests stay in "pending" instead of
    # being configured (and potentially failed depending on the task_data)
    def test_get_architecture_status_host_architecture_not_in_task_data(
        self,
    ):
        """
        get_architecture_status work requests without "host_architecture".

        Work requests without system_architecture. Internally the
        Task.host_architecture() method is used.
        """
        # Create one work request of type simplesystemimagebuild
        expected_architecture = SimpleSystemImageBuildTests.SAMPLE_TASK_DATA[
            "bootstrap_options"
        ]["architecture"]

        self.playground.create_work_request(
            task_name="simplesystemimagebuild",
            task_data=SimpleSystemImageBuildTests.SAMPLE_TASK_DATA,
            status=WorkRequest.Statuses.PENDING,
        )

        actual = WorkRequest.objects.get_architecture_status()

        expected = {
            expected_architecture: WorkersPendingWorkRequests(
                workers=0, work_requests_pending=1
            ),
        }

        self.assertEqual(actual, expected)

    def test_get_architecture_status_no_host_architecture(self):
        """
        get_architecture_status: work request without architecture.

        Work request without "host_architecture" and host_architecture()"
        returns None.
        """
        self.playground.create_work_request(
            task_name="noop",
            status=WorkRequest.Statuses.PENDING,
        )

        actual = WorkRequest.objects.get_architecture_status()

        self.assertEqual(actual, {})

    def test_get_architecture_status_no_pending_work_requests(self):
        """
        get_architecture_status: no pending work requests.

        Empty dictionary because no pending work requests and no workers.
        """
        self.playground.create_work_request(
            task_name="sbuild",
            task_data={"host_architecture": "amd64"},
            status=WorkRequest.Statuses.RUNNING,
        )

        self.assertEqual(WorkRequest.objects.get_architecture_status(), {})

    def test_get_architecture_status_without_work_requests_connected_workers(
        self,
    ):
        """
        Architectures without work requests neither connected worker.

        Those architectures are listed with workers=0 work_requests_pending=0
        """
        self.playground.create_worker(
            extra_dynamic_metadata={"system:architectures": ["arch1", "arch2"]}
        )

        worker = self.playground.create_worker(
            extra_dynamic_metadata={"system:architectures": ["arch3"]}
        )
        worker.token.disable()

        self.assertEqual(
            WorkRequest.objects.get_architecture_status(),
            {
                "arch1": WorkersPendingWorkRequests(
                    workers=0, work_requests_pending=0
                ),
                "arch2": WorkersPendingWorkRequests(
                    workers=0, work_requests_pending=0
                ),
            },
        )


class WorkRequestTests(TestCase):
    """Tests for the WorkRequest class."""

    user: ClassVar[User]
    template: ClassVar[WorkflowTemplate]
    workflow: ClassVar[WorkRequest]

    @classmethod
    def setUpTestData(cls) -> None:
        """Add a sample user to the test fixture."""
        super().setUpTestData()
        cls.user = cls.playground.get_default_user()
        cls.template = cls.playground.create_workflow_template(
            name="test",
            task_name="noop",
        )
        cls.workflow = WorkRequest.objects.create_workflow(
            template=cls.template, data={}, created_by=cls.user
        )

    def assert_data_must_be_dictionary(
        self, wr: WorkRequest, messages: list[str] | None = None
    ):
        """Check that a WorkRequest's task_data is a dictionary."""
        if messages is None:
            messages = ["task data must be a dictionary"]
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()
        self.assertEqual(
            raised.exception.message_dict,
            {"task_data": messages},
        )

    def test_clean_task_data_must_be_dict(self) -> None:
        """Ensure that task_data is a dict."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_name="noop",
            task_data={},
        )
        wr.full_clean()

        wr.task_data = None
        self.assert_data_must_be_dictionary(wr)

        wr.task_data = ""
        self.assert_data_must_be_dictionary(wr)

        wr.task_data = 3
        self.assert_data_must_be_dictionary(wr)

        wr.task_data = []
        self.assert_data_must_be_dictionary(wr)

        wr.task_data = object()
        self.assert_data_must_be_dictionary(
            wr,
            messages=[
                'Value must be valid JSON.',
                'task data must be a dictionary',
            ],
        )

    def test_clean_task_name_must_be_valid_worker(self) -> None:
        """Check validation of worker task names."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_name="noop",
            task_data={},
        )
        wr.full_clean()

        wr.task_name = "does-not-exist"
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()

        self.assertEqual(
            raised.exception.message_dict,
            {"task_name": ['does-not-exist: invalid Worker task name']},
        )

        wr.task_name = "does_not_exist"
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()

        self.assertEqual(
            raised.exception.message_dict,
            {
                "task_name": [
                    "does_not_exist: invalid Worker task name",
                ]
            },
        )

    def test_clean_task_name_must_be_valid_server(self) -> None:
        """Check validation of server task names."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_type=TaskTypes.SERVER,
            task_name="servernoop",
            task_data={},
        )
        wr.full_clean()

        wr.task_name = "does-not-exist"
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()

        self.assertEqual(
            raised.exception.message_dict,
            {"task_name": ['does-not-exist: invalid Server task name']},
        )

    def test_clean_task_name_must_be_valid_internal(self) -> None:
        """Check validation of internal task names."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_type=TaskTypes.INTERNAL,
            task_name="synchronization_point",
            task_data={},
        )
        wr.full_clean()

        wr.task_name = "workflow"
        wr.full_clean()

        wr.task_name = "does_not_exist"
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()
        self.assertEqual(
            raised.exception.message_dict,
            {
                "task_name": [
                    "does_not_exist: invalid task name for internal task"
                ]
            },
        )

    def test_clean_task_name_must_be_valid_workflow(self) -> None:
        """Check validation of workflow task names."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_type=TaskTypes.WORKFLOW,
            task_name="servernoop",
            task_data={},
        )
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()
        self.assertEqual(
            raised.exception.message_dict,
            {"task_name": ["servernoop: invalid workflow name"]},
        )

        wr.task_name = "noop"
        wr.full_clean()

    def test_clean_task_name_must_be_valid_signing(self) -> None:
        """Check validation of signing task names."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_type=TaskTypes.SIGNING,
            task_name="generatekey",
            task_data={"purpose": "uefi", "description": "A UEFI key"},
        )
        wr.full_clean()

        wr.task_name = "does-not-exist"
        with self.assertRaises(ValidationError) as raised:
            wr.full_clean()

        self.assertEqual(
            raised.exception.message_dict,
            {"task_name": ['does-not-exist: invalid Signing task name']},
        )

    def test_clean_invalid_task_type(self) -> None:
        """An invalid work request task type is rejected."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.user,
            task_type="invalid",
            task_name="noop",
            task_data={},
        )
        with self.assertRaises(NotImplementedError):
            wr.full_clean()

    def test_clean_data_task_data_raise_validation_error(self) -> None:
        """Check validation of task data."""

        def make_task_data():
            return {
                "build_components": ["any", "all"],
                "host_architecture": "amd64",
                "environment": (
                    "debian/match:codename=bookworm:architecture=amd64"
                ),
                "input": {
                    "source_artifact": 5,
                },
            }

        wr = WorkRequest.objects.create(
            created_by=self.user,
            workspace=default_workspace(),
            task_name="sbuild",
            task_data=make_task_data(),
        )
        wr.full_clean()

        wr.task_data = make_task_data()
        wr.task_data["invalid-key"] = None
        with self.assertRaisesRegex(
            ValidationError,
            "extra fields not permitted",
        ):
            wr.full_clean()

        wr.task_data = make_task_data()
        del wr.task_data["environment"]
        with self.assertRaisesRegex(
            ValidationError,
            'The backend "auto" requires "environment" to be set',
        ):
            wr.full_clean()

    def test_clean_workflow_data_raise_validation_error(self) -> None:
        """Check validation of workflow data."""
        wr = WorkRequest.objects.create(
            created_by=self.user,
            workspace=default_workspace(),
            task_type=TaskTypes.WORKFLOW,
            task_name="noop",
            task_data={},
        )
        wr.full_clean()

        wr.task_data["spurious"] = True
        with self.assertRaisesRegex(
            ValidationError,
            "extra fields not permitted",
        ):
            wr.full_clean()

    def test_workflow_data_accessors(self) -> None:
        """Test the custom pydantic accessors for workflow_data."""
        common_args = {
            "created_by": self.user,
            "workspace": default_workspace(),
            "task_type": TaskTypes.WORKER,
            "task_name": "noop",
            "task_data": {},
        }

        # Empty workflow_data
        wr = WorkRequest.objects.create(**common_args)
        self.assertEqual(wr.workflow_data_json, {})
        self.assertIsNone(wr.workflow_data)

        wr = WorkRequest.objects.create(**common_args)
        self.assertEqual(wr.workflow_data_json, {})
        self.assertIsNone(wr.workflow_data)

        # Filled workflow data
        wr = WorkRequest.objects.create(
            workflow_data_json={"display_name": "Test", "step": "test"},
            **common_args,
        )
        self.assertEqual(
            wr.workflow_data_json,
            {"display_name": "Test", "step": "test"},
        )
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(display_name="Test", step="test"),
        )

        wr.workflow_data = None
        self.assertEqual(wr.workflow_data_json, {})
        self.assertIsNone(wr.workflow_data)

        wr.workflow_data = WorkRequestWorkflowData(
            display_name="Test1", step="test1"
        )
        self.assertEqual(
            wr.workflow_data_json,
            {"display_name": "Test1", "step": "test1"},
        )
        self.assertEqual(
            wr.workflow_data,
            WorkRequestWorkflowData(display_name="Test1", step="test1"),
        )

    def test_supersedes(self) -> None:
        """Test behaviour of the supersedes field."""
        wr = self.create_work_request(task_name="noop")
        self.assertIsNone(wr.supersedes)
        self.assertFalse(hasattr(wr, "superseded"))

        wr1 = self.create_work_request(task_name="noop")
        wr1.supersedes = wr
        wr1.save()

        wr.refresh_from_db()
        wr1.refresh_from_db()

        self.assertIsNone(wr.supersedes)
        self.assertEqual(wr1.supersedes, wr)
        self.assertEqual(wr.superseded, wr1)
        self.assertFalse(hasattr(wr1, "superseded"))

        self.assertQuerysetEqual(
            WorkRequest.objects.filter(superseded__isnull=False), [wr]
        )
        self.assertQuerysetEqual(
            WorkRequest.objects.filter(supersedes__isnull=False), [wr1]
        )

    def test_supersedes_expired(self) -> None:
        """Test expiring a superseded work request."""
        wr = self.create_work_request(task_name="noop")
        wr.created_at = timezone.now() - timedelta(days=365)
        wr.expiration_delay = timedelta(days=7)
        wr.save()
        wr1 = self.create_work_request(task_name="noop")
        wr1.supersedes = wr
        wr1.save()

        # Expire the superseded work request
        with (
            io.StringIO() as stdout,
            io.StringIO() as stderr,
            DeleteOperation(out=stdout, err=stderr, dry_run=False) as operation,
        ):
            delete_expired = DeleteExpiredWorkRequests(operation)
            delete_expired.run()

        self.assertQuerysetEqual(WorkRequest.objects.filter(pk=wr.pk), [])
        wr1.refresh_from_db()

        # Supersedes is set to None
        self.assertIsNone(wr1.supersedes)

    def test_cant_retry_superseded(self) -> None:
        """Superseded work requests cannot be retried."""
        wr = self.workflow.create_child("noop")
        wr.status = WorkRequest.Statuses.ABORTED
        wr.save()
        self.assertTrue(wr.can_retry())

        wr1 = self.workflow.create_child("noop")
        wr1.supersedes = wr
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(
            CannotRetry, r"^Cannot retry old superseded tasks$"
        ):
            wr.retry()

    def test_can_retry_aborted_failed(self) -> None:
        """Only aborted/failed work requests can be retried."""
        wr = self.workflow.create_child("noop")
        wr.status = WorkRequest.Statuses.COMPLETED
        wr.result = WorkRequest.Results.SUCCESS
        wr.save()

        msg = r"^Only aborted or failed tasks can be retried$"

        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

        wr.status = WorkRequest.Statuses.COMPLETED
        wr.result = WorkRequest.Results.ERROR
        wr.save()
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

        wr.status = WorkRequest.Statuses.PENDING
        wr.result = WorkRequest.Results.NONE
        wr.save()
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

        wr.status = WorkRequest.Statuses.RUNNING
        wr.result = WorkRequest.Results.NONE
        wr.save()
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

        wr.status = WorkRequest.Statuses.BLOCKED
        wr.result = WorkRequest.Results.NONE
        wr.save()
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

    def test_can_retry_task_type(self) -> None:
        """Only some task types can be retried."""
        wr = self.workflow.create_child("noop")
        wr.status = WorkRequest.Statuses.COMPLETED
        wr.result = WorkRequest.Results.SUCCESS
        wr.save()

        msg = (
            r"^Only workflow, worker, internal, or signing tasks"
            " can be retried$"
        )

        wr.task_type = TaskTypes.SERVER
        wr.status = WorkRequest.Statuses.ABORTED
        wr.result = WorkRequest.Results.NONE
        wr.save()
        self.assertFalse(wr.can_retry())
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

    def test_cant_retry_invalid_task_data(self) -> None:
        """Work request with invalid task data cannot be retried."""
        wr = self.workflow.create_child("noop")
        wr.status = WorkRequest.Statuses.ABORTED
        wr.result = WorkRequest.Results.NONE
        wr.task_name = "sbuild"
        wr.save()
        self.assertFalse(wr.can_retry())
        msg = "^Task dependencies cannot be satisfied$"
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

    @preserve_task_registry()
    def test_cant_retry_if_lookups_fail(self) -> None:
        """Work requests with unsatisfiable lookups can't retry."""
        wr = self.workflow.create_child("noop")
        wr.status = WorkRequest.Statuses.ABORTED
        wr.result = WorkRequest.Results.NONE
        wr.save()

        collection = self.playground.create_collection(
            name="debian",
            category=CollectionCategory.ENVIRONMENTS,
        )

        class LookupTask(Noop):
            """Test task that performs a lookup."""

            def compute_dynamic_data(
                self, task_database: TaskDatabaseInterface
            ) -> BaseDynamicTaskData | None:
                """Perform a lookup."""
                task_database.lookup_single_artifact(
                    "debian@debian:environments/match:codename=bookworm"
                )
                return None

        wr = self.create_work_request(
            task_name="lookuptask",
            task_data={},
            status=WorkRequest.Statuses.ABORTED,
        )

        self.assertFalse(wr.can_retry())
        msg = r"^Task dependencies cannot be satisfied$"
        with self.assertRaisesRegex(CannotRetry, msg):
            wr.retry()

        # Create the looked up artifact
        tarball, _ = self.create_artifact(
            category=ArtifactCategory.SYSTEM_TARBALL,
            data={"architecture": "amd64", "codename": "bookworm"},
        )
        collection.manager.add_artifact(tarball, user=self.user)

        self.assertTrue(wr.can_retry())
        wr.retry()

    def test_retry_supersede(self) -> None:
        """Test behaviour of retry, superseding work requests."""
        # An aborted workflow, with a failed task and one depending on it
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.result = WorkRequest.Results.FAILURE
        self.workflow.completed_at = timezone.now()
        self.workflow.save()

        wr_failed = self.workflow.create_child("noop")
        wr_failed.status = WorkRequest.Statuses.COMPLETED
        wr_failed.result = WorkRequest.Results.FAILURE
        wr_failed.save()

        # Two reverse dependencies
        wr_dep1 = self.workflow.create_child(
            "noop", status=WorkRequest.Statuses.PENDING
        )
        wr_dep1.dependencies.add(wr_failed)
        wr_dep2 = self.workflow.create_child(
            "noop", status=WorkRequest.Statuses.COMPLETED
        )
        wr_dep2.dependencies.add(wr_failed)
        # One forward dependency
        wr_fdep1 = self.workflow.create_child(
            "noop", status=WorkRequest.Statuses.COMPLETED
        )
        wr_failed.dependencies.add(wr_fdep1)

        # wr_failed gets retried
        self.assertTrue(wr_failed.can_retry())
        wr_new = wr_failed.retry()

        self.assertEqual(wr_new.workspace, wr_failed.workspace)
        self.assertGreater(wr_new.created_at, wr_failed.created_at)
        self.assertIsNone(wr_new.started_at)
        self.assertIsNone(wr_new.completed_at)
        self.assertEqual(wr_new.created_by, wr_failed.created_by)
        self.assertEqual(wr_new.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr_new.result, WorkRequest.Results.NONE)
        self.assertIsNone(wr_new.worker)
        self.assertEqual(wr_new.task_type, wr_failed.task_type)
        self.assertEqual(wr_new.task_name, wr_failed.task_name)
        self.assertEqual(wr_new.task_data, wr_failed.task_data)
        self.assertIsNone(wr_new.dynamic_task_data)
        self.assertEqual(wr_new.priority_base, wr_failed.priority_base)
        self.assertEqual(
            wr_new.priority_adjustment, wr_failed.priority_adjustment
        )
        self.assertEqual(wr_new.unblock_strategy, wr_failed.unblock_strategy)
        self.assertEqual(wr_new.parent, self.workflow)
        self.assertEqual(
            wr_new.workflow_data_json,
            {**wr_failed.workflow_data_json, "retry_count": 1},
        )
        self.assertEqual(
            wr_new.event_reactions_json, wr_failed.event_reactions_json
        )
        self.assertIsNone(wr_new.internal_collection)
        self.assertEqual(wr_new.expiration_delay, wr_failed.expiration_delay)
        self.assertEqual(wr_new.supersedes, wr_failed)
        self.assertEqual(wr_failed.superseded, wr_new)

        self.assertQuerysetEqual(wr_new.dependencies.all(), [wr_fdep1])

        self.assertQuerysetEqual(wr_dep1.dependencies.all(), [wr_new])
        wr_dep1.refresh_from_db()
        self.assertEqual(wr_dep1.status, WorkRequest.Statuses.BLOCKED)

        self.assertQuerysetEqual(wr_dep2.dependencies.all(), [wr_new])
        wr_dep2.refresh_from_db()
        self.assertEqual(wr_dep2.status, WorkRequest.Statuses.COMPLETED)

        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.assertEqual(self.workflow.result, WorkRequest.Results.NONE)
        self.assertIsNone(self.workflow.completed_at)

    def test_retry_supersede_blocked_by_dep(self) -> None:
        """A retried work request is BLOCKED if it has incomplete deps."""
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.result = WorkRequest.Results.FAILURE
        self.workflow.completed_at = timezone.now()
        self.workflow.save()
        wr = self.workflow.create_child(
            "noop", status=WorkRequest.Statuses.ABORTED
        )
        wr_dep = self.workflow.create_child("noop")
        wr.dependencies.add(wr_dep)

        wr_new = wr.retry()

        self.assertEqual(wr_new.status, WorkRequest.Statuses.BLOCKED)
        self.assertEqual(wr_new.result, WorkRequest.Results.NONE)
        self.assertEqual(wr_new.unblock_strategy, wr.unblock_strategy)
        self.assertEqual(wr_new.supersedes, wr)
        self.assertEqual(wr.superseded, wr_new)
        self.assertQuerysetEqual(wr_new.dependencies.all(), [wr_dep])

    def test_retry_supersede_blocked_manual(self) -> None:
        """A retried work request is BLOCKED if it needs manual unblocking."""
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.result = WorkRequest.Results.FAILURE
        self.workflow.completed_at = timezone.now()
        self.workflow.save()
        wr = self.workflow.create_child(
            "noop", status=WorkRequest.Statuses.ABORTED
        )
        wr.unblock_strategy = WorkRequest.UnblockStrategy.MANUAL
        wr.save()

        wr_new = wr.retry()

        self.assertEqual(wr_new.status, WorkRequest.Statuses.BLOCKED)
        self.assertEqual(wr_new.result, WorkRequest.Results.NONE)
        self.assertEqual(wr_new.unblock_strategy, wr.unblock_strategy)
        self.assertEqual(wr_new.supersedes, wr)
        self.assertEqual(wr.superseded, wr_new)

    def test_retry_supersede_increments_retry_count(self) -> None:
        """Retrying a task increments its retry count."""
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.result = WorkRequest.Results.FAILURE
        self.workflow.completed_at = timezone.now()
        self.workflow.save()
        wr = self.workflow.create_child(
            "noop",
            status=WorkRequest.Statuses.ABORTED,
            workflow_data=WorkRequestWorkflowData(retry_count=1),
        )

        wr_new = wr.retry()

        self.assertEqual(wr_new.workflow_data_json, {"retry_count": 2})

    def test_retry_in_running_workflow(self) -> None:
        """Test retrying a task in a running workflow."""
        # An aborted workflow, with a failed task and one depending on it
        now = timezone.now()
        self.workflow.status = WorkRequest.Statuses.RUNNING
        # Bogus value to check that it does not get reset
        self.workflow.completed_at = now
        self.workflow.save()

        wr_failed = self.workflow.create_child("noop")
        wr_failed.status = WorkRequest.Statuses.COMPLETED
        wr_failed.result = WorkRequest.Results.FAILURE
        wr_failed.save()

        # wr_failed gets retried
        wr_failed.retry()

        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.assertEqual(self.workflow.completed_at, now)

    def test_retry_in_failed_workflow(self) -> None:
        """Test retrying a task in a failed workflow."""
        # An aborted workflow, with a failed task and one depending on it
        now = timezone.now()
        self.workflow.status = WorkRequest.Statuses.COMPLETED
        self.workflow.result = WorkRequest.Results.FAILURE
        # Bogus value to check that it does not get reset
        self.workflow.completed_at = now
        self.workflow.save()

        wr_failed = self.workflow.create_child("noop")
        wr_failed.status = WorkRequest.Statuses.COMPLETED
        wr_failed.result = WorkRequest.Results.FAILURE
        wr_failed.save()

        # wr_failed gets retried
        wr_failed.retry()

        # The workflow is set to running again
        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.assertIsNone(self.workflow.completed_at)

    def test_retry_internal(self) -> None:
        """Internal work requests can be retried."""
        wr_aborted = WorkRequest.objects.create_synchronization_point(
            parent=self.workflow,
            step="test",
            status=WorkRequest.Statuses.ABORTED,
        )

        self.assertTrue(wr_aborted.can_retry())
        wr_aborted.retry()

    def test_retry_empty_workflow(self) -> None:
        """Test behaviour of retrying for an empty workflow."""
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.save()

        wr = self.workflow.retry()

        # Retrying a workflow does not create a new work request
        self.assertEqual(wr.pk, self.workflow.pk)

        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(self.workflow.result, WorkRequest.Results.SUCCESS)

    def test_retry_workflow(self) -> None:
        """Test behaviour of retrying for an empty workflow."""
        wr_completed = self.workflow.create_child("noop")
        wr_aborted = self.workflow.create_child("noop")

        # Simulate the previous workflow lifecycle
        self.assertTrue(self.workflow.mark_running())
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.workflow.unblock_workflow_children()
        wr_completed.refresh_from_db()
        self.assertTrue(
            wr_completed.mark_completed(WorkRequest.Results.SUCCESS)
        )
        wr_completed.completed_at = (
            orig_completed := timezone.now() - timedelta(days=7)
        )
        wr_completed.save()
        wr_aborted.refresh_from_db()
        self.assertTrue(wr_aborted.mark_aborted())

        # The workflow ended up as aborted
        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.ABORTED)

        # Retry it
        wr = self.workflow.retry()
        self.assertEqual(wr.pk, self.workflow.pk)

        self.workflow.refresh_from_db()
        wr_completed.refresh_from_db()
        wr_aborted.refresh_from_db()

        # The workflow is running
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.assertEqual(self.workflow.result, WorkRequest.Results.NONE)
        self.assertEqual(self.workflow.workflow_data_json, {"retry_count": 1})
        self.assertEqual(wr_completed.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(wr_completed.result, WorkRequest.Results.SUCCESS)
        # Successful work requests did not get restarted
        self.assertEqual(wr_completed.completed_at, orig_completed)
        self.assertEqual(wr_aborted.status, WorkRequest.Statuses.ABORTED)
        self.assertEqual(wr_aborted.result, WorkRequest.Results.NONE)
        wr_aborted_new = wr_aborted.superseded
        self.assertEqual(wr_aborted_new.status, WorkRequest.Statuses.PENDING)
        self.assertEqual(wr_aborted_new.result, WorkRequest.Results.NONE)

        # Complete the one running task
        wr_aborted_new.mark_completed(WorkRequest.Results.SUCCESS)
        self.workflow.refresh_from_db()

        # Workflow completed successfully
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(self.workflow.result, WorkRequest.Results.SUCCESS)

    def test_retry_workflow_calls(self) -> None:
        """Test behaviour of retrying for an empty workflow."""
        wr_aborted = self.workflow.create_child("noop")

        # Simulate the previous workflow lifecycle
        self.assertTrue(self.workflow.mark_running())
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.RUNNING)
        self.workflow.unblock_workflow_children()
        wr_aborted.refresh_from_db()
        self.assertTrue(wr_aborted.mark_aborted())

        # The workflow ended up as aborted
        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.ABORTED)

        with (
            mock.patch(
                "debusine.server.workflows.base.orchestrate_workflow"
            ) as orchestrate_workflow,
            mock.patch(
                "debusine.db.models.work_requests.WorkRequest"
                ".maybe_finish_workflow"
            ) as maybe_finish_workflow,
        ):
            self.workflow.retry()

        orchestrate_workflow.assert_called_with(self.workflow)
        maybe_finish_workflow.assert_called()

    def test_retry_workflow_child_fails(self) -> None:
        """Test retrying a workflow with a child that cannot retry."""
        wr_aborted1 = self.workflow.create_child("noop")
        wr_unretriable = self.workflow.create_child("noop")
        wr_aborted2 = self.workflow.create_child("noop")

        self.assertTrue(self.workflow.mark_running())
        self.workflow.unblock_workflow_children()
        wr_aborted1.refresh_from_db()
        wr_unretriable.refresh_from_db()
        wr_aborted2.refresh_from_db()
        self.assertTrue(wr_aborted1.mark_aborted())
        self.assertTrue(wr_unretriable.mark_aborted())
        self.assertTrue(wr_aborted2.mark_aborted())

        # The workflow ended up as aborted
        self.workflow.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.ABORTED)

        # Make wr_unretriable unretriable
        orig_check_retry = WorkRequest._check_retry

        def test_check_retry(self) -> None:
            if self.pk == wr_unretriable.pk:
                raise CannotRetry("expected failure")
            else:
                orig_check_retry(self)

        try:
            with transaction.atomic():
                with (
                    mock.patch(
                        "debusine.db.models.WorkRequest._check_retry",
                        side_effect=test_check_retry,
                        autospec=True,
                    ),
                    self.assertRaises(CannotRetry) as exc,
                ):
                    self.workflow.retry()

                self.assertEqual(
                    exc.exception.args[0],
                    f"child work request {wr_unretriable.pk} cannot be retried",
                )
                self.assertIsNotNone(exc.exception.__context__)
                assert exc.exception.__context__ is not None
                self.assertEqual(
                    exc.exception.__context__.args[0], "expected failure"
                )

                self.workflow.refresh_from_db()
                wr_aborted1.refresh_from_db()
                wr_aborted1_new = wr_aborted1.superseded
                wr_unretriable.refresh_from_db()
                wr_aborted2.refresh_from_db()
                self.assertFalse(hasattr(wr_aborted2, "superseded"))
                self.assertEqual(
                    self.workflow.status, WorkRequest.Statuses.RUNNING
                )
                self.assertEqual(
                    wr_aborted1_new.status, WorkRequest.Statuses.PENDING
                )
                self.assertEqual(
                    wr_unretriable.status, WorkRequest.Statuses.ABORTED
                )
                self.assertEqual(
                    wr_aborted2.status, WorkRequest.Statuses.ABORTED
                )
                # Raise to roll back the transaction
                raise exc.exception
        except CannotRetry as exc1:
            self.assertIs(exc1, exc.exception)

        self.workflow.refresh_from_db()
        wr_aborted1.refresh_from_db()
        wr_unretriable.refresh_from_db()
        wr_aborted2.refresh_from_db()
        self.assertEqual(self.workflow.status, WorkRequest.Statuses.ABORTED)
        self.assertEqual(wr_aborted1.status, WorkRequest.Statuses.ABORTED)
        self.assertEqual(wr_unretriable.status, WorkRequest.Statuses.ABORTED)
        self.assertEqual(wr_aborted2.status, WorkRequest.Statuses.ABORTED)

    def test_retry_workflow_increments_retry_count(self) -> None:
        """Retrying a workflow increments its retry count."""
        self.workflow.status = WorkRequest.Statuses.ABORTED
        self.workflow.workflow_data = WorkRequestWorkflowData(retry_count=1)
        self.workflow.save()

        wr = self.workflow.retry()
        self.assertEqual(wr.pk, self.workflow.pk)
        self.assertEqual(wr.workflow_data_json, {"retry_count": 2})

    def test_retry_update_debusine_promise_work_request_id(self):
        """Test retrying updates promise of the expected work request id."""
        user = self.get_test_user()

        template = WorkflowTemplate.objects.create(
            name="retry-update-debusine-promise-work-request-id",
            workspace=default_workspace(),
            task_name="noop",
        )

        workflow = WorkRequest.objects.create_workflow(
            template=template, created_by=user, data={}
        )

        collection = workflow.internal_collection

        wr_to_retry = self.create_work_request(
            parent=workflow,
            task_name="noop",
        )
        wr_aborted = self.create_work_request(
            parent=workflow,
            task_name="noop",
        )

        wr_to_retry.mark_aborted()
        wr_aborted.mark_aborted()

        data = {
            "promise_workflow_id": wr_to_retry.parent.id,
            "promise_category": "something",
        }
        debusine_promise_updated = collection.manager.add_bare_data(
            category=BareDataCategory.PROMISE,
            name="test-1",
            data={**data, "promise_work_request_id": wr_to_retry.id},
            user=user,
        )
        debusine_promise_not_updated = collection.manager.add_bare_data(
            category=BareDataCategory.PROMISE,
            name="test-2",
            data={**data, "promise_work_request_id": wr_aborted.id},
            user=user,
        )

        wr_supersedes = wr_to_retry.retry()

        debusine_promise_updated.refresh_from_db()
        debusine_promise_not_updated.refresh_from_db()

        # Updated to the new promise_work_request_id
        self.assertEqual(
            debusine_promise_updated.data["promise_work_request_id"],
            wr_supersedes.id,
        )

        # Still the old promise_work_request_id
        self.assertEqual(
            debusine_promise_not_updated.data["promise_work_request_id"],
            wr_aborted.id,
        )

    def test_is_workflow_task_type_workflow(self) -> None:
        """A WORKFLOW work request is a workflow."""
        self.assertTrue(self.workflow.is_workflow)

    def test_is_workflow_child(self) -> None:
        """A WORKER work request is not a workflow, even if it's part of one."""
        wr = self.create_work_request(parent=self.workflow)

        self.assertFalse(wr.is_workflow)

    def test_is_part_of_workflow_standalone(self) -> None:
        """A standalone work request is not part of a workflow."""
        wr = self.create_work_request()

        self.assertFalse(wr.is_part_of_workflow)

    def test_is_part_of_workflow_root(self) -> None:
        """A WORKFLOW work request with no parent is not part of a workflow."""
        self.assertFalse(self.workflow.is_part_of_workflow)

    def test_is_part_of_workflow_child(self) -> None:
        """A work request with a parent is part of a workflow."""
        wr = self.create_work_request(parent=self.workflow)

        self.assertTrue(wr.is_part_of_workflow)

    def test_get_workflow_root_standalone(self) -> None:
        """A standalone work request has no workflow root."""
        wr = self.create_work_request()

        self.assertIsNone(wr.get_workflow_root())

    def test_get_workflow_root_root(self) -> None:
        """A top-level work request in a workflow is its own root."""
        self.assertEqual(self.workflow.get_workflow_root(), self.workflow)

    def test_get_workflow_root_child(self) -> None:
        """A child work request in a workflow has a root."""
        wr = self.create_work_request(parent=self.workflow)

        self.assertEqual(wr.get_workflow_root(), self.workflow)

    def test_get_workflow_root_grandchild(self) -> None:
        """A grandchild work request in a workflow has a root."""
        root = self.workflow
        child_template = WorkflowTemplate.objects.create(
            name="child",
            workspace=default_workspace(),
            task_name="noop",
            task_data={},
        )
        child = WorkRequest.objects.create_workflow(
            template=child_template,
            data={},
            created_by=self.get_test_user(),
            parent=root,
        )
        wr = self.create_work_request(parent=child)

        self.assertEqual(wr.get_workflow_root(), root)

    def test_is_workflow_root(self):
        """Tests for is_workflow_root()."""
        # Standalone work request is not a workflow
        self.assertFalse(self.playground.create_work_request().is_workflow_root)

        root_workflow = self.playground.create_workflow()
        self.assertTrue(root_workflow.is_workflow_root)

        sub_workflow = self.playground.create_workflow()
        sub_workflow.parent = root_workflow
        sub_workflow.save()

        # A workflow in a workflow is not a root workflow
        self.assertFalse(sub_workflow.is_workflow_root)

    def test_workflow_display_name_explicit(self) -> None:
        """`workflow_display_name` returns an explicitly-set display name."""
        wr = self.create_work_request(
            workflow_data_json={"display_name": "Foo", "step": "foo"}
        )

        self.assertEqual(wr.workflow_display_name, "Foo")

    def test_workflow_display_name_task_name(self) -> None:
        """`workflow_display_name` defaults to the task name."""
        wr = self.create_work_request(task_name="noop")

        self.assertEqual(wr.workflow_display_name, "noop")

    def test_requires_signature_blocked_external_debsign(self) -> None:
        """A blocked `Wait/ExternalDebsign` task requires a signature."""
        wr = self.create_work_request(
            task_type=TaskTypes.WAIT,
            task_name="externaldebsign",
            status=WorkRequest.Statuses.BLOCKED,
        )

        self.assertTrue(wr.requires_signature)

    def test_requires_signature_completed_external_debsign(self) -> None:
        """A completed `Wait/ExternalDebsign` task requires no signature."""
        wr = self.create_work_request(
            task_type=TaskTypes.WAIT,
            task_name="externaldebsign",
            status=WorkRequest.Statuses.COMPLETED,
        )

        self.assertFalse(wr.requires_signature)

    def test_requires_signature_other_task(self) -> None:
        """A non-`Wait/ExternalDebsign` task requires no signature."""
        wr = self.create_work_request(task_name="noop")

        self.assertFalse(wr.requires_signature)

    def test_effective_expiration_delay(self) -> None:
        """Test WorkRequest.effective_expiration_delay() method."""
        wr = self.create_work_request(task_name="noop")

        wr.expiration_delay = timedelta(days=1)
        self.assertEqual(wr.effective_expiration_delay(), timedelta(days=1))

        wr.expiration_delay = timedelta(days=0)
        self.assertEqual(wr.effective_expiration_delay(), timedelta(days=0))

        workspace = self.playground.create_workspace(name="test")
        workspace.default_expiration_delay = timedelta(days=2)
        wr.workspace = workspace

        wr.expiration_delay = timedelta(days=1)
        self.assertEqual(wr.effective_expiration_delay(), timedelta(days=1))
        self.assertEqual(wr.expire_at, wr.created_at + timedelta(days=1))

        wr.expiration_delay = timedelta(days=0)
        self.assertEqual(wr.effective_expiration_delay(), timedelta(days=0))

        wr.expiration_delay = None
        self.assertEqual(wr.effective_expiration_delay(), timedelta(days=2))

    def test_get_task(self) -> None:
        """Test get_task with various task types."""
        workspace = self.playground.get_default_workspace()
        for task_type, task_name, expected_class, task_data in (
            (TaskTypes.WORKER, "noop", Noop, {}),
            (TaskTypes.SERVER, "servernoop", ServerNoop, {}),
            (TaskTypes.SIGNING, "noop", SigningNoop, {}),
            (TaskTypes.WORKFLOW, "noop", NoopWorkflow, {}),
            (
                TaskTypes.WAIT,
                "delay",
                Delay,
                {"delay_until": timezone.now().isoformat()},
            ),
        ):
            with self.subTest(task_type=task_type):
                wr = WorkRequest(
                    workspace=workspace,
                    task_type=task_type,
                    task_name=task_name,
                    task_data=task_data,
                )
                task = wr.get_task()
                self.assertIsInstance(task, expected_class)

    def test_get_task_internal_workflow_callback(self) -> None:
        """A workflow callback is instantiated as its workflow."""
        parent = self.playground.create_workflow(task_name="noop")
        wr = WorkRequest.objects.create_workflow_callback(
            parent=parent, step="test"
        )
        task = wr.get_task()
        self.assertIsInstance(task, NoopWorkflow)

    def test_get_task_internal_workflow_callback_outside_workflow(self) -> None:
        """A workflow callback outside a workflow cannot be instantiated."""
        wr = self.create_work_request(
            task_type=TaskTypes.INTERNAL, task_name="workflow"
        )
        with self.assertRaisesRegex(
            InternalTaskError,
            "Workflow callback is not contained in a workflow",
        ):
            wr.get_task()

    def test_get_task_internal_not_workflow_callback(self) -> None:
        """A non-workflow-callback internal task cannot be instantiated."""
        wr = self.create_work_request(
            task_type=TaskTypes.INTERNAL, task_name="synchronization_point"
        )
        with self.assertRaisesRegex(
            InternalTaskError,
            "Internal tasks other than workflow callbacks cannot be "
            "instantiated",
        ):
            wr.get_task()

    def test_get_task_worker_host_architecture(self) -> None:
        """get_task() sets worker_host_architecture if it has a worker."""
        wr = self.create_work_request(
            task_type=TaskTypes.WORKER, task_name="noop"
        )
        worker = self.playground.create_worker(
            extra_dynamic_metadata={"system:host_architecture": "i386"}
        )

        task = wr.get_task(worker=worker)

        self.assertEqual(task.worker_host_architecture, "i386")

    def test_get_task_no_worker_host_architecture(self) -> None:
        """get_task() skips worker_host_architecture if it has no worker."""
        wr = self.create_work_request(
            task_type=TaskTypes.WORKER, task_name="noop"
        )

        task = wr.get_task()

        self.assertIsNone(task.worker_host_architecture)

    def test_get_label_from_data(self) -> None:
        """Test getting label from data."""
        wr = WorkRequest(
            task_type=TaskTypes.WORKER, task_name="noop", task_data={}
        )
        self.assertEqual(wr.get_label(), "noop")

    def test_get_label_from_bad_task_data(self) -> None:
        """Test getting label from inconsistent task data."""
        wr = WorkRequest(
            pk=42, task_type=TaskTypes.WORKER, task_name="sbuild", task_data={}
        )

        with override_settings(DEBUG=True):
            with self.assertRaises(TaskConfigError):
                wr.get_label()

        with override_settings(DEBUG=False):
            with self.assertLogs("debusine.db.models.work_requests") as log:
                self.assertEqual(wr.get_label(), "sbuild")
        self.assertIn("task 42 has invalid data", log.output[0].split("\n")[0])

    def test_get_label_for_internal_task(self) -> None:
        """Test getting label for an internal task."""
        wr = WorkRequest.objects.create_synchronization_point(
            parent=self.workflow,
            step="test",
            display_name="Test",
            status=WorkRequest.Statuses.BLOCKED,
        )
        self.assertEqual(wr.get_label(), "Test")

    def test_get_label_from_data_reuse_model(self) -> None:
        """Test getting label reusing the model instance."""
        wr = WorkRequest(
            task_type=TaskTypes.WORKER,
            task_name="blhc",
            task_data={"artifact": "TODO"},
        )
        task = Noop({})
        self.assertEqual(wr.get_label(task), "noop")


class WorkRequestWorkerTests(ChannelsHelpersMixin, TestCase):
    """Tests for the WorkRequest class scheduling."""

    def setUp(self) -> None:
        """Set up WorkRequest to be used in the tests."""
        worker = Worker.objects.create_with_fqdn(
            "computer.lan", Token.objects.create()
        )

        self.work_request = self.create_work_request(
            task_name='request-01',
            worker=worker,
            status=WorkRequest.Statuses.PENDING,
        )

    def test_str(self) -> None:
        """Test WorkerRequest.__str__ return WorkRequest.id."""
        self.assertEqual(self.work_request.__str__(), str(self.work_request.id))

    def test_mark_running_from_aborted(self) -> None:
        """Test WorkRequest.mark_running() doesn't change (was aborted)."""
        self.work_request.status = WorkRequest.Statuses.ABORTED
        self.work_request.save()

        self.assertFalse(self.work_request.mark_running())

        self.work_request.refresh_from_db()
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.ABORTED)

    def test_mark_running(self) -> None:
        """Test WorkRequest.mark_running() change status to running."""
        self.work_request.status = WorkRequest.Statuses.PENDING
        self.work_request.save()
        self.assertIsNone(self.work_request.started_at)

        self.assertTrue(self.work_request.mark_running())

        self.work_request.refresh_from_db()
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.RUNNING)
        assert self.work_request.started_at is not None
        self.assertLess(self.work_request.started_at, timezone.now())

        # Marking as running again (a running WorkRequest) is a no-op
        started_at = self.work_request.started_at
        self.assertTrue(self.work_request.mark_running())

        self.work_request.refresh_from_db()
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.RUNNING)
        self.assertEqual(self.work_request.started_at, started_at)

    def test_mark_running_fails_worker_already_running(self) -> None:
        """WorkRequest.mark_running() return False: worker already running."""
        self.work_request.status = WorkRequest.Statuses.PENDING
        self.work_request.save()

        other_work_request = self.create_work_request(
            status=WorkRequest.Statuses.RUNNING, worker=self.work_request.worker
        )

        with self.assertLogsContains(
            f"Cannot mark WorkRequest {self.work_request.pk} as running - the "
            f"assigned worker {self.work_request.worker} is running too many "
            f"other WorkRequests: [{other_work_request!r}]",
            logger="debusine.db.models",
            level=logging.DEBUG,
        ):
            self.assertFalse(self.work_request.mark_running())

    def test_mark_running_fails_no_assigned_worker(self) -> None:
        """WorkRequest.mark_running() return False: no worker assigned."""
        self.work_request.status = WorkRequest.Statuses.PENDING
        self.work_request.worker = None
        self.work_request.save()

        self.assertFalse(self.work_request.mark_running())

    def test_mark_completed_from_aborted(self) -> None:
        """Test WorkRequest.mark_completed() doesn't change (was aborted)."""
        self.work_request.status = WorkRequest.Statuses.ABORTED
        self.work_request.save()

        self.work_request.refresh_from_db()

        self.assertFalse(
            self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        )

        self.work_request.refresh_from_db()
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.ABORTED)

    def test_mark_completed(self) -> None:
        """Test WorkRequest.mark_completed() changes status to completed."""
        self.work_request.status = WorkRequest.Statuses.RUNNING
        self.work_request.save()

        self.assertIsNone(self.work_request.completed_at)
        self.assertEqual(self.work_request.result, WorkRequest.Results.NONE)

        self.assertTrue(
            self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        )

        self.work_request.refresh_from_db()
        self.assertEqual(
            self.work_request.status, WorkRequest.Statuses.COMPLETED
        )
        self.assertEqual(self.work_request.result, WorkRequest.Results.SUCCESS)
        assert self.work_request.completed_at is not None
        self.assertLess(self.work_request.completed_at, timezone.now())

    def test_mark_aborted(self) -> None:
        """Test WorkRequest.mark_aborted() changes status to aborted."""
        self.assertIsNone(self.work_request.completed_at)

        self.assertTrue(self.work_request.mark_aborted())

        self.work_request.refresh_from_db()
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.ABORTED)
        assert self.work_request.completed_at is not None
        self.assertLess(self.work_request.completed_at, timezone.now())

    def test_maybe_finish_workflow_not_workflow(self) -> None:
        """`maybe_finish_workflow` does nothing on non-workflows."""
        self.assertFalse(self.work_request.maybe_finish_workflow())

    def test_maybe_finish_workflow_already_aborted(self) -> None:
        """`maybe_finish_workflow` does nothing if it was already aborted."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        workflow.create_child("noop", status=WorkRequest.Statuses.COMPLETED)
        self.assertTrue(workflow.mark_aborted())

        self.assertFalse(workflow.maybe_finish_workflow())

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.ABORTED)

    def test_maybe_finish_workflow_has_children_in_progress(self) -> None:
        """`maybe_finish_workflow` does nothing if a child is in progress."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [workflow.create_child("noop") for _ in range(2)]
        workflow.mark_running()
        workflow.unblock_workflow_children()
        children[0].refresh_from_db()
        self.assertTrue(children[0].mark_completed(WorkRequest.Results.SUCCESS))

        self.assertFalse(workflow.maybe_finish_workflow())

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.RUNNING)

    def test_maybe_finish_workflow_all_succeeded(self) -> None:
        """All children succeeded: COMPLETED/SUCCESS."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [workflow.create_child("noop") for _ in range(2)]
        workflow.mark_running()
        workflow.unblock_workflow_children()

        for child in children:
            child.refresh_from_db()
            self.assertTrue(child.mark_completed(WorkRequest.Results.SUCCESS))

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(workflow.result, WorkRequest.Results.SUCCESS)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())

    def test_maybe_finish_workflow_allow_failure(self) -> None:
        """Some children failed but are allowed to fail: COMPLETED/SUCCESS."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [
            workflow.create_child(
                "noop",
                workflow_data=WorkRequestWorkflowData(
                    allow_failure=True, display_name="test", step="test"
                ),
            )
            for _ in range(2)
        ]
        workflow.mark_running()
        workflow.unblock_workflow_children()

        children[0].refresh_from_db()
        children[0].mark_completed(WorkRequest.Results.FAILURE)
        children[1].refresh_from_db()
        children[1].mark_completed(WorkRequest.Results.SUCCESS)

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(workflow.result, WorkRequest.Results.SUCCESS)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())

    def test_maybe_finish_workflow_some_failed(self) -> None:
        """Some children failed: COMPLETED/FAILURE."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [workflow.create_child("noop") for _ in range(2)]
        workflow.mark_running()
        workflow.unblock_workflow_children()

        children[0].refresh_from_db()
        children[0].mark_completed(WorkRequest.Results.FAILURE)
        children[1].refresh_from_db()
        children[1].mark_completed(WorkRequest.Results.SUCCESS)

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(workflow.result, WorkRequest.Results.FAILURE)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())

    def test_maybe_finish_workflow_some_aborted(self) -> None:
        """Some children were aborted: `maybe_finish_workflow` sets ABORTED."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [workflow.create_child("noop") for _ in range(2)]
        workflow.mark_running()
        workflow.unblock_workflow_children()

        children[0].refresh_from_db()
        children[0].mark_aborted()
        children[1].refresh_from_db()
        children[1].mark_completed(WorkRequest.Results.SUCCESS)

        workflow.refresh_from_db()
        self.assertEqual(workflow.status, WorkRequest.Statuses.ABORTED)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())

    def test_maybe_finish_workflow_superseded_aborted(self) -> None:
        """Some aborted children were superseded: ignores them."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        children = [workflow.create_child("noop") for _ in range(2)]
        workflow.started_at = timezone.now()
        workflow.status = WorkRequest.Statuses.RUNNING
        workflow.save()

        children[0].completed_at = timezone.now()
        children[0].status = WorkRequest.Statuses.ABORTED
        children[0].save()
        children[1].completed_at = timezone.now()
        children[1].status = WorkRequest.Statuses.COMPLETED
        children[1].result = WorkRequest.Results.SUCCESS
        children[1].save()

        self.assertTrue(workflow.maybe_finish_workflow())
        self.assertEqual(workflow.status, WorkRequest.Statuses.ABORTED)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())
        prev_completed_at = workflow.completed_at

        new_wr = workflow.create_child("noop")
        new_wr.supersedes = children[0]
        new_wr.status = WorkRequest.Statuses.COMPLETED
        new_wr.result = WorkRequest.Results.SUCCESS
        new_wr.save()
        workflow.status = WorkRequest.Statuses.RUNNING
        workflow.save()

        self.assertTrue(workflow.maybe_finish_workflow())
        self.assertEqual(workflow.status, WorkRequest.Statuses.COMPLETED)
        assert workflow.completed_at is not None
        self.assertLess(workflow.completed_at, timezone.now())
        self.assertGreater(workflow.completed_at, prev_completed_at)

    async def test_assign_worker(self) -> None:
        """Assign Worker to WorkRequest."""

        def setup_work_request() -> Worker:
            worker = self.work_request.worker
            assert worker is not None

            self.work_request.worker = None
            self.work_request.save()

            self.work_request.refresh_from_db()

            # Initial status (no worker)
            self.assertIsNone(self.work_request.worker)
            self.assertEqual(
                self.work_request.status, WorkRequest.Statuses.PENDING
            )

            return worker

        worker = await sync_to_async(setup_work_request)()
        assert worker.token is not None

        channel = await self.create_channel(worker.token.hash)

        # Assign the worker to the WorkRequest
        await sync_to_async(self.work_request.assign_worker)(worker)

        await self.assert_channel_received(
            channel, {"type": "work_request.assigned"}
        )

        def assert_final_status() -> None:
            self.work_request.refresh_from_db()

            # Assert final status
            self.assertEqual(self.work_request.worker, worker)
            self.assertEqual(
                self.work_request.status, WorkRequest.Statuses.PENDING
            )

        await sync_to_async(assert_final_status)()

    def test_de_assign_worker_from_running(self) -> None:
        """Worker is de-assigned from a work request (was running)."""
        worker = Worker.objects.create_with_fqdn(
            "test", token=Token.objects.create()
        )
        self.work_request.assign_worker(worker)

        self.work_request.mark_running()

        self.assertEqual(self.work_request.worker, worker)
        self.assertIsInstance(self.work_request.started_at, datetime)
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.RUNNING)

        self.assertTrue(self.work_request.de_assign_worker())

        self.work_request.refresh_from_db()

        self.assertIsNone(self.work_request.worker)
        self.assertIsNone(self.work_request.started_at)
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.PENDING)

    def test_de_assign_worker_from_pending(self) -> None:
        """Worker is de-assigned from a work request (was pending)."""
        worker = Worker.objects.create_with_fqdn(
            "test", token=Token.objects.create()
        )
        self.work_request.assign_worker(worker)

        self.assertEqual(self.work_request.worker, worker)
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.PENDING)

        self.assertTrue(self.work_request.de_assign_worker())

        self.work_request.refresh_from_db()

        self.assertIsNone(self.work_request.worker)
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.PENDING)

    def test_de_assign_worker_returns_false(self) -> None:
        """Worker cannot be de-assigned: was not in pending or running."""
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)

        log_msg = (
            f"WorkRequest {self.work_request.pk} cannot be de-assigned: "
            f"current status: {self.work_request.status}"
        )

        with self.assertLogsContains(
            log_msg, logger="debusine.db.models", level=logging.DEBUG
        ):
            self.assertFalse(self.work_request.de_assign_worker())

    def test_duration(self) -> None:
        """duration() returns the correct duration."""
        self.work_request.started_at = datetime(2022, 3, 7, 10, 51)
        self.work_request.completed_at = datetime(2022, 3, 9, 12, 53)
        duration = self.work_request.completed_at - self.work_request.started_at

        self.assertEqual(
            self.work_request.duration,
            duration.total_seconds(),
        )

    def test_duration_is_none_completed_at_is_none(self) -> None:
        """duration() returns None because completed_at is None."""
        self.work_request.started_at = datetime(2022, 3, 7, 10, 51)
        self.work_request.completed_at = None

        self.assertIsNone(self.work_request.duration)

    def test_duration_is_none_started_at_is_none(self) -> None:
        """duration() returns None because started_at is None."""
        self.work_request.started_at = None
        self.work_request.completed_at = datetime(2022, 3, 7, 10, 51)

        self.assertIsNone(self.work_request.duration)

    def test_priority_effective(self) -> None:
        """priority_effective() returns the effective priority."""
        self.work_request.priority_base = 10
        self.work_request.priority_adjustment = 1

        self.assertEqual(self.work_request.priority_effective, 11)

    def test_invalid_pending(self) -> None:
        """Reject invalid pending state change."""
        self.work_request.status = WorkRequest.Statuses.RUNNING
        self.assertFalse(self.work_request.mark_pending())
        self.assertEqual(self.work_request.status, WorkRequest.Statuses.RUNNING)

    def test_add_dependency_within_workflow(self) -> None:
        """Dependencies within a workflow are allowed."""
        template = WorkflowTemplate.objects.create(
            name="test", workspace=default_workspace(), task_name="noop"
        )
        workflow = WorkRequest.objects.create_workflow(
            template=template, created_by=self.get_test_user(), data={}
        )
        children = [workflow.create_child("noop") for _ in range(2)]

        children[0].add_dependency(children[1])

        self.assertQuerysetEqual(children[0].dependencies.all(), [children[1]])

    def test_add_dependency_outside_workflow(self) -> None:
        """Dependencies from within a workflow to outside are forbidden."""
        template = WorkflowTemplate.objects.create(
            name="test", workspace=default_workspace(), task_name="noop"
        )
        workflow = WorkRequest.objects.create_workflow(
            template=template, created_by=self.get_test_user(), data={}
        )
        child = workflow.create_child("noop")
        outside = self.create_work_request()

        with self.assertRaisesRegex(
            ValueError,
            "Work requests in a workflow may not depend on other work "
            "requests outside that workflow",
        ):
            child.add_dependency(outside)

    def test_add_dependency_auto_block(self) -> None:
        """`add_dependency` automatically blocks PENDING/DEPS work requests."""
        rdep = self.create_work_request()
        rdep.add_dependency(self.work_request)
        self.assertEqual(rdep.status, WorkRequest.Statuses.BLOCKED)

    def test_add_dependency_no_auto_block_not_pending(self) -> None:
        """`add_dependency` doesn't automatically block non-PENDING requests."""
        rdep = self.create_work_request()
        rdep.status = WorkRequest.Statuses.ABORTED
        rdep.save()
        rdep.add_dependency(self.work_request)
        self.assertEqual(rdep.status, WorkRequest.Statuses.ABORTED)

    def test_add_dependency_no_auto_block_not_deps(self) -> None:
        """`add_dependency` doesn't automatically block non-DEPS requests."""
        rdep = self.create_work_request()
        rdep.unblock_strategy = WorkRequest.UnblockStrategy.MANUAL
        rdep.save()
        rdep.add_dependency(self.work_request)
        self.assertEqual(rdep.status, WorkRequest.Statuses.PENDING)

    def test_add_dependency_no_auto_block_already_completed(self) -> None:
        """`add_dependency` doesn't auto-block on completed dependencies."""
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep = self.create_work_request()
        rdep.add_dependency(self.work_request)
        self.assertEqual(rdep.status, WorkRequest.Statuses.PENDING)

    def create_rdep(self) -> WorkRequest:
        """Create a reverse-dependency for the default work request."""
        rdep = self.create_work_request(
            task_name='rdep', worker=self.work_request.worker
        )
        rdep.add_dependency(self.work_request)
        return rdep

    def test_dependencies_failure(self) -> None:
        """Failure result gets rdep aborted."""
        rdep = self.create_rdep()
        self.work_request.mark_completed(WorkRequest.Results.FAILURE)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.ABORTED)

    def test_dependencies_success_only_dep(self) -> None:
        """Success result, with nothing else to do, gets rdep pending."""
        rdep = self.create_rdep()
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.PENDING)

    def test_dependencies_allow_failure(self) -> None:
        """Failure result, with allow_failure, gets rdep pending."""
        rdep = self.create_rdep()
        self.work_request.workflow_data = WorkRequestWorkflowData(
            allow_failure=True, display_name="test", step="test"
        )
        self.work_request.mark_completed(WorkRequest.Results.FAILURE)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.PENDING)

    def test_dependencies_success_manual(self) -> None:
        """Success result with manual unblock keeps rdep blocked."""
        rdep = self.create_rdep()
        rdep.unblock_strategy = WorkRequest.UnblockStrategy.MANUAL
        rdep.save()
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.BLOCKED)

    def test_dependencies_success_more_deps(self) -> None:
        """Success result with something else to do keeps rdep blocked."""
        rdep = self.create_rdep()
        work_request2 = self.create_work_request(
            task_name='request-02',
            worker=self.work_request.worker,
            status=WorkRequest.Statuses.RUNNING,
        )
        rdep.add_dependency(work_request2)
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.BLOCKED)

    def test_dependencies_in_pending_workflow(self) -> None:
        """Parent workflow that isn't running yet keeps rdep blocked."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        rdep = workflow.create_child("noop")
        self.work_request.parent = workflow
        self.work_request.save()
        rdep.add_dependency(self.work_request)
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.BLOCKED)

    def test_dependencies_in_running_workflow(self) -> None:
        """Parent workflow in running status gets rdep pending."""
        template = self.playground.create_workflow_template("test", "noop")
        workflow = WorkRequest.objects.create_workflow(
            template=template, data={}, created_by=self.get_test_user()
        )
        rdep = workflow.create_child("noop")
        self.work_request.parent = workflow
        self.work_request.save()
        rdep.add_dependency(self.work_request)
        workflow.mark_running()
        self.work_request.mark_completed(WorkRequest.Results.SUCCESS)
        rdep.refresh_from_db()
        self.assertEqual(rdep.status, WorkRequest.Statuses.PENDING)

    def test_create_child_non_workflow(self) -> None:
        """Non-WORKFLOW work requests may not have children."""
        with self.assertRaisesRegex(
            ValueError, r"Only workflows may have child work requests\."
        ):
            self.work_request.create_child("noop")

    def test_update_collection_with_artifacts(self) -> None:
        """Update collection with artifacts on task completion."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        c = self.playground.create_collection(
            name=f"workflow-{wr.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        cm = CollectionManagerInterface.get_manager_for(c)
        a_replaced, _ = self.create_artifact(
            category=ArtifactCategory.TEST,
            data={
                "deb_fields": {
                    "Package": "actiontest",
                    "Version": "1.0-1",
                },
                "type": "rebuild",
            },
        )
        cm.add_artifact(
            a_replaced, user=wr.created_by, name="actiontest_rebuild"
        )
        item = cm.lookup("name:actiontest_rebuild")
        assert item is not None
        self.assertEqual(item.artifact, a_replaced)

        a_ignored, _ = self.create_artifact(
            category=ArtifactCategory.TEST,
            data={
                "deb_fields": {
                    "Package": "ignored",
                    "Version": "1.0-1",
                },
                "type": "rebuild",
            },
        )
        cm.add_artifact(a_ignored, user=wr.created_by, name="ignored_rebuild")
        item = cm.lookup("name:ignored_rebuild")
        assert item is not None
        self.assertEqual(item.artifact, a_ignored)

        a_replacement, _ = self.create_artifact(
            category=ArtifactCategory.TEST,
            data={
                'deb_fields': {
                    "Package": "actiontest",
                    "Version": "1.0-1+b1",
                },
                "type": "rebuild",
            },
            work_request=wr,
        )

        a_added, _ = self.create_artifact(
            category=ArtifactCategory.TEST,
            data={
                "deb_fields": {
                    "Package": "actiontest2",
                    "Version": "2.0-1+b1",
                },
                "type": "rebuild",
            },
            work_request=wr,
        )

        action = ActionUpdateCollectionWithArtifacts.parse_obj(
            {
                "artifact_filters": {
                    "category": ArtifactCategory.TEST,
                    "data__deb_fields__Package__startswith": "actiontest",
                },
                "collection": "internal@collections",
                "name_template": "{name}_{type}",
                "variables": {
                    "$name": "deb_fields.Package",
                    "$type": "type",
                },
            }
        )
        wr.event_reactions = EventReactions(
            on_success=[action],
            on_failure=[action],
        )
        wr.mark_completed(WorkRequest.Results.SUCCESS)
        self.assertIsNotNone(
            CollectionItem.objects.get(
                parent_collection=c, artifact=a_replaced
            ).removed_at
        )
        item = cm.lookup("name:ignored_rebuild")
        assert item is not None
        self.assertEqual(item.artifact, a_ignored)
        item = cm.lookup("name:actiontest_rebuild")
        assert item is not None
        self.assertEqual(item.artifact, a_replacement)
        item = cm.lookup("name:actiontest2_rebuild")
        assert item is not None
        self.assertEqual(item.artifact, a_added)

    def test_update_collection_with_artifacts_no_name_template(self) -> None:
        """Without name_template, variables are passed through."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        c = Collection.objects.create(
            name="debian",
            category=CollectionCategory.ENVIRONMENTS,
            workspace=default_workspace(),
        )
        tarball, _ = self.create_artifact(
            category=ArtifactCategory.SYSTEM_TARBALL,
            data={"architecture": "amd64"},
            work_request=wr,
        )
        action = ActionUpdateCollectionWithArtifacts.parse_obj(
            {
                "artifact_filters": {
                    "category": ArtifactCategory.SYSTEM_TARBALL
                },
                "collection": "debian@debian:environments",
                "variables": {"codename": "bookworm", "variant": "buildd"},
            }
        )
        wr.event_reactions = EventReactions(on_success=[action])

        wr.mark_completed(WorkRequest.Results.SUCCESS)

        item = c.manager.lookup(
            "match:format=tarball:codename=bookworm:architecture=amd64:"
            "variant=buildd"
        )
        assert item is not None
        self.assertIsNotNone(item.artifact, tarball)

    def test_update_collection_with_artifacts_constraint_violation(
        self,
    ) -> None:
        """Violating collection constraints is an error."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        self.playground.create_collection(
            name=f"workflow-{wr.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        self.create_artifact(category=ArtifactCategory.TEST, work_request=wr)
        # This action does not specify an item name.
        action = ActionUpdateCollectionWithArtifacts(
            artifact_filters={"category": ArtifactCategory.TEST},
            collection="internal@collections",
        )
        wr.event_reactions = EventReactions(on_success=[action])
        wr.status = WorkRequest.Statuses.RUNNING
        with self.assertLogsContains(
            "Cannot replace or add artifact",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            wr.mark_completed(WorkRequest.Results.SUCCESS)

    def test_update_collection_with_artifacts_invalid_artifact_filters(
        self,
    ) -> None:
        """A failure to filter artifacts is an error."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        self.playground.create_collection(
            name=f"workflow-{wr.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        action = ActionUpdateCollectionWithArtifacts.parse_obj(
            {
                "artifact_filters": {"xxx": "yyy"},
                "collection": "internal@collections",
            }
        )
        wr.event_reactions = EventReactions(on_success=[action])
        with self.assertLogsContains(
            "Invalid update-collection-with-artifacts artifact_filters",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            wr.mark_completed(WorkRequest.Results.SUCCESS)

    def test_update_collection_with_artifacts_invalid_variables(self) -> None:
        """A failure to expand variables is an error."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        self.playground.create_collection(
            name=f"workflow-{wr.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        self.create_artifact(category=ArtifactCategory.TEST, work_request=wr)
        action = ActionUpdateCollectionWithArtifacts.parse_obj(
            {
                "artifact_filters": {"category": ArtifactCategory.TEST},
                "collection": "internal@collections",
                "name_template": "{name}",
                "variables": {"$name": "zzz"},
            }
        )
        wr.event_reactions = EventReactions(on_success=[action])
        with self.assertLogsContains(
            "Invalid update-collection-with-artifacts variables",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            wr.mark_completed(WorkRequest.Results.SUCCESS)

    def test_update_collection_with_artifacts_different_workspace(self) -> None:
        """The updated collection must be in the same workspace."""
        wr = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        different_workspace = self.playground.create_workspace(name="test")
        Collection.objects.create(
            name="test_differentworkspace",
            category=CollectionCategory.WORKFLOW_INTERNAL,
            workspace=different_workspace,
        )
        action = ActionUpdateCollectionWithArtifacts.parse_obj(
            {
                "collection": (
                    "test_differentworkspace@debusine:workflow-internal"
                ),
                "artifact_filters": {"category": ArtifactCategory.TEST},
            }
        )
        wr.event_reactions = EventReactions(on_success=[action])
        with self.assertLogsContains(
            "'test_differentworkspace@debusine:workflow-internal' does not "
            "exist or is hidden",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            wr.mark_completed(WorkRequest.Results.SUCCESS)

    def test_update_collection_with_data_on_creation(self) -> None:
        """
        Update collection with data on work request creation.

        This can also be done on completion, but
        `update-collection-with-artifacts` is usually more useful in that
        case.
        """
        workflow = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        c = self.playground.create_collection(
            name=f"workflow-{workflow.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        cm = CollectionManagerInterface.get_manager_for(c)

        action = ActionUpdateCollectionWithData.parse_obj(
            {
                "collection": "internal@collections",
                "category": BareDataCategory.TEST,
                "name_template": "{package}_{version}",
                "data": {"package": "hello", "version": "1.0-1"},
            }
        )
        workflow.create_child(
            "noop", event_reactions=EventReactions(on_creation=[action])
        )
        item = cm.lookup("name:hello_1.0-1")
        assert item is not None
        self.assertEqual(item.data, {"package": "hello", "version": "1.0-1"})

    def test_update_collection_with_data_on_unblock(self) -> None:
        """Update collection with data when a work request is unblocked."""
        workflow = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        c = self.playground.create_collection(
            name=f"workflow-{workflow.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )
        cm = CollectionManagerInterface.get_manager_for(c)

        action = ActionUpdateCollectionWithData.parse_obj(
            {
                "collection": "internal@collections",
                "category": BareDataCategory.TEST,
                "name_template": "{package}_{version}",
                "data": {"package": "hello", "version": "1.0-1"},
            }
        )
        wr = workflow.create_child(
            "noop", event_reactions=EventReactions(on_unblock=[action])
        )
        wr.unblock_strategy = WorkRequest.UnblockStrategy.MANUAL
        wr.save()

        self.assertIsNone(cm.lookup("name:hello_1.0-1"))

        wr.mark_pending()

        item = cm.lookup("name:hello_1.0-1")
        assert item is not None
        self.assertEqual(item.data, {"package": "hello", "version": "1.0-1"})

    def test_update_collection_with_data_no_name_template(self) -> None:
        """Without name_template, name=None is passed."""
        workflow = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        c = self.playground.create_collection(
            name=f"workflow-{workflow.id}",
            category=CollectionCategory.WORKFLOW_INTERNAL,
        )

        action = ActionUpdateCollectionWithData.parse_obj(
            {
                "collection": "internal@collections",
                "category": BareDataCategory.TEST,
                "data": {"package": "hello", "version": "1.0-1"},
            }
        )
        # We don't currently have a collection that accepts creating bare
        # data items without a name, but that's OK because the log message
        # is reasonably distinctive.
        with self.assertLogsContains(
            f"Cannot replace or add bare data of category "
            f"{BareDataCategory.TEST} to collection {c}",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            workflow.create_child(
                "noop", event_reactions=EventReactions(on_creation=[action])
            )

    def test_update_collection_with_data_different_workspace(self) -> None:
        """The updated collection must be in the same workspace."""
        workflow = WorkRequest.objects.create(
            workspace=default_workspace(),
            created_by=self.get_test_user(),
            task_type=TaskTypes.WORKFLOW,
            task_name="test",
        )
        different_workspace = self.playground.create_workspace(name="test")
        Collection.objects.create(
            name="test_differentworkspace",
            category=CollectionCategory.WORKFLOW_INTERNAL,
            workspace=different_workspace,
        )
        action = ActionUpdateCollectionWithData.parse_obj(
            {
                "collection": (
                    "test_differentworkspace@debusine:workflow-internal"
                ),
                "category": BareDataCategory.TEST,
            }
        )
        with self.assertLogsContains(
            "'test_differentworkspace@debusine:workflow-internal' does not "
            "exist or is hidden",
            logger="debusine.db.models",
            level=logging.ERROR,
        ):
            workflow.create_child(
                "noop", event_reactions=EventReactions(on_creation=[action])
            )

    def test_retry_with_delays(self) -> None:
        """Automatically retry a work request on failure, with a delay."""
        workflow = self.playground.create_workflow(
            status=WorkRequest.Statuses.RUNNING
        )
        action = ActionRetryWithDelays(delays=["30m", "1h", "2d", "1w"])
        wr = workflow.create_child(
            "noop",
            status=WorkRequest.Statuses.PENDING,
            event_reactions=EventReactions(on_failure=[action]),
        )

        for retry_count, delay in (
            (1, timedelta(minutes=30)),
            (2, timedelta(hours=1)),
            (3, timedelta(days=2)),
            (4, timedelta(weeks=1)),
        ):
            now = timezone.now()

            with mock.patch("django.utils.timezone.now", return_value=now):
                wr.mark_completed(WorkRequest.Results.FAILURE)

            self.assertEqual(wr.status, WorkRequest.Statuses.COMPLETED)
            self.assertEqual(wr.result, WorkRequest.Results.FAILURE)
            wr_new = wr.superseded
            assert wr_new.workflow_data is not None
            self.assertEqual(wr_new.workflow_data.retry_count, retry_count)
            self.assertEqual(wr_new.dependencies.count(), retry_count)
            wr_delay = wr_new.dependencies.latest("created_at")
            self.assertEqual(wr_delay.status, WorkRequest.Statuses.PENDING)
            self.assertEqual(
                wr_delay.task_data["delay_until"],
                DjangoJSONEncoder().default(now + delay),
            )

            # Pretend that the delay task has expired.
            wr_delay.mark_completed(WorkRequest.Results.SUCCESS)
            wr_new.refresh_from_db()

            wr = wr_new

        # Failing once more does not retry again, since the delays list has
        # been exhausted.
        wr.mark_completed(WorkRequest.Results.FAILURE)

        self.assertEqual(wr.status, WorkRequest.Statuses.COMPLETED)
        self.assertEqual(wr.result, WorkRequest.Results.FAILURE)
        self.assertFalse(hasattr(wr, "superseded"))

    def test_retry_with_delays_outside_workflow(self) -> None:
        """The retry-with-delays action may only be used in a workflow."""
        wr = self.playground.create_work_request(
            event_reactions=EventReactions(
                on_failure=[ActionRetryWithDelays(delays=["1m"])]
            )
        )

        with self.assertRaisesRegex(
            CannotRetry,
            "retry-with-delays action may only be used in a workflow",
        ):
            wr.mark_completed(WorkRequest.Results.FAILURE)


class WorkflowTemplateTests(TestCase):
    """Unit tests for the WorkflowTemplate class."""

    def assert_data_must_be_dictionary(
        self, wt: WorkflowTemplate, messages: list[str] | None = None
    ):
        """Check that a WorkflowTemplate's task_data is a dictionary."""
        if messages is None:
            messages = ["task data must be a dictionary"]
        with self.assertRaises(ValidationError) as raised:
            wt.full_clean()
        self.assertEqual(
            raised.exception.message_dict,
            {"task_data": messages},
        )

    def test_clean_task_data_must_be_dict(self) -> None:
        """Ensure that task_data is a dict."""
        wt = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name="noop",
            task_data={},
        )
        wt.full_clean()

        wt.task_data = None
        self.assert_data_must_be_dictionary(wt)

        wt.task_data = ""
        self.assert_data_must_be_dictionary(wt)

        wt.task_data = 3
        self.assert_data_must_be_dictionary(wt)

        wt.task_data = []
        self.assert_data_must_be_dictionary(wt)

        wt.task_data = object()
        self.assert_data_must_be_dictionary(
            wt,
            messages=[
                'Value must be valid JSON.',
                'task data must be a dictionary',
            ],
        )

    @preserve_task_registry()
    def test_validate_task_data(self) -> None:
        """Test orchestrator validation of task data."""

        class ValidatingWorkflow(
            TestTaskMixin, Workflow[BaseWorkflowData, BaseDynamicTaskData]
        ):
            """Workflow used to test validation of template data."""

            @classmethod
            def validate_template_data(cls, data: dict[str, Any]) -> None:
                """data-controlled validation."""
                if msg := data.get("error"):
                    raise ValueError(msg)

            def populate(self) -> None:
                """Unused abstract method from Workflow."""
                raise NotImplementedError()

        wt = WorkflowTemplate.objects.create(
            name="test",
            workspace=default_workspace(),
            task_name="validatingworkflow",
            task_data={},
        )
        wt.full_clean()

        wt.task_data = {"error": "example message"}
        with self.assertRaises(ValidationError) as raised:
            wt.full_clean()
        self.assertEqual(
            raised.exception.message_dict,
            {"task_data": ["example message"]},
        )
