# Copyright 2021-2023 The Debusine Developers
# See the AUTHORS file at the top-level directory of this distribution
#
# This file is part of Debusine. It is subject to the license terms
# in the LICENSE file found in the top-level directory of this
# distribution. No part of Debusine, including this file, may be copied,
# modified, propagated, or distributed except according to the terms
# contained in the LICENSE file.

"""Tests for the management command delete_expired."""

import contextlib
import io
import threading
from collections.abc import Iterable, Iterator
from datetime import timedelta
from typing import Any, Optional
from unittest import mock

from django.db import OperationalError, connection, transaction
from django.db.models.manager import Manager
from django.utils import timezone

from debusine.artifacts.models import ArtifactCategory, CollectionCategory
from debusine.db.models import (
    Artifact,
    ArtifactRelation,
    Collection,
    CollectionItem,
    File,
    FileInArtifact,
    FileInStore,
    FileStore,
    WorkRequest,
    WorkflowTemplate,
    default_file_store,
    default_workspace,
)
from debusine.db.tests.utils import (
    RunInParallelTransaction,
    RunInThreadAndCloseDBConnections,
)
from debusine.server.collections import DebianEnvironmentsManager
from debusine.server.management.commands.delete_expired import (
    DeleteExpiredArtifacts,
    DeleteExpiredWorkRequests,
    DeleteOperation,
)
from debusine.server.management.commands.tests import call_command
from debusine.test.django import TestCase, TransactionTestCase
from debusine.test.utils import tomorrow


class DeleteExpiredCommandTests(TestCase):
    """Tests for delete_expired command."""

    def setUp(self):
        """Set up common objects for the tests."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()
        artifact_2, _ = self.create_artifact(expiration_delay=1)
        artifact_2.created_at = timezone.now() - timedelta(days=2)
        artifact_2.save()

        ArtifactRelation.objects.create(
            artifact=artifact_1,
            target=artifact_2,
            type=ArtifactRelation.Relations.BUILT_USING,
        )

    def test_delete_expired(self):
        """Test delete_expired command delete artifacts."""
        self.assertEqual(Artifact.objects.count(), 2)
        self.assertEqual(ArtifactRelation.objects.count(), 1)

        now = timezone.now()

        func_dfh = mock.Mock()
        func_dm = mock.Mock()
        func_tzn = mock.Mock()
        func_tzn.return_value = now
        with (
            mock.patch(
                "debusine.db.models.collections"
                ".CollectionItemManager.drop_full_history",
                func_dfh,
            ),
            mock.patch(
                "debusine.db.models.collections"
                ".CollectionItemManager.drop_metadata",
                func_dm,
            ),
            mock.patch("django.utils.timezone.now", func_tzn),
        ):
            stdout, stderr, retval = call_command("delete_expired")
        func_dfh.assert_called_once_with(now)
        func_dm.assert_called_once_with(now)

        self.assertEqual(Artifact.objects.count(), 0)
        self.assertEqual(ArtifactRelation.objects.count(), 0)

        self.assertEqual(stdout, "")
        self.assertEqual(stderr, "")
        self.assertEqual(retval, 0)

    def test_delete_expired_dry_run_verbose(self):
        """Test delete_expired artifacts dry-run: no deletion."""
        artifact_id_1, artifact_id_2 = Artifact.objects.all()

        # Add an artifact with a file
        artifact_id_3, _ = self.create_artifact(
            paths=["README"], create_files=True, expiration_delay=1
        )
        artifact_id_3.created_at = timezone.now() - timedelta(days=2)
        artifact_id_3.save()

        collection_1 = self.create_collection(
            "internal", CollectionCategory.WORKFLOW_INTERNAL
        )
        work_request_1 = self.create_work_request(
            created_at=timezone.now() - timedelta(days=365),
            expiration_delay=timedelta(days=1),
            internal_collection=collection_1,
        )

        self.assertEqual(Artifact.objects.count(), 3)
        self.assertEqual(ArtifactRelation.objects.count(), 1)
        self.assertEqual(FileInStore.objects.all().count(), 1)
        self.assertEqual(WorkRequest.objects.all().count(), 1)
        self.assertEqual(Collection.objects.all().count(), 1)

        stdout, stderr, retval = call_command(
            "delete_expired", "--dry-run", verbosity=2
        )

        self.assertEqual(
            stdout,
            "dry-run mode: no changes will be made\n"
            + _format_deleted_work_requests([work_request_1], [collection_1])
            + _format_deleted_artifacts(
                {artifact_id_1, artifact_id_2, artifact_id_3}
            )
            + "dry-run mode: no changes were made\n",
        )

        # Artifacts, ArtifactRelation didn't get deleted
        self.assertEqual(Artifact.objects.count(), 3)
        self.assertEqual(ArtifactRelation.objects.count(), 1)

        # Files didn't get deleted
        self.assertEqual(FileInStore.objects.all().count(), 1)

        # Work requests and their internal collections didn't get deleted
        self.assertEqual(WorkRequest.objects.all().count(), 1)
        self.assertEqual(Collection.objects.all().count(), 1)

    def test_delete_expired_dry_run(self):
        """Test --dry-run without verbose does not write to stdout."""
        stdout, stderr, retval = call_command("delete_expired", "--dry-run")

        # dry-run without verbose: no messages are written to stdout.
        self.assertEqual(stdout, "")

    def test_delete_expired_verbose(self):
        """Test delete_expired verbose: print information."""
        artifact_id_1 = Artifact.objects.all()[0]
        artifact_id_2 = Artifact.objects.all()[1]
        collection_1 = self.create_collection(
            "internal", CollectionCategory.WORKFLOW_INTERNAL
        )
        collection_1.manager.add_artifact(
            artifact_id_1, user=self.get_test_user(), name="artifact1"
        )
        collection_1.manager.add_artifact(
            artifact_id_2, user=self.get_test_user(), name="artifact2"
        )
        work_request_1 = self.create_work_request(
            created_at=timezone.now() - timedelta(days=365),
            expiration_delay=timedelta(days=1),
            internal_collection=collection_1,
        )
        self.assertEqual(WorkRequest.objects.count(), 1)
        self.assertEqual(Collection.objects.count(), 1)

        stdout, stderr, retval = call_command("delete_expired", verbosity=2)

        self.assertEqual(
            stdout,
            _format_deleted_work_requests([work_request_1], [collection_1])
            + _format_deleted_artifacts({artifact_id_1, artifact_id_2}),
        )
        self.assertEqual(Artifact.objects.count(), 0)
        self.assertEqual(ArtifactRelation.objects.count(), 0)
        self.assertEqual(WorkRequest.objects.count(), 0)
        self.assertEqual(Collection.objects.count(), 0)

    def test_delete_expired_verbose_nothing_deleted(self):
        """Test delete_expired verbose: print no artifact expired."""
        Artifact.objects.update(created_at=tomorrow())
        stdout, stderr, retval = call_command("delete_expired", verbosity=2)

        self.assertEqual(
            stdout,
            _format_deleted_work_requests([], [])
            + "There were no expired artifacts\n",
        )


class DeleteExpiredArtifactsTests(TestCase):
    """Tests for the DeleteExpiredArtifacts class."""

    playground_memory_file_store = False

    def setUp(self):
        """Initialize test."""
        self.out = io.StringIO()
        self.err = io.StringIO()
        self.operation = DeleteOperation(out=self.out, err=self.err)
        self.operation.initial_time = timezone.now()
        self.delete_expired = DeleteExpiredArtifacts(self.operation)

    def test_mark_to_keep_artifact_no_expiration_date(self):
        """One artifact, not expired: keep it."""
        artifact, _ = self.create_artifact(expiration_delay=0)

        self.assertEqual(self.delete_expired._mark_to_keep(), {artifact.id})

    def test_mark_to_keep_artifact_expires_tomorrow(self):
        """One artifact, not expired (expires tomorrow): keep it."""
        artifact, _ = self.create_artifact(expiration_delay=1)

        self.assertEqual(self.delete_expired._mark_to_keep(), {artifact.id})

    def test_mark_to_keep_only_artifact_is_expired(self):
        """One artifact, expired yesterday. No artifacts are kept."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()

        collection_parent_1 = Collection.objects.create(
            name="collection-1",
            category="collection-1",
            workspace=default_workspace(),
            retains_artifacts=Collection.RetainsArtifacts.NEVER,
        )
        CollectionItem.objects.create(
            name="test-2",
            category="test-2",
            parent_collection=collection_parent_1,
            child_type=CollectionItem.Types.ARTIFACT,
            artifact=artifact_1,
            created_by_user=self.get_test_user(),
        )

        self.assertEqual(self.delete_expired._mark_to_keep(), set())

    def test_mark_to_keep_only_artifact_is_retained(self):
        """One artifact, expired yesterday. retains_artifacts is set."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()

        collection_parent_1 = Collection.objects.create(
            name="collection-1",
            category="collection-1",
            workspace=default_workspace(),
        )
        CollectionItem.objects.create(
            name="test-2",
            category="test-2",
            parent_collection=collection_parent_1,
            child_type=CollectionItem.Types.ARTIFACT,
            artifact=artifact_1,
            created_by_user=self.get_test_user(),
        )

        self.assertEqual(self.delete_expired._mark_to_keep(), {artifact_1.id})

    def test_mark_to_keep_artifacts_retained_by_workflow(self) -> None:
        """RetainsArtifacts.WORKFLOWS retains while the workflow is active."""
        template = WorkflowTemplate.objects.create(
            name="test", workspace=default_workspace(), task_name="noop"
        )
        collections: dict[WorkRequest.Statuses, Collection] = {}
        artifacts: dict[WorkRequest.Statuses, Artifact] = {}
        for status in WorkRequest.Statuses:
            workflow = WorkRequest.objects.create_workflow(
                template=template, data={}, created_by=self.get_test_user()
            )
            workflow.status = status
            workflow.save()
            assert workflow.internal_collection is not None
            collections[status] = workflow.internal_collection
            collections[status].retains_artifacts = (
                Collection.RetainsArtifacts.WORKFLOW
            )
            collections[status].save()
            artifacts[status], _ = self.create_artifact(expiration_delay=1)
            artifacts[status].created_at = timezone.now() - timedelta(days=2)
            artifacts[status].save()
            collections[status].manager.add_artifact(
                artifacts[status], user=self.get_test_user(), name=str(status)
            )

        self.assertEqual(
            self.delete_expired._mark_to_keep(),
            {
                artifacts[WorkRequest.Statuses.PENDING].id,
                artifacts[WorkRequest.Statuses.RUNNING].id,
                artifacts[WorkRequest.Statuses.BLOCKED].id,
            },
        )

    def test_mark_to_keep_all_artifacts_non_expired_or_targeted(self):
        """Keep both artifacts: the expired is targeted by the not-expired."""
        artifact_expired, _ = self.create_artifact(expiration_delay=1)
        artifact_expired.created_at = timezone.now() - timedelta(days=2)
        artifact_expired.save()
        artifact_not_expired, _ = self.create_artifact(expiration_delay=1)

        self.create_artifact_relation(artifact_not_expired, artifact_expired)

        self.assertEqual(
            self.delete_expired._mark_to_keep(),
            {artifact_expired.id, artifact_not_expired.id},
        )

    def test_mark_to_keep_no_artifact_all_expired_two(self):
        """Two artifacts (related), both expired."""
        artifact_expired_1, _ = self.create_artifact(expiration_delay=1)
        artifact_expired_1.created_at = timezone.now() - timedelta(days=2)
        artifact_expired_1.save()
        artifact_expired_2, _ = self.create_artifact(expiration_delay=1)
        artifact_expired_2.created_at = timezone.now() - timedelta(days=2)
        artifact_expired_2.save()

        self.create_artifact_relation(artifact_expired_1, artifact_expired_2)

        self.assertEqual(self.delete_expired._mark_to_keep(), set())

    def test_mark_to_keep_no_artifact_all_expired_three(self):
        """All artifacts are expired."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()
        artifact_2, _ = self.create_artifact(expiration_delay=1)
        artifact_2.created_at = timezone.now() - timedelta(days=2)
        artifact_2.save()
        artifact_3, _ = self.create_artifact(expiration_delay=1)
        artifact_3.created_at = timezone.now() - timedelta(days=2)
        artifact_3.save()

        self.create_artifact_relation(artifact_2, artifact_1)
        self.create_artifact_relation(artifact_3, artifact_2)

        self.assertEqual(self.delete_expired._mark_to_keep(), set())

    def test_mark_to_keep_two_artifacts_non_expired_and_targeted(self):
        """Three artifacts, keep two (non-expired and target of non-expired)."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()
        artifact_2, _ = self.create_artifact(expiration_delay=0)
        artifact_3, _ = self.create_artifact(expiration_delay=1)
        artifact_3.created_at = timezone.now() - timedelta(days=2)
        artifact_3.save()

        self.create_artifact_relation(artifact_2, artifact_1)
        self.create_artifact_relation(artifact_3, artifact_2)

        self.assertEqual(
            self.delete_expired._mark_to_keep(),
            {artifact_1.id, artifact_2.id},
        )

    def test_mark_to_keep_isolated(self):
        """Two expired and one non-expired isolated."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()
        artifact_2, _ = self.create_artifact(expiration_delay=1)
        artifact_2.created_at = timezone.now() - timedelta(days=2)
        artifact_2.save()
        artifact_3, _ = self.create_artifact(expiration_delay=0)

        self.create_artifact_relation(artifact_2, artifact_1)
        self.create_artifact_relation(artifact_1, artifact_2)

        self.assertEqual(self.delete_expired._mark_to_keep(), {artifact_3.id})

    def test_mark_to_keep_artifacts_circular_dependency(self):
        """Artifacts are not expired (keep), have a circular dependency."""
        artifact_1, _ = self.create_artifact(expiration_delay=0)
        artifact_2, _ = self.create_artifact(expiration_delay=0)

        self.create_artifact_relation(artifact_2, artifact_1)
        self.create_artifact_relation(artifact_1, artifact_2)

        self.assertEqual(
            self.delete_expired._mark_to_keep(),
            {artifact_1.id, artifact_2.id},
        )

    def test_sweep_delete_artifacts(self):
        """Sweep() delete the artifacts and print progress."""
        artifact_1, _ = self.create_artifact(expiration_delay=1)
        artifact_1.created_at = timezone.now() - timedelta(days=2)
        artifact_1.save()
        artifact_2, _ = self.create_artifact(expiration_delay=1)
        artifact_2.created_at = timezone.now() - timedelta(days=2)
        artifact_2.save()

        self.create_artifact_relation(artifact_2, artifact_1)
        self.create_artifact_relation(artifact_1, artifact_2)

        artifact_3, _ = self.create_artifact(expiration_delay=1)
        artifact_3.created_at = timezone.now() - timedelta(days=2)
        artifact_3.save()
        artifact_4, _ = self.create_artifact(expiration_delay=0)

        self.create_artifact_relation(artifact_4, artifact_3)

        artifacts_to_keep = self.delete_expired._mark_to_keep()

        self.operation._verbosity = 2
        self.operation.dry_run = False
        self.delete_expired._sweep(artifacts_to_keep)

        self.assertEqual(Artifact.objects.all().count(), 2)
        self.assertEqual(
            self.out.getvalue(),
            _format_deleted_artifacts({artifact_1, artifact_2}),
        )

    def test_sweep_delete_artifacts_no_retains(self):
        """Sweep() delete the artifacts from the collections."""
        artifact, _ = self.create_artifact(
            category=ArtifactCategory.SYSTEM_TARBALL,
            data={
                "codename": "bookworm",
                "architecture": "amd64",
                "variant": "apt",
                "with_dev": True,
            },
            expiration_delay=1,
        )
        artifact.created_at = timezone.now() - timedelta(days=2)
        artifact.save()

        collection = Collection.objects.create(
            name="Debian",
            category=CollectionCategory.ENVIRONMENTS,
            workspace=default_workspace(),
            retains_artifacts=Collection.RetainsArtifacts.NEVER,
        )

        manager = DebianEnvironmentsManager(collection=collection)
        manager.add_artifact(
            artifact,
            user=self.get_test_user(),
            variables={"codename": "trixie"},
        )

        artifacts_to_keep = self.delete_expired._mark_to_keep()

        self.operation._verbosity = 2
        self.operation.dry_run = False
        self.delete_expired._sweep(artifacts_to_keep)

        self.assertEqual(Artifact.objects.all().count(), 0)
        self.assertEqual(
            self.out.getvalue(),
            _format_deleted_artifacts({artifact}),
        )

    def test_delete_artifact_all_related_files(self):
        """_delete_artifact() deletes all the artifact's related models."""
        path_in_artifact = "README"
        artifact, _ = self.create_artifact(
            paths=[path_in_artifact], create_files=True
        )

        # The file was added in the artifact and in the store
        self.assertEqual(FileInArtifact.objects.all().count(), 1)
        self.assertEqual(FileInStore.objects.all().count(), 1)
        self.assertEqual(File.objects.all().count(), 1)

        # Retrieve the store_backend for the file
        fileobj = File.objects.all().first()
        file_in_store = FileInStore.objects.all().first()
        store_backend = file_in_store.store.get_backend_object()

        # The file exists
        filepath = store_backend.get_local_path(fileobj)
        self.assertTrue(filepath.exists())

        # The Artifact is going to be deleted and the only file with it
        self.operation.dry_run = False
        self.delete_expired._delete_artifact(artifact)

        self.assertFalse(Artifact.objects.filter(id=artifact.id).exists())
        self.assertEqual(FileInArtifact.objects.all().count(), 0)

        # FileInStore and File are not deleted: they are deleted after
        # the artifact is deleted in a different transaction (done in
        # DeleteExpiredArtifacts.run() )
        self.assertEqual(FileInStore.objects.all().count(), 1)
        self.assertEqual(File.objects.all().count(), 1)

        self.assertTrue(filepath.exists())

    def test_delete_artifact_not_related_files(self):
        """
        Two files created, _delete_artifact() delete the Artifact.

        Check that only the files that are supposed to be deleted are deleted.
        """
        file_to_keep_name = "README"  # used in two artifacts
        file_to_delete_name = (
            "another-file"  # used only in the deleted artifact
        )

        # Create the artifacts
        artifact_to_delete, _ = self.create_artifact(
            paths=[file_to_keep_name, file_to_delete_name], create_files=True
        )
        artifact_to_keep, _ = self.create_artifact()

        # Get both files
        file_to_keep = FileInArtifact.objects.get(
            artifact=artifact_to_delete, path=file_to_keep_name
        ).file
        file_to_delete = FileInArtifact.objects.get(
            artifact=artifact_to_delete, path=file_to_delete_name
        ).file

        # Add file_to_keep in the artifact that is not being deleted
        # (so the file is kept)
        FileInArtifact.objects.create(
            artifact=artifact_to_keep, file=file_to_keep
        )

        store_backend = FileInStore.objects.get(
            file=file_to_keep
        ).store.get_backend_object()

        file_to_keep_path = store_backend.get_local_path(file_to_keep)
        file_to_delete_path = store_backend.get_local_path(file_to_delete)

        # Both files exist on disk
        self.assertTrue(file_to_keep_path.exists())
        self.assertTrue(file_to_delete_path.exists())

        self.operation.dry_run = False

        files_to_delete = self.delete_expired._delete_artifact(
            artifact_to_delete
        )

        self.assertCountEqual(files_to_delete, [file_to_keep, file_to_delete])

        # Both files still exist on disk (they would be deleted by
        # DeleteExpiredArtifacts._delete_files_from_stores)
        self.assertTrue(file_to_keep_path.exists())
        self.assertTrue(file_to_delete_path.exists())

    def test_delete_artifact_dry_run(self):
        """_delete_artifact() does nothing: running in dry-run."""
        artifact, _ = self.create_artifact(expiration_delay=1)
        artifact.created_at = timezone.now() - timedelta(days=2)
        artifact.save()

        # By default DeleteExpiredArtifacts runs in dry-run
        self.assertTrue(self.operation.dry_run)

        callables = self.delete_expired._delete_artifact(artifact)

        # Artifact was not deleted
        self.assertTrue(Artifact.objects.filter(id=artifact.id).exists())

        # No callables to delete files after the transaction returned
        self.assertEqual(len(callables), 0)

    def test_delete_files_from_stores(self):
        """_delete_files_from_stores delete the file."""
        fileobj, _ = File.get_or_create(
            hash_digest=b"19acff6019bb9ce73083b9df3b7a838cf47df117", size=1024
        )

        FileInStore.objects.create(store=default_file_store(), file=fileobj)

        self.delete_expired._delete_files_from_stores({fileobj})

        self.assertFalse(File.objects.filter(id=fileobj.id).exists())

    def test_delete_files_from_multiple_stores(self):
        """_delete_files_from_stores delete files from more than one store."""
        fileobj, _ = File.get_or_create(
            hash_digest=b"19acff6019bb9ce73083b9df3b7a838cf47df117", size=1024
        )

        # Add the file in the default_file_store()
        FileInStore.objects.create(store=default_file_store(), file=fileobj)

        # Add the same file in a secondary store
        secondary_store = FileStore.objects.create(
            name="Test",
            backend=FileStore.BackendChoices.LOCAL,
            configuration={"base_directory": "/"},
        )
        FileInStore.objects.create(store=secondary_store, file=fileobj)

        # _delete_files_from_stores should delete the file from multiple stores
        self.delete_expired._delete_files_from_stores({fileobj})

        # And the file is gone...
        self.assertFalse(File.objects.filter(id=fileobj.id).exists())

    def test_delete_files_from_stores_file_was_re_added(self):
        """
        _delete_files_from_stores does not delete the file.

        The file exist in another Artifact: cannot be deleted.
        """
        fileobj, _ = File.get_or_create(
            hash_digest=b"19acff6019bb9ce73083b9df3b7a838cf47df117", size=1024
        )
        files_to_delete = [fileobj]

        artifact, _ = self.create_artifact()
        FileInArtifact.objects.create(artifact=artifact, file=fileobj)

        self.delete_expired._delete_files_from_stores(files_to_delete)

        self.assertTrue(File.objects.filter(id=fileobj.id).exists())


class DeleteExpiredArtifactsTransactionTests(TransactionTestCase):
    """Tests for DeleteExpiredArtifacts that require transactions."""

    playground_memory_file_store = False

    def setUp(self):
        """Initialize test."""
        super().setUp()
        self.workspace = self.playground.get_default_workspace()

        self.artifact_1, _ = self.playground.create_artifact(
            expiration_delay=1, paths=["README"], create_files=True
        )
        self.artifact_1.created_at = timezone.now() - timedelta(days=2)
        self.artifact_1.save()
        self.artifact_2, _ = self.playground.create_artifact(expiration_delay=1)
        self.artifact_2.created_at = timezone.now() - timedelta(days=2)
        self.artifact_2.save()

        self.artifact_relation = self.playground.create_artifact_relation(
            self.artifact_1, self.artifact_2
        )

        self.out = io.StringIO()
        self.err = io.StringIO()

        self.operation = DeleteOperation(out=self.out, err=self.err)
        self.delete_expired = DeleteExpiredArtifacts(self.operation)

    def assert_delete_expired_run_failed(
        self,
        *,
        expected_artifacts: int,
        expected_artifact_relations: int,
        timeout: float,
        err: io.StringIO,
    ):
        """Assert result of deletion of artifacts when lock table failed."""
        self.assertEqual(Artifact.objects.all().count(), expected_artifacts)
        self.assertEqual(
            ArtifactRelation.objects.all().count(), expected_artifact_relations
        )

        self.assertEqual(
            err.getvalue(),
            f"Lock timed out ({timeout} seconds). Try again.\n",
        )

    def test_run_cannot_lock_artifact_relation_return_false(self):
        """
        run() cannot lock the tables of Artifact or ArtifactRelation.

        There is another transaction (created via RunInParallelTransaction).
        DeleteExpiredArtifacts.run() cannot lock the tables.
        """
        instances = [self.artifact_1, self.artifact_relation]

        for instance in instances:
            err = io.StringIO()
            self.operation._err = err

            delete_expired = DeleteExpiredArtifacts(self.operation)
            delete_expired._lock_timeout_secs = 0.1

            with self.subTest(instance._meta.object_name):
                expected_artifacts = Artifact.objects.all().count()
                expected_relations = ArtifactRelation.objects.all().count()

                manager = instance._meta.default_manager

                thread = RunInParallelTransaction(
                    lambda: manager.select_for_update().get(id=instance.id)
                )

                thread.start_transaction()

                with self.operation:
                    self.operation._verbosity = 1
                    self.operation.dry_run = False
                    delete_expired.run()

                thread.stop_transaction()

                self.assert_delete_expired_run_failed(
                    expected_artifacts=expected_artifacts,
                    expected_artifact_relations=expected_relations,
                    timeout=delete_expired._lock_timeout_secs,
                    err=err,
                )

    @staticmethod
    def check_table_locked_for(manager: Manager[Any]):
        """
        Call manager.first(). Expect timeout.

        If it does not time out: raise self.failureException().
        """
        with connection.cursor() as cursor:
            cursor.execute("SET lock_timeout TO '0.1s';")

            try:
                with transaction.atomic():
                    manager.first()
            except OperationalError:
                table_was_locked = True
            else:
                table_was_locked = False  # pragma: no cover

        return table_was_locked

    def test_tables_locked_on_sweep(self):
        """Check relevant tables are locked when _sweep() is called."""

        def wait_for_check(*args, **kwargs):
            """Notify that _sweep() is called, wait for the check."""
            sweep_is_called.set()
            wait_for_check_tables_locked.wait()
            return []

        sweep_is_called = threading.Event()
        wait_for_check_tables_locked = threading.Event()

        patcher = mock.patch.object(
            self.delete_expired, "_sweep", autospec=True
        )
        mocked_sweep = patcher.start()
        mocked_sweep.side_effect = wait_for_check
        self.addCleanup(patcher.stop)

        with self.operation:
            self.operation._verbosity = 2
            self.operation.dry_run = False
            delete_expired_run = RunInThreadAndCloseDBConnections(
                self.delete_expired.run
            )
            delete_expired_run.start_in_thread()

            sweep_is_called.wait()

            self.assertTrue(self.check_table_locked_for(Artifact.objects))
            self.assertTrue(
                self.check_table_locked_for(ArtifactRelation.objects)
            )
            self.assertTrue(self.check_table_locked_for(FileInArtifact.objects))
            self.assertTrue(self.check_table_locked_for(FileInStore.objects))
            self.assertTrue(self.check_table_locked_for(File.objects))

            wait_for_check_tables_locked.set()

            delete_expired_run.join()

    def test_on_commit_cleanup_delete_files(self):
        """File is deleted from the store if the transaction is committed."""
        artifact, _ = self.playground.create_artifact(
            paths=["README"], create_files=True, expiration_delay=1
        )
        artifact.created_at = timezone.now() - timedelta(days=2)
        artifact.save()

        message_artifacts_deleted = _format_deleted_artifacts(
            Artifact.objects.all()
        )

        self.assertTrue(Artifact.objects.filter(id=artifact.id).exists())
        file_in_store = FileInStore.objects.first()
        fileobj = file_in_store.file
        store_backend = file_in_store.store.get_backend_object()

        file_path = store_backend.get_local_path(fileobj)
        self.assertTrue(file_path.exists())

        with self.operation:
            self.operation._verbosity = 2
            self.operation.dry_run = False
            self.delete_expired.run()

        self.assertFalse(Artifact.objects.filter(id=artifact.id).exists())

        # The file does not exist anymore
        self.assertFalse(file_path.exists())

        self.assertEqual(
            self.out.getvalue(),
            message_artifacts_deleted + "Deleting files from the store\n",
        )

    def test_deleting_files_artifact_table_is_locked(self):
        """Files get deleted while the Artifact table is still locked."""
        # Delete a relation and an artifact, and create an artifact
        # that expires tomorrow. This is to simplify the test
        self.artifact_relation.delete()
        self.artifact_2.delete()
        self.playground.create_artifact(expiration_delay=1)

        patcher = mock.patch.object(
            self.delete_expired,
            "_delete_files_from_stores",
            autospec=True,
        )
        delete_files_mocked = patcher.start()
        run_in_thread = RunInThreadAndCloseDBConnections(
            self.check_table_locked_for, Artifact.objects
        )
        delete_files_mocked.side_effect = (
            lambda file_objs: run_in_thread.run_and_wait()  # noqa: U100
        )
        self.addCleanup(patcher.stop)

        with self.operation:
            self.operation._verbosity = 1
            self.operation.dry_run = False
            self.delete_expired.run()

        delete_files_mocked.assert_called()

    def test_delete_files_from_stores_raise_artifact_is_deleted(self):
        """Artifact is deleted even if the file store deletion fail."""
        exception = RuntimeError
        patcher = mock.patch.object(
            self.delete_expired,
            "_delete_files_from_stores",
            autospec=True,
        )
        delete_files_mocked = patcher.start()
        delete_files_mocked.side_effect = exception
        self.addCleanup(patcher.stop)

        with self.operation:
            self.operation._verbosity = 0
            self.operation.dry_run = False
            try:
                self.delete_expired.run()
            except exception:
                pass

        # _delete_files_from_stores raised an exception. Assert that the
        # artifact_2 deletion transaction is committed
        self.assertFalse(
            Artifact.objects.filter(id=self.artifact_2.id).exists()
        )


class MockCollection:
    """Minimal subset of Collection needed to test expiration."""

    pass


class MockWorkRequest:
    """Minimal subset of WorkRequest needed to test expiration."""

    def __init__(
        self,
        parent: Optional["MockWorkRequest"] = None,
        internal_collection: MockCollection | None = None,
    ) -> None:
        """Set the mock fields."""
        self.parent = parent
        self.internal_collection = internal_collection


class DeleteExpiredWorkRequestsTests(TestCase):
    """Tests for the DeleteExpiredWorkRequests class."""

    def setUp(self):
        """Initialize test."""
        self.out = io.StringIO()
        self.err = io.StringIO()
        self.operation = DeleteOperation(
            out=self.out, err=self.err, dry_run=False
        )
        self.operation.initial_time = timezone.now()
        self.delete_expired = DeleteExpiredWorkRequests(self.operation)

    @contextlib.contextmanager
    def expired(self, *work_requests: MockWorkRequest) -> Iterator[None]:
        """Simulate a given set of expired work requests."""
        with mock.patch(
            "debusine.db.models.work_requests.WorkRequestManager.expired",
            return_value=(wr for wr in work_requests),
        ):
            yield

    @contextlib.contextmanager
    def mock_perform_deletions(self) -> Iterator[mock.MagicMock]:
        """Shortcut to mock perform_deletions."""
        with mock.patch(
            "debusine.server.management.commands.delete_expired"
            ".DeleteExpiredWorkRequests.perform_deletions"
        ) as perform_deletions:
            yield perform_deletions

    @contextlib.contextmanager
    def mock_verbose(self) -> Iterator[mock.MagicMock]:
        """Shortcut to mock operation.verbose."""
        with mock.patch(
            "debusine.server.management.commands.delete_expired"
            ".DeleteOperation.verbose"
        ) as verbose:
            yield verbose

    def assert_dry_run_does_not_delete(self) -> None:
        """Make sure that, with dry_run, perform_deletions is not called."""
        old_dry_run = self.operation.dry_run
        self.operation.dry_run = True
        try:
            with self.expired(*self.delete_expired.work_requests):
                with self.mock_perform_deletions() as perform_deletions:
                    self.delete_expired.run()
            perform_deletions.assert_not_called()
        finally:
            self.operation.dry_run = old_dry_run

    def test_no_expired(self) -> None:
        """No expired work requests."""
        with (
            self.expired(),
            self.mock_perform_deletions() as perform_deletions,
            self.mock_verbose() as verbose,
        ):
            self.delete_expired.run()

        self.assertEqual(self.delete_expired.work_requests, set())
        self.assertEqual(self.delete_expired.collections, set())
        perform_deletions.assert_not_called()
        verbose.assert_called_once_with("There were no expired work requests\n")

    def test_one_expired(self) -> None:
        """One expired work request."""
        wr1 = MockWorkRequest()
        with (
            self.expired(wr1),
            self.mock_perform_deletions() as perform_deletions,
            self.mock_verbose() as verbose,
        ):
            self.delete_expired.run()

        self.assertEqual(self.delete_expired.work_requests, {wr1})
        self.assertEqual(self.delete_expired.collections, set())
        perform_deletions.assert_called()
        verbose.assert_called_once_with(
            "Deleting 1 expired work requests and 0 expired collections\n"
        )
        self.assert_dry_run_does_not_delete()

    def test_multiple_expired(self) -> None:
        """Multiple expired work requests."""
        wr1 = MockWorkRequest()
        wr2 = MockWorkRequest()
        wr3 = MockWorkRequest(parent=wr2)
        with (
            self.expired(wr1, wr2, wr3),
            self.mock_perform_deletions() as perform_deletions,
            self.mock_verbose() as verbose,
        ):
            self.delete_expired.run()

        self.assertEqual(self.delete_expired.work_requests, {wr1, wr2, wr3})
        self.assertEqual(self.delete_expired.collections, set())
        perform_deletions.assert_called()
        verbose.assert_called_once_with(
            "Deleting 3 expired work requests and 0 expired collections\n"
        )
        self.assert_dry_run_does_not_delete()

    def test_expired_but_valid_parent(self) -> None:
        """Keep work requests with non-expired parent."""
        wr1 = MockWorkRequest()
        wr2 = MockWorkRequest(parent=wr1)
        wr3 = MockWorkRequest()
        with (
            self.expired(wr2, wr3),
            self.mock_perform_deletions() as perform_deletions,
            self.mock_verbose() as verbose,
        ):
            self.delete_expired.run()

        self.assertEqual(self.delete_expired.work_requests, {wr3})
        self.assertEqual(self.delete_expired.collections, set())
        perform_deletions.assert_called()
        verbose.assert_called_once_with(
            "Deleting 1 expired work requests and 0 expired collections\n"
        )
        self.assert_dry_run_does_not_delete()

    def test_expired_with_collection(self) -> None:
        """Also delete internal collections."""
        c1 = MockCollection()
        wr1 = MockWorkRequest(internal_collection=c1)
        with (
            self.expired(wr1),
            self.mock_perform_deletions() as perform_deletions,
            self.mock_verbose() as verbose,
        ):
            self.delete_expired.run()

        self.assertEqual(self.delete_expired.work_requests, {wr1})
        self.assertEqual(self.delete_expired.collections, {c1})
        perform_deletions.assert_called()
        verbose.assert_called_once_with(
            "Deleting 1 expired work requests and 1 expired collections\n"
        )
        self.assert_dry_run_does_not_delete()


def _format_deleted_work_requests(
    work_requests: list[WorkRequest], collections: list[Collection]
) -> str:
    message = ""

    if work_requests or collections:
        message += (
            f"Deleting {len(work_requests)} expired work requests"
            f" and {len(collections)} expired collections\n"
        )
    else:
        message += "There were no expired work requests\n"

    return message


def _format_deleted_artifacts(artifacts: Iterable[Artifact]) -> str:
    message = ""

    for artifact in sorted(artifacts, key=lambda x: x.id):
        message += f"Deleted artifact {artifact.id}\n"

    return message
