r"""
Models of the django app *mini_buildd*

Naming conventions
==================

Model class and field names
---------------------------
All model class names and all field names must be **human
readable with no abbreviations** (as django, per default,
displays the internal names intelligently to the end user).

*Model class names* must be in **CamelCase**.

*Field names* must be all **lowercase** and **separated by underscores**.

For example, **don't** try to do sort of "grouping" using names like::

  email_notify
  email_allow_regex

This should rather read::

  notify
  allow_emails_to

To group fields together for the end user, use AdminModel's *fieldset* option.

Methods
-------
Any methods that represent mini-buildd logic should go into the models
directly, but must be prefixed with "mbd\_". This avoids conflicts
with method names form the django model's class, but still keeps the
logic where it belongs.

"""

import datetime
import re
import logging
import tempfile
import urllib.request
import urllib.error
import contextlib
import functools
import copy
import os
import shutil
import glob
import email.utils
import inspect
import shlex
import collections
import time
from contextlib import closing

import django.db.models
import django.contrib.admin
import django.contrib.messages
import django.template.response
import django.utils.timezone
import django.utils.html
import django.core.exceptions
import django.core.validators
import django.db
import django.contrib.auth.models
import django.core.mail

import debian.deb822
import debian.debian_support

from mini_buildd import config, events, dist, files, call, gnupg, sbuild, util, reprepro, schroot, net, ftpd, client, api, builtins

LOG = logging.getLogger(__name__)

MBD_MSGLEVEL2LOG = {
    django.contrib.messages.DEBUG: logging.DEBUG,
    django.contrib.messages.INFO: logging.INFO,
    django.contrib.messages.SUCCESS: logging.INFO,
    django.contrib.messages.WARNING: logging.WARNING,
    django.contrib.messages.ERROR: logging.ERROR,
}


def mbd_msg(request, msg_lines, prefix="", level=django.contrib.messages.INFO, level_followups=django.contrib.messages.INFO):
    """Convey message to user via django messaging, with extra multiline ('followups') formatting support"""
    if request is not None:
        for s in msg_lines.split("\n"):
            msg = f"{prefix}{s}"
            LOG.log(MBD_MSGLEVEL2LOG[level], msg)
            django.contrib.messages.add_message(request, level, django.utils.html.format_html(msg))
            level = level_followups
            prefix = "↳ "


def mbd_msg_exception(request, exception, prefix=""):
    mbd_msg(request, str(util.e2http(exception)), prefix=prefix, level=django.contrib.messages.ERROR, level_followups=django.contrib.messages.WARNING)
    util.log_exception(LOG, prefix, exception)


class UnixTextField(django.db.models.TextField):
    r"""Normalizes ``\r\n`` newlines (usually send by browsers) to Unix newlines ``\n``"""

    def get_prep_value(self, value):
        return value.replace("\r\n", "\n")


class Model(django.db.models.Model):
    """
    Abstract father model for all mini-buildd models

    This just makes sure no config is changed or deleted while
    the daemon is running.
    """

    MBD_HELP_EXTRA_OPTIONS = ""  # Set this to non-empty in derived models to "activate"
    extra_options = UnixTextField(blank=True, editable=True,
                                  help_text="Extra options (in the form ``<KEY>: <VALUE>``, one per line):")

    # DEPRECATED: In 1.0.x: Daemon: Kept last packages as pickled data (base64 encoded).
    pickled_data = django.db.models.TextField(blank=True)

    @classmethod
    def mbd_get_or_none(cls, **kwargs):
        """Get unambiguous instance or ``None`` if no such instance (``objects.get()`` w/o exception)"""
        try:
            return cls.objects.get(**kwargs)
        except cls.DoesNotExist:
            return None

    @classmethod
    def mbd_get_fields(cls, exclude=None):
        return [f for f in cls._meta.get_fields() if f.name not in ([] if exclude is None else exclude)]

    class Meta():
        abstract = True
        app_label = "mini_buildd"

    class Admin(django.contrib.admin.ModelAdmin):
        @classmethod
        def _mbd_on_change(cls, obj):
            """Global actions to take when an object changes"""
            for o in obj.mbd_get_reverse_dependencies():
                if o.LETHAL_DEPENDENCIES:
                    o.mbd_set_changed()
                    o.save()

        @classmethod
        def _mbd_on_activation(cls, obj):
            """Global actions to take when an object becomes active"""

        @classmethod
        def _mbd_on_deactivation(cls, obj):
            """Global actions to take when an object becomes inactive"""

        def save_model(self, request, obj, form, change):
            if change:
                self._mbd_on_change(obj)
            super().save_model(request, obj, form, change)

        def delete_model(self, request, obj):
            self._mbd_on_change(obj)

            is_prepared_func = getattr(obj, "mbd_is_prepared", None)
            if is_prepared_func and is_prepared_func():
                self.mbd_remove(obj)

            super().delete_model(request, obj)

        # All below is only to automatically disable/reorder and get custom help texts for 'extra_options'.
        def _mbd_help_fields(self):
            return {"pickled_data": "", "extra_options": self.model.MBD_HELP_EXTRA_OPTIONS}

        def get_fields(self, *args, **kwargs):
            """Make 'extra_options' and 'pickled_data' opt-on and last fields"""
            fields = super().get_fields(*args, **kwargs)

            for field, help_text in self._mbd_help_fields().items():
                if field in fields:
                    fields.remove(field)
                    if help_text:
                        fields.append(field)

            return fields

        def __init__(self, *args, **kwargs):
            """Postprocess fields help_text"""
            super().__init__(*args, **kwargs)

            help_fields = self._mbd_help_fields()
            patched = "_mbd_help_text_monkey_patched"
            for field_obj in self.model._meta.fields:
                if not hasattr(field_obj, patched):
                    if field_obj.name in help_fields and help_fields[field_obj.name]:
                        field_obj.help_text = f"{field_obj.help_text}\n{help_fields[field_obj.name]}"
                    field_obj.help_text = f"<pre>{django.utils.html.escape(field_obj.help_text)}</pre>"
                    setattr(field_obj, patched, True)

    def mbd_class_name(self):
        """Allow to get class name in templates"""
        return self.__class__.__name__

    def mbd_get_extra_options(self):
        """In case there are multiple entries, the last wins"""
        result = {}
        for line in self.extra_options.splitlines():
            lkey, _lsep, lvalue = line.partition(":")
            if lkey:
                result[lkey] = lvalue.lstrip()
        return result

    def mbd_get_extra_option(self, key, default=None):
        return self.mbd_get_extra_options().get(key, default)

    @classmethod
    def mbd_validate_regex(cls, regex, value, field_name):
        if not re.match(regex, value):
            raise django.core.exceptions.ValidationError({field_name: f"Field does not match regex {regex}"})

    def mbd_get_dependencies(self):
        return []

    def mbd_get_reverse_dependencies(self):
        return []

    @classmethod
    def mbd_get_default(cls, field_name):
        return cls._meta.get_field(field_name).get_default()  # pylint: disable=protected-access


class StatusModel(Model):
    """Abstract model class for all models that carry a status"""

    # The main statuses: removed, prepared, active
    STATUS_REMOVED = 0
    STATUS_PREPARED = 1
    STATUS_ACTIVE = 2
    STATUS_CHOICES = [
        (STATUS_REMOVED, "Removed"),
        (STATUS_PREPARED, "Prepared"),
        (STATUS_ACTIVE, "Active")]
    status = django.db.models.IntegerField(choices=STATUS_CHOICES, default=STATUS_REMOVED, editable=False)

    # Statuses of the prepared data, relevant for status "Prepared" only.
    # For "Removed" it's always NONE, for "Active" it's always the stamp of the last check.
    CHECK_NONE = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=datetime.timezone.utc)
    CHECK_CHANGED = datetime.datetime(datetime.MINYEAR, 1, 2, tzinfo=datetime.timezone.utc)
    CHECK_FAILED = datetime.datetime(datetime.MINYEAR, 1, 3, tzinfo=datetime.timezone.utc)
    CHECK_REACTIVATE = datetime.datetime(datetime.MINYEAR, 1, 4, tzinfo=datetime.timezone.utc)
    _CHECK_MAX = CHECK_REACTIVATE
    CHECK_STRINGS = {
        CHECK_NONE: {"char": "-", "string": "Unchecked -- please run check"},
        CHECK_CHANGED: {"char": "*", "string": "Changed -- please prepare again"},
        CHECK_FAILED: {"char": "x", "string": "Failed -- please fix and check again"},
        CHECK_REACTIVATE: {"char": "A", "string": "Failed in active state -- will auto-activate when check succeeds again"}}
    last_checked = django.db.models.DateTimeField(default=CHECK_NONE, editable=False)

    LETHAL_DEPENDENCIES = True

    # Obsoleted by CHECK_REACTIVATE prepared data state (but we need to keep it to not change the db scheme)
    auto_reactivate = django.db.models.BooleanField(default=False, editable=False)

    class Meta(Model.Meta):
        abstract = True

    class Admin(Model.Admin):
        def save_model(self, request, obj, form, change):
            if change:
                obj.mbd_set_changed()
            super().save_model(request, obj, form, change)

        @classmethod
        def _mbd_run_dependencies(cls, obj, func, **kwargs):
            """
            Run action for all dependencies

            But don't fail and run all checks for models with
            LETHAL_DEPENDENCIES set to False. Practical use case is
            the Daemon model only, where we want to run all checks on
            all dependencies, but not fail ourselves.
            """
            for o in obj.mbd_get_dependencies():
                try:
                    func(o, **kwargs)
                except BaseException as e:
                    if obj.LETHAL_DEPENDENCIES:
                        raise
                    util.log_exception(LOG, f"Check on '{o}' failed", e)

        @classmethod
        def mbd_prepare(cls, obj):
            if not obj.mbd_is_prepared():
                # Fresh prepare
                cls._mbd_run_dependencies(obj, cls.mbd_prepare)
                obj.mbd_prepare()
                obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_NONE
                obj.save()
            elif obj.mbd_is_changed():
                # Update data on change
                cls._mbd_run_dependencies(obj, cls.mbd_prepare)
                obj.mbd_sync()
                obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_NONE
                obj.save()

        @classmethod
        def mbd_check(cls, obj, force=False, needs_activation=False):
            if not obj.mbd_is_prepared() or obj.mbd_is_changed():
                raise util.HTTPBadRequest(f"Can't check removed or changed object (run 'prepare' first): {obj}")

            try:
                # Also run for all status dependencies
                cls._mbd_run_dependencies(obj, cls.mbd_check,
                                          force=force,
                                          needs_activation=obj.mbd_is_active() or obj.last_checked == obj.CHECK_REACTIVATE)

                if force or not obj.mbd_is_checked():
                    obj.mbd_check()

                    # Handle special flags
                    reactivated = False
                    if obj.last_checked == obj.CHECK_REACTIVATE:
                        obj.status = StatusModel.STATUS_ACTIVE
                        reactivated = True

                    # Finish up
                    obj.last_checked = django.utils.timezone.now()
                    obj.save()

                    # Run activation hook if reactivated
                    if reactivated:
                        cls._mbd_on_activation(obj)

                if needs_activation and not obj.mbd_is_active():
                    raise util.HTTPBadRequest(f"Not active, but a (tobe-)active item depends on it. Activate this first: {obj}")
            except BaseException:
                # Check failed, auto-deactivate and re-raise exception
                obj.last_checked = max(obj.last_checked, obj.CHECK_FAILED)
                if obj.mbd_is_active():
                    obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_REACTIVATE
                obj.save()
                raise

        @classmethod
        def mbd_pc(cls, obj, force=False, needs_activation=False):
            cls.mbd_prepare(obj)
            cls.mbd_check(obj, force=force, needs_activation=needs_activation)

        @classmethod
        def mbd_activate(cls, obj):
            if not obj.mbd_is_active():
                if obj.mbd_is_prepared() and obj.mbd_is_checked():
                    cls._mbd_run_dependencies(obj, cls.mbd_activate)
                    obj.status = obj.STATUS_ACTIVE
                    obj.save()
                elif obj.mbd_is_prepared() and obj.last_checked in [obj.CHECK_FAILED, obj.CHECK_NONE]:
                    obj.last_checked = obj.CHECK_REACTIVATE
                    obj.save()
                else:
                    raise util.HTTPBadRequest(f"Prepare and check first: {obj}")

            if obj.mbd_is_active():
                cls._mbd_on_activation(obj)

        @classmethod
        def mbd_pca(cls, obj, force=False, needs_activation=False):
            cls.mbd_prepare(obj)
            cls.mbd_check(obj, force=force, needs_activation=needs_activation)
            cls.mbd_activate(obj)

        @classmethod
        def mbd_deactivate(cls, obj):
            obj.status = min(obj.STATUS_PREPARED, obj.status)
            if obj.last_checked == obj.CHECK_REACTIVATE:
                obj.last_checked = obj.CHECK_FAILED
            obj.save()
            cls._mbd_on_deactivation(obj)

        @classmethod
        def mbd_remove(cls, obj):
            if obj.mbd_is_prepared():
                obj.mbd_remove()
                obj.status, obj.last_checked = obj.STATUS_REMOVED, obj.CHECK_NONE
                obj.save()

        @classmethod
        def mbd_action(cls, request, queryset, action, **kwargs):
            """
            Try to run action on each object in queryset

            Emit error message on failure, but don't fail ourself.
            """
            name = action.__name__[4:]
            for o in queryset:
                try:
                    action(o, **kwargs)
                    mbd_msg(request, f"{name} succeeded: {o}")
                except BaseException as e:
                    mbd_msg_exception(request, e, f"{name} failed: {o}: ")

        # Admin actions: pylint: disable=no-self-argument
        #
        # Unlike ``mbd_<action>`` class methods, these act on querysets && will not fail but use the django message system to deliver errors
        #
        # .. attention:: **compat** (django < 3.2): Drop monkey-patching ``short_description`` in favor of ``@django.contrib.admin.action(description="Foo")`` (https://docs.djangoproject.com/en/4.1/releases/3.2/#new-decorators-for-the-admin-site)
        def mbd_action_prepare(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_prepare)
        mbd_action_prepare.short_description = "Prepare"

        def mbd_action_check(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_check, force=True)
        mbd_action_check.short_description = "Check"

        def mbd_action_pc(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_pc)
        mbd_action_pc.short_description = "PC"

        def mbd_action_activate(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_activate)
        mbd_action_activate.short_description = "Activate"

        def mbd_action_pca(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_pca)
        mbd_action_pca.short_description = "PCA"

        def mbd_action_deactivate(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_deactivate)
        mbd_action_deactivate.short_description = "Deactivate"

        def mbd_action_remove(modeladmin, request, queryset):
            if request.POST.get("confirm"):
                modeladmin.mbd_action(request, queryset, modeladmin.mbd_remove)
                return None
            return django.template.response.TemplateResponse(
                request,
                "admin/mini_buildd/mbd_action_remove_confirm.html",
                {
                    "title": ("Are you sure?"),
                    "queryset": queryset,
                    "action": "mbd_action_remove",
                    "desc": ("Unpreparing means all the data associated by preparation will be\n"
                             "removed from the system. Especially for repositories,\n"
                             "this would mean losing all packages!\n"),
                    "action_checkbox_name": django.contrib.admin.helpers.ACTION_CHECKBOX_NAME})
        mbd_action_remove.short_description = "Remove"
        # Admin actions: pylint: enable=no-self-argument

        @classmethod
        def colored_status(cls, obj):
            # pylint: disable=no-member  # pylint false-positive? on builtins.mbd_cssmap
            return django.utils.html.format_html(f'<span class="{builtins.mbd_cssmap("model_status", obj.get_status_display())}" title="{obj.mbd_get_status_display(typ="string")}">{obj.mbd_get_status_display(typ="char")}</span>')

        actions = [mbd_action_prepare, mbd_action_check, mbd_action_pc, mbd_action_activate, mbd_action_pca, mbd_action_deactivate, mbd_action_remove]
        list_display = ["colored_status", "__str__"]
        list_display_links = ["__str__"]

    def mbd_set_changed(self):
        if self.mbd_is_active():
            self.status = self.STATUS_PREPARED
            LOG.warning(f"Deactivated due to changes: {self}")
        self.last_checked = self.CHECK_CHANGED
        LOG.warning(f"Marked as changed: {self}")

    #
    # Action hooks helpers
    #
    def _mbd_remove_and_prepare(self):
        StatusModel.Admin.mbd_remove(self)
        StatusModel.Admin.mbd_prepare(self)

    #
    # Status abstractions and helpers
    #
    def mbd_is_prepared(self):
        return self.status >= self.STATUS_PREPARED

    def mbd_is_active(self):
        return self.status >= self.STATUS_ACTIVE

    def mbd_is_checked(self):
        return self.last_checked > self._CHECK_MAX

    def mbd_is_changed(self):
        return self.last_checked == self.CHECK_CHANGED

    def mbd_is_reactivate(self):
        return self.last_checked == self.CHECK_REACTIVATE

    @classmethod
    def mbd_get_active(cls):
        return cls.objects.filter(status__gte=cls.STATUS_ACTIVE)

    @classmethod
    def mbd_get_active_or_auto_reactivate(cls):
        return cls.objects.filter(django.db.models.Q(status__gte=cls.STATUS_ACTIVE) | django.db.models.Q(last_checked=cls.CHECK_REACTIVATE))

    @classmethod
    def mbd_get_prepared(cls):
        return cls.objects.filter(status__gte=cls.STATUS_PREPARED)

    def mbd_get_check_display(self, typ="string"):
        return {"string": self.last_checked.strftime("Checked on %Y-%m-%d %H:%M"), "char": "C"}[typ] if self.mbd_is_checked() else self.CHECK_STRINGS[self.last_checked][typ]

    def mbd_get_status_display(self, typ="string"):
        return f"{self.get_status_display()} ({self.mbd_get_check_display(typ)})"


class GnuPGPublicKey(StatusModel):
    key_id = django.db.models.CharField(max_length=100, blank=True, default="",
                                        help_text="Give a key id here to retrieve the actual key automatically per configured key server.")
    key = UnixTextField(blank=True, default="",
                        help_text="ASCII-armored GnuPG public key. Leave the key id blank if you fill this manually.")

    key_long_id = django.db.models.CharField(max_length=254, blank=True, default="")
    key_created = django.db.models.CharField(max_length=254, blank=True, default="")
    key_expires = django.db.models.CharField(max_length=254, blank=True, default="")
    key_name = django.db.models.CharField(max_length=254, blank=True, default="")
    key_fingerprint = django.db.models.CharField(max_length=254, blank=True, default="")

    class Meta(StatusModel.Meta):
        abstract = True
        app_label = "mini_buildd"

    class Admin(StatusModel.Admin):
        search_fields = ["key_id", "key_long_id", "key_name", "key_fingerprint"]
        readonly_fields = ["key_long_id", "key_created", "key_expires", "key_name", "key_fingerprint"]

    def __str__(self):
        return f"GnuPGPublicKey '{self.key_long_id if self.key_long_id else self.key_id}' ({self.key_name})"

    def mbd_key_id(self):
        """
        Get valid key id while preferring long key id

        Traditionally, also short ids were used for ``key_id``. This assures a valid key is returned regardless of the
        state (if unprepared, ``key_long_id`` would be empty) while also preferring the long key if it exists.
        """
        return self.key_long_id if self.key_long_id else self.key_id

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.key_id and len(self.key_id) < 8:
            raise django.core.exceptions.ValidationError({"key_id": "The key id, if given, must be at least 8 bytes long"})

    @classmethod
    def mbd_filter_key(cls, key_id):
        regex = rf"{key_id[-8:]}$"
        return cls.objects.filter(django.db.models.Q(key_long_id__iregex=regex) | django.db.models.Q(key_id__iregex=regex))

    def mbd_prepare(self):
        key, info = gnupg.PublicKeyCache().keyinfo(self.key_id, self.key)
        self.key, self.key_long_id, self.key_name, self.key_fingerprint, self.key_created, self.key_expires = key, info["key"], info["user"], info["fingerprints"][0], info["created"], info["expires"]
        self.key_id = self.key_long_id

    def mbd_remove(self):
        self.key_long_id = ""
        self.key_created = ""
        self.key_expires = ""
        self.key_name = ""
        self.key_fingerprint = ""

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_check(self):
        """Check that we actually have the key and long_id. This should always be true after ``prepare``"""
        if not self.mbd_key_id():
            raise util.HTTPBadRequest("GnuPG key with inconsistent state -- try remove,prepare to fix")

    def mbd_verify(self, signed_message):
        with closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-pubkey-"})) as _gnupg:
            _gnupg.add_pub_key(self.key)
            data, _ = _gnupg.gpgme_verify(signed_message)
            return data


class AptKey(GnuPGPublicKey):
    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)

        if self.key_id:
            matching_key = self.mbd_filter_key(self.key_id)
            if matching_key.count() > 0 and self.pk != matching_key.get().pk:
                raise django.core.exceptions.ValidationError({"key_id": f"Another such key id already exists: {matching_key[0]}"})


class Uploader(GnuPGPublicKey):
    user = django.db.models.OneToOneField(django.contrib.auth.models.User,
                                          on_delete=django.db.models.CASCADE)
    may_upload_to = django.db.models.ManyToManyField("Repository", blank=True)

    class Admin(GnuPGPublicKey.Admin):
        search_fields = GnuPGPublicKey.Admin.search_fields + ["user__username"]
        readonly_fields = GnuPGPublicKey.Admin.readonly_fields + ["user"]
        filter_horizontal = ("may_upload_to",)

    def __str__(self):
        return f"Uploader '{self.user}' may upload to '{','.join([r.identity for r in self.may_upload_to.all()])}' with key '{super().__str__()}'"


def cb_create_user_profile(sender, instance, created, **kwargs):
    """Automatically create a user profile with every user that is created"""
    if created:
        Uploader.objects.create(user=instance)


django.db.models.signals.post_save.connect(cb_create_user_profile, sender=django.contrib.auth.models.User)


class Remote(GnuPGPublicKey):
    http = django.db.models.CharField(primary_key=True, max_length=255, default="tcp:host=YOUR_REMOTE_HOST:port=8066",
                                      help_text=f"HTTP Remote Endpoint: {inspect.getdoc(net.ClientEndpoint)}")

    wake_command = django.db.models.CharField(max_length=255, default="", blank=True, help_text="Any command to wake this remote. For example ``/usr/bin/wakeonlan -i DEST_IP MAC``.")

    class Admin(GnuPGPublicKey.Admin):
        search_fields = GnuPGPublicKey.Admin.search_fields + ["http"]
        readonly_fields = GnuPGPublicKey.Admin.readonly_fields + ["key", "key_id"]

    def mbd_url(self):
        return net.ClientEndpoint(self.http, protocol="http").geturl()

    def mbd_api(self, command, timeout=None):
        return client.Client(self.mbd_url()).api(command, timeout=timeout)

    def __str__(self):
        return f"Remote '{self.mbd_url()}' with {super().__str__()}"

    def mbd_get_status(self, timeout=2, wake=False, wake_sleep=5, wake_attempts=2):
        attempt = 0
        while True:
            try:
                return self.mbd_api(api.Status(), timeout=timeout)
            except util.HTTPUnavailable as e:
                if not wake or attempt >= wake_attempts:
                    raise
                if not self.wake_command:
                    raise util.HTTPUnavailable(f"{e.rfc7807.detail} (no wake command configured for remote {self})")

                LOG.info(f"Trying to wake '{self.mbd_url()}' via '{self.wake_command}' (#{attempt}/{wake_attempts})...")
                call.Call(shlex.split(self.wake_command)).check()
                attempt += 1
                time.sleep(wake_sleep)

    def mbd_prepare(self):
        # Be sure to wake if needed
        self.mbd_get_status(wake=True)

        # We prepare the GPG data from downloaded key data, so key_id _must_ be empty (see super(mbd_prepare))
        self.key_id = ""
        self.key = self.mbd_api(api.PubKey())

        if self.key:
            LOG.warning("Downloaded remote key integrated: Please check key manually before activation!")
        else:
            raise util.HTTPBadRequest(f"Empty remote key from '{self.mbd_url()}' -- maybe the remote is not prepared yet?")
        super().mbd_prepare()

    def mbd_check(self):
        """Check whether the remote mini-buildd is up, running and serving for us"""
        super().mbd_check()

        # Get status
        status = self.mbd_get_status()

        # GnuPG handshake
        signed_message = self.mbd_api(api.Handshake(signed_message=util.daemon().handshake_message()))
        self.mbd_verify(signed_message)

        # Check if our instance is activated on remote
        if util.http_endpoint().geturl() not in status["remotes"]:
            raise util.HTTPBadRequest(f"{self}: Our instance is not activated on remote")

        # Check if our instance is started
        if status["load"] < 0.0:
            raise util.HTTPBadRequest(f"{self}: Remote is stopped")


class Builders(list):
    """
    All active builders (ourselves as Meta-Remote plus all actual Remotes) with status

    Is used to choose builder to upload to, and as data for the Builder's View.
    """

    Builder = collections.namedtuple("Builder", "remote status exception ourselves")

    def __init__(self, wake=False, check=False):
        super().__init__()

        # Always add ourselves
        ourselves = Remote(http=util.http_endpoint().geturl(), status=Remote.STATUS_ACTIVE, last_checked=django.utils.timezone.now())
        self.append(Builders.Builder(ourselves, ourselves.mbd_get_status(), None, True))

        for r in Remote.mbd_get_active_or_auto_reactivate():
            status, exception = None, None
            try:
                status = r.mbd_get_status(wake=wake)
                if check:
                    Remote.Admin.mbd_check(r, force=True)
            except BaseException as e:
                exception = util.e2http(e)
            finally:
                self.append(Builders.Builder(r, status, exception, False))

    def get(self, codename, arch):
        """Get a dict {<load>: <status>} of active builders for <codename>/<arch>"""
        return {builder.status["load"]: builder.status for builder in self if builder.remote.mbd_is_active() and builder.status and builder.status["load"] >= 0.0 and f"{codename}:{arch}" in builder.status["chroots"]}


class Archive(Model):
    URLOPEN_TIMEOUT = 15  # seconds

    url = django.db.models.URLField(primary_key=True, max_length=512,
                                    default="http://ftp.debian.org/debian/",
                                    help_text=("The URL of an apt archive (there must be a 'dists/' infrastructure below).\n"
                                               "\n"
                                               "Use the 'directory' notation with exactly one trailing slash (like 'http://example.org/path/')."))
    ping = django.db.models.FloatField(default=-1.0, editable=False)

    class Meta(Model.Meta):
        ordering = ["url"]

    class Admin(Model.Admin):
        search_fields = ["url"]

    def __str__(self):
        return f"Archive '{self.url}' (ping {round(self.ping, 2)} ms)"

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.url[-1] != "/" or self.url[-2] == "/":  # pylint: disable=unsubscriptable-object
            raise django.core.exceptions.ValidationError({"url": f"Invalid archive url '{self.url}': Must have exactly one trailing slash (like 'http://example.org/path/')"})

    class ReleaseFile():
        @classmethod
        def _download(cls, file_, url):
            with urllib.request.urlopen(url, timeout=Archive.URLOPEN_TIMEOUT) as response:
                file_.write(response.read())
                file_.flush()
                file_.seek(0)

        def __init__(self, url_base):
            self.url_base = url_base

            # pylint: disable=consider-using-with  # See close(), use this class with contextlib.closing()
            self.release_file = tempfile.NamedTemporaryFile()
            self.signature_file = tempfile.NamedTemporaryFile()
            self.release_url, self.signature_url = None, None

            try:
                self.release_url = self.url_base + "InRelease"
                self._download(self.release_file, self.release_url)
            except urllib.error.HTTPError:
                try:
                    self.release_url = self.url_base + "Release"
                    self._download(self.release_file, self.release_url)
                    self.signature_url = self.release_url + ".gpg"
                    self._download(self.signature_file, self.signature_url)
                except urllib.error.HTTPError as e:
                    raise util.HTTPNotFound(f"{self.url_base}: Neither 'InRelease' nor 'Release[.gpg]' found") from e

            self.release = debian.deb822.Release(self.release_file)

        def close(self):
            for t in [self.release_file, self.signature_file]:
                if t is not None:
                    t.close()

        def match(self, source):
            LOG.debug(f"match(): {self.release_url} matches {source}?")

            # Check release file fields
            for key, value in list(source.mbd_release_file_values().items()):
                if value != self.release.get(key):
                    raise util.HTTPNotFound(f"'{self.release_url}' does not match (has '{key}: {self.release.get(key)}', we need '{value}')")

            # Pre-Check 'Valid-Until'
            #
            # Some Release files contain an expire date via the
            # 'Valid-Until' tag. If such an expired archive is used,
            # builds will fail. Furthermore, it could be only the
            # selected archive not updating, while the source may be
            # perfectly fine from another archive.
            #
            # This pre-check avoids such a situation, or at least it
            # can be fixed by re-checking the source.
            try:
                valid_until = util.Datetime.parse(self.release["Valid-Until"])
                if valid_until < util.Datetime.now():
                    if source.mbd_get_extra_option("X-Check-Valid-Until", "yes").lower() in ("no", "false", "0"):
                        LOG.warning(f"{self.release_url} expired, but source marked to ignore valid-until (Valid-Until='{valid_until}').")
                    else:
                        LOG.warning(f"{self.release_url} expired, maybe the archive has problems? (Valid-Until='{valid_until}').")
                        raise util.HTTPUnavailable(f"{self.release_url} expired")
            except KeyError:
                pass  # We can assume Release file has no "Valid-Until", and be quiet
            except BaseException as e:
                util.log_exception(LOG, f"Ignoring error checking 'Valid-Until' on {self.release_url}", e)

        def verify(self, _gnupg):
            return _gnupg.gpgme_verify_release(self.release_file, self.signature_file if self.signature_url is not None else None, accept_expired=True, name=self.url_base)

    def mbd_ping(self):
        """Ping and update the ping value"""
        try:
            t0 = django.utils.timezone.now()
            # Append dists to URL for ping check: Archive may be
            # just fine, but not allow to access to base URL
            # (like ourselves ;). Any archive _must_ have dists/ anyway.
            try:
                with urllib.request.urlopen(f"{self.url}/dists", timeout=self.URLOPEN_TIMEOUT) as _:
                    pass
            except urllib.error.HTTPError as e:
                # Allow HTTP 4xx client errors through; these might be valid use cases like:
                # 404 Usage Information: apt-cacher-ng
                if not 400 <= e.code <= 499:
                    raise

            delta = django.utils.timezone.now() - t0
            self.ping = delta.total_seconds() * (10 ** 3)
            self.save()
        except Exception as e:
            self.ping = -1.0
            self.save()
            raise util.HTTPUnavailable(f"'{self}' does not ping: {e}") from e

    def mbd_get_reverse_dependencies(self):
        """Return all sources (and their deps) that use us"""
        result = list(self.source_set.all())
        for s in self.source_set.all():
            result += s.mbd_get_reverse_dependencies()
        return result


class Architecture(Model):
    name = django.db.models.CharField(primary_key=True, max_length=50)

    def __str__(self):
        return f"Architecture '{self.name}'"


class Component(Model):
    """A Debian component (like 'main', 'contrib', 'non-free')"""

    name = django.db.models.CharField(primary_key=True, max_length=50)

    def __str__(self):
        return f"Component '{self.name}'"


@functools.total_ordering
class Source(StatusModel):
    MBD_HELP_EXTRA_OPTIONS = """
=================================================
Origin, Codename, Suite, Archive, Version, Label
=================================================
If needed, you may use these fields (same as in a Release file) to
further specify this source. These are also later used to pin the
source via apt.

For some sources, ``Codename`` (as we use it above) does
not match resp. value as given in the Release file. When
``Codename`` is overridden in this manner, be sure to also
add one further flag to identify the source -- else apt pinning later
would likely not be unambiguous. Real world examples that need this
extra handling are ``Debian Security`` (``buster`` or
older), ``Ubuntu Security and Backports``:
---
Codename: bionic
Suite: bionic-backports
---
=================================================
X-Check-Valid-Until: no
=================================================
Some sources have a ``Valid-Until`` field that is no longer
updated. If you really still want to use it anyway, use:
---
X-Check-Valid-Until: no
---
This will, 1st, ignore mini-buildd's own 'Valid-Until check' and 2nd,
create apt lines for this source with the ``[check-valid-until=no]``
option. I.e., at least from ``stretch`` onwards, the check is disabled
per source. For ``jessie`` or worse (where this apt option does not
work), a global workaround via schroot is still in place.

=================================================
X-Remove-From-Component: <prefix>
=================================================
Some (actually, we only know of ``Debian Security``) sources
have weird ``Components`` that need to be fixed to work with
mini-buildd. For example, ``Debian Security`` (buster or older)
needs:

---
Codename: stretch
Label: Debian-Security
X-Remove-From-Component: updates/
---

Fwiw, in ``Debian Security`` for ``bullseye`` or
younger, Codename is now unambiguous (like
``bullseye-security``), but the component oddity remains; so
these cases still need:

---
X-Remove-From-Component: updates/
---
"""

    # Identity
    origin = django.db.models.CharField(max_length=60, default="Debian",
                                        help_text="The exact string of the 'Origin' field of the resp. Release file.")
    codename = django.db.models.CharField(max_length=60, default="sid",
                                          help_text=("The name of the directory below ``dists/`` in archives this source\n"
                                                     "refers to. This is also the 3rd part of an apt line.\n"
                                                     "\n"
                                                     "With no extra options given, this source will be identified comparing\n"
                                                     "``Origin`` and ``Codename`` with the values of the ``Release`` file\n"
                                                     "found.\n"
                                                     "\n"
                                                     "Some sources need some more care via ``Extra options``, see below."))

    # Apt Secure
    apt_keys = django.db.models.ManyToManyField(AptKey, blank=True,
                                                help_text=("At least one key this source's 'Release' file is signed with.\n"
                                                           "Hint: If you leave this empty and run 'check' on it, you will get a message listing all signer keys."))

    # Extra
    description = django.db.models.CharField(max_length=100, editable=False, blank=True, default="")
    codeversion = django.db.models.CharField(
        max_length=50, editable=True, blank=True, default="",
        help_text=("Codeversion in use (relevant for a base source only -- will be used for version restrictions).\n"
                   "When empty, this will updated with the guessed codeversion from the release file on every check.\n"
                   "When non-empty, this value just stays.\n"
                   "You will need to update (manually), for example, when a rolling source gets released, and becomes stable.\n"
                   "Unless the guessed codeversion is wrong, you will want the same value here.\n"))
    codeversion_override = django.db.models.CharField(
        max_length=50, editable=False, blank=True, default="",
        help_text=("Codeversion guessed from release file.\n"
                   "This will be updated every time the source is checked."))
    archives = django.db.models.ManyToManyField(Archive, blank=True)
    components = django.db.models.ManyToManyField(Component, blank=True)
    architectures = django.db.models.ManyToManyField(Architecture, blank=True)

    class Meta(StatusModel.Meta):
        unique_together = ("origin", "codename")
        ordering = ["origin", "-codeversion", "codename"]

    class Admin(StatusModel.Admin):
        list_display = StatusModel.Admin.list_display + ["origin", "codeversion", "codename"]
        search_fields = ["origin", "codeversion", "codename"]
        ordering = ["origin", "-codeversion", "codename"]

        readonly_fields = ["codeversion_override", "archives", "components", "architectures", "description"]
        filter_horizontal = ("apt_keys",)

        def get_readonly_fields(self, _request, obj=None):
            """Forbid to change identity on existing source (usually a bad idea; repos/chroots that refer to us may break)"""
            fields = copy.copy(self.readonly_fields)
            if obj:
                fields.append("origin")
                fields.append("codename")
            return fields

        @classmethod
        def mbd_filter_active_base_sources(cls):
            """Filter active base sources; needed in chroot and distribution wizards"""
            return Source.objects.filter(status__gte=Source.STATUS_ACTIVE,
                                         origin__in=dist.SETUP["origin"].keys(),
                                         codename__regex=r"^[a-z]+$").exclude(codename="experimental")

    def __str__(self):
        return f"Source '{self.origin} {self.codename}' ({self.codeversion})"

    def mbd_release_file_values(self):
        """Compute a dict of values a matching release file must have"""
        values = {k: v for k, v in self.mbd_get_extra_options().items() if not k.startswith("X-")}  # Keep "X-<header>" for special purposes. All other keys are like in a Release file.

        # Set Origin and Codename (may be overwritten) from fields
        values["Origin"] = self.origin
        if not values.get("Codename"):
            values["Codename"] = self.codename

        return values

    def mbd_get_archive(self):
        """Get fastest archive"""
        oa_list = self.archives.all().filter(ping__gte=0.0).order_by("ping")
        if oa_list:
            return oa_list[0]
        raise util.HTTPBadRequest(f"{self}: No archive found (network down? ``Archive`` missing? ``Source`` with incorrect keys?)")

    def mbd_get_apt_line(self, limit_components=None):
        return files.AptLine(self.mbd_get_archive().url,
                             "",
                             self.codename,
                             dist.sorted_components([c.name for c in self.components.all() if limit_components is None or c in limit_components]),
                             options=f"check-valid-until={self.mbd_get_extra_option('X-Check-Valid-Until', 'yes')}",
                             comment=f"{self}: {self.mbd_get_apt_pin()}")

    def mbd_get_apt_pin(self):
        """Apt 'pin line' (for use in a apt 'preference' file)"""
        # See man apt_preferences for the field/pin mapping
        supported_fields = {"Origin": "o", "Codename": "n", "Suite": "a", "Archive": "a", "Version": "v", "Label": "l"}
        pins = []
        for key, value in list(self.mbd_release_file_values().items()):
            k = supported_fields.get(key)
            if k:
                pins.append(f"{k}={value}")
        return "release " + ", ".join(pins)

    def mbd_prepare(self):
        self.clean()

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_remove(self):
        self.archives.set([])
        self.components.set([])
        self.architectures.set([])
        self.description = ""

    def mbd_check(self):
        """Rescan all archives, and check that there is at least one working"""
        self.archives.set([])

        # Gather failed information to debug if no archive was found
        failed = {}

        with contextlib.closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-sourcecheck-"})) as gpg:
            for k in self.apt_keys.all():
                gpg.add_pub_key(k.key)

            for archive in Archive.objects.all():
                try:
                    archive.mbd_ping()

                    with closing(Archive.ReleaseFile(f"{archive.url}/dists/{self.codename}/")) as release_file:
                        release_file.match(self)
                        release_file.verify(gpg)

                        # Implicitly save ping value for this archive
                        self.archives.add(archive)
                        self.description = release_file.release.get("Description", "")

                        # Set codeversion
                        self.codeversion_override = dist.guess_codeversion(release_file.release.get("Origin"), release_file.release.get("Codename"), release_file.release.get("Version"))
                        if not self.codeversion:
                            self.codeversion = self.codeversion_override

                        # Set architectures and components (may be auto-added)
                        if release_file.release.get("Architectures"):
                            for a in release_file.release["Architectures"].split(" "):
                                new_arch, _created = Architecture.objects.get_or_create(name=a)
                                self.architectures.add(new_arch)
                        if release_file.release.get("Components"):
                            for c in release_file.release["Components"].split(" "):
                                new_component, _created = Component.objects.get_or_create(name=c.replace(self.mbd_get_extra_option("X-Remove-From-Component", ""), ""))
                                self.components.add(new_component)
                except util.HTTPError as e:
                    util.log_exception(LOG, f"{self}: Skipping {archive}", e, level=logging.DEBUG)
                    failed[archive] = e

        # Check that at least one archive can be found
        try:
            fastest_archive = self.mbd_get_archive()
            LOG.debug(f"{self}: Fastest archive: {fastest_archive}")
        except util.HTTPError as e:
            errs = '\n'.join([f'{archive}: {error}' for archive, error in failed.items()])
            raise util.HTTPNotFound(f"{e.detail}:\n{errs}") from e

    def mbd_get_dependencies(self):
        return self.apt_keys.all()

    def mbd_get_reverse_dependencies(self):
        """Return all chroots and repositories that use us"""
        result = list(self.chroot_set.all())
        for d in self.distribution_set.all():
            result += d.mbd_get_reverse_dependencies()
        return result

    __hash__ = StatusModel.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return dist.Codename(self.codename, origin=self.origin) == dist.Codename(other.codename, origin=other.origin)
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return dist.Codename(self.codename, origin=self.origin) < dist.Codename(other.codename, origin=other.origin)
        return NotImplemented

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.mbd_get_extra_option("Origin"):
            raise django.core.exceptions.ValidationError({"extra_options": "You may not override 'Origin', just use the origin field"})


class PrioritySource(Model):
    source = django.db.models.ForeignKey(Source,
                                         on_delete=django.db.models.CASCADE)
    priority = django.db.models.IntegerField(default=1,
                                             help_text="A apt pin priority value (see 'man apt_preferences')."
                                             "Examples: 1=not automatic, 1001=downgrade'")

    class Meta(Model.Meta):
        unique_together = ('source', 'priority')

    def __str__(self):
        return f"{self.source} with priority {self.priority}"

    def mbd_get_apt_preferences(self):
        return f"Package: *\nPin: {self.source.mbd_get_apt_pin()}\nPin-Priority: {self.priority}\n"


class Suite(Model):
    name = django.db.models.CharField(
        max_length=100,
        help_text="A suite to support, usually s.th. like 'experimental', 'unstable', 'testing' or 'stable'.")

    def __str__(self):
        return f"Suite '{self.name}'"

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        self.mbd_validate_regex(r"^[a-z]+$", self.name, "name")


class SuiteOption(Model):
    layout = django.db.models.ForeignKey("Layout",
                                         on_delete=django.db.models.CASCADE)
    suite = django.db.models.ForeignKey("Suite",
                                        on_delete=django.db.models.CASCADE)

    uploadable = django.db.models.BooleanField(
        default=True,
        help_text="Whether this suite should accept user uploads.")

    experimental = django.db.models.BooleanField(
        default=False,
        help_text=("Experimental suites must be uploadable and must not\n"
                   "migrate. Also, the packager treats failing extra QA checks (like\n"
                   "lintian) as non-lethal, and will install anyway."))

    migrates_to = django.db.models.ForeignKey(
        "self", blank=True, null=True,
        on_delete=django.db.models.CASCADE,
        help_text="Give another suite where packages may migrate to (you may need to save this 'blank' first before you see choices here).")

    build_keyring_package = django.db.models.BooleanField(
        default=False,
        help_text="Build keyring package for this suite (i.e., when the resp. Repository action is called).")

    auto_migrate_after = django.db.models.IntegerField(
        default=0,
        help_text="For future use. Automatically migrate packages after x days.")

    not_automatic = django.db.models.BooleanField(
        default=True,
        help_text="Include 'NotAutomatic' in the Release file.")

    but_automatic_upgrades = django.db.models.BooleanField(
        default=True,
        help_text="Include 'ButAutomaticUpgrades' in the Release file.")

    class Meta(Model.Meta):
        unique_together = ("suite", "layout")

    def __str__(self):
        return f"{self.suite} for {self.layout}"

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.build_keyring_package and not self.uploadable:
            raise django.core.exceptions.ValidationError("You can only build keyring packages on uploadable suites")
        if self.experimental and self.migrates_to:
            raise django.core.exceptions.ValidationError("Experimental suites may not migrate")
        if self.experimental and not self.uploadable:
            raise django.core.exceptions.ValidationError("Experimental suites must be uploadable")
        if self.migrates_to and self.migrates_to.layout != self.layout:
            raise django.core.exceptions.ValidationError("Migrating suites must be in the same layout (you need to save once to make new suites visible)")
        if self.migrates_to and self.migrates_to.uploadable:
            raise django.core.exceptions.ValidationError("You may not migrate to an uploadable suite")

    @property
    def rollback(self):
        """Rollback field temporarily implemented as extra_option"""
        return int(self.mbd_get_extra_option("Rollback", "0"))

    def mbd_get_sort_no(self):
        """Compute number that may be used to sort suites from 'stable' (0) towards 'experimental'"""
        no = 0
        if self.uploadable:
            no += 5
        if self.migrates_to:
            no += 5
        if self.experimental:
            no += 20
        return no


class SuiteOptionInline(django.contrib.admin.TabularInline):
    model = SuiteOption
    extra = 1
    exclude = ("pickled_data",)


class Layout(Model):
    MBD_HELP_EXTRA_OPTIONS = """
Meta-Distributions: META=CODENAME-SUITE[ META=CODENAME-SUITE[...: Support METAs alone as distribution identifier

Meta distribution identifiers should be unique across all
repositories; usually, a layout with meta distributions should
only be used by at most one repository.

Example:
---
Meta-Distributions: unstable=sid-unstable experimental=sid-experimental
---

(See standard layout 'Debian Developer'): allows upload/testing of
packages (to unstable,experimental,..) aimed for Debian.
"""

    name = django.db.models.CharField(primary_key=True, max_length=100)

    suites = django.db.models.ManyToManyField(Suite, through=SuiteOption)

    # Version magic
    default_version = django.db.models.CharField(
        max_length=100, default="~%IDENTITY%%CODEVERSION%+1",
        help_text=("Version string to append to the original version for automated ports; you may use these placeholders:\n"
                   "\n"
                   "%IDENTITY%: Repository identity (see 'Repository').\n"
                   "%CODEVERSION%: Numerical base distribution version (see 'Source').\n"))

    mandatory_version_regex = django.db.models.CharField(
        max_length=100, default=r"~%IDENTITY%%CODEVERSION%\+[1-9]",
        help_text="Mandatory version regex; you may use the same placeholders as for 'default version'.")

    # Version magic (experimental)
    experimental_default_version = django.db.models.CharField(
        max_length=30, default="~%IDENTITY%%CODEVERSION%+0",
        help_text="Like 'default version', but for suites flagged 'experimental'.")

    experimental_mandatory_version_regex = django.db.models.CharField(
        max_length=100, default=r"~%IDENTITY%%CODEVERSION%\+0",
        help_text="Like 'mandatory version', but for suites flagged 'experimental'.")

    class Admin(Model.Admin):
        inlines = (SuiteOptionInline,)

    def __str__(self):
        return f"Layout '{self.name}'"

    def mbd_get_reverse_dependencies(self):
        """When the layout changes, all repos that use that layout also change"""
        return list(self.repository_set.all())


class ArchitectureOption(Model):
    architecture = django.db.models.ForeignKey("Architecture",
                                               on_delete=django.db.models.CASCADE)
    distribution = django.db.models.ForeignKey("Distribution",
                                               on_delete=django.db.models.CASCADE)

    optional = django.db.models.BooleanField(
        default=False,
        help_text="Don't care if the architecture can't be build.")

    build_architecture_all = django.db.models.BooleanField(
        default=False,
        help_text="Use to build arch-all packages.")

    class Meta(Model.Meta):
        unique_together = ("architecture", "distribution")

    def __str__(self):
        return f"{self.architecture} for {self.distribution}"

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.build_architecture_all and self.optional:
            raise django.core.exceptions.ValidationError("Optional architectures must not be architecture all")


class ArchitectureOptionInline(django.contrib.admin.TabularInline):
    model = ArchitectureOption
    exclude = ("pickled_data", "extra_options",)
    extra = 1


@functools.total_ordering
class Distribution(Model):
    MBD_HELP_EXTRA_OPTIONS = f"""
===============================================================================================
{sbuild.CONFIG_BLOCKS.usage()}

===============================================================================================
{sbuild.SETUP_BLOCKS.usage()}

===============================================================================================
Autopkgtest-Mode: <MODE>: {dist.SbuildCheck.desc()}

{dist.SbuildCheck.usage()}

===============================================================================================
Internal-APT-Priority: <PRIORITY>: Set APT priority for internal apt sources in builds.

The default is 1, which means you will only build against newer
packages in our own repositories in case it's really needed by the
package's build dependencies. This is the recommended behaviour,
producing sparse dependencies.

However, some packages with incorrect build dependencies might
break anyway, while they would work fine when just build against
the newest version available.

You may still solve this on a per-package basis, using the
resp. upload option via changelog. However, in case you don't care
about sparse dependencies in this distribution in general, you can
pimp the internal priority up here.

Example: Always build against newer internal packages:
---
Internal-APT-Priority: 500
---
===============================================================================================
Deb-Build-Options: ...: Set extra build options.

Values to add to environment variable ``DEB_BUILD_OPTIONS`` used when building packages. See

* ``https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options`` and
* ``https://lists.debian.org/debian-devel/2015/12/msg00262.html'>Debian dbgsym announcement``

for valid options. Value may be overridden by the resp. user upload option.

Option ``noddebs`` is useful for Ubuntu distributions that for
some reason create their automated debug packages with file appendix
'ddeb', as (see ``https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=730572``) reprepo fails on them.

Example: Never build automatic debug packages (Ubuntu bionic, cosmic):
---
Deb-Build-Options: noddebs
---
===============================================================================================
Deb-Build-Profiles: ...: Set extra build profiles.

Contents of DEB_BUILD_PROFILES environment for building (see ``https://wiki.debian.org/BuildProfileSpec``).
Value may be overridden by the resp. user upload option.
"""

    base_source = django.db.models.ForeignKey(Source,
                                              on_delete=django.db.models.CASCADE)
    extra_sources = django.db.models.ManyToManyField(PrioritySource, blank=True)
    components = django.db.models.ManyToManyField(Component)

    architectures = django.db.models.ManyToManyField(Architecture, through=ArchitectureOption)

    RESOLVER_APT = 0
    RESOLVER_APTITUDE = 1
    RESOLVER_INTERNAL = 2
    RESOLVER_CHOICES = (
        (RESOLVER_APT, "apt"),
        (RESOLVER_APTITUDE, "aptitude"),
        (RESOLVER_INTERNAL, "internal"))
    build_dep_resolver = django.db.models.IntegerField(choices=RESOLVER_CHOICES, default=RESOLVER_APTITUDE)

    apt_allow_unauthenticated = django.db.models.BooleanField(default=False,
                                                              help_text="DEPRECATED/NOT USED: In 2.0.x: Use resp. sbuild setup block")

    lintian_mode = django.db.models.IntegerField(choices=dist.SbuildCheck.CHOICES, default=dist.SbuildCheck.Mode.ERRFAIL.value,
                                                 help_text=f"""\
{dist.SbuildCheck.desc()}

{dist.SbuildCheck.usage()}
""")

    lintian_extra_options = django.db.models.CharField(max_length=200, default="--info")

    # piuparts not used atm; placeholder for later
    piuparts_mode = django.db.models.IntegerField(choices=dist.SbuildCheck.CHOICES, default=dist.SbuildCheck.Mode.DISABLED.value,
                                                  help_text=(f"EXPERIMENTAL: {dist.SbuildCheck.desc()}\n"
                                                             "\n"
                                                             f"{dist.SbuildCheck.usage()}\n"
                                                             "\n"
                                                             "EXPERIMENTAL: All modes but 'Run check and ignore errors' are currently ignored, and even that might not be called correctly.\n"))
    piuparts_extra_options = django.db.models.CharField(max_length=200, default="--info")
    piuparts_root_arg = django.db.models.CharField(max_length=200, default="sudo")

    chroot_setup_script = UnixTextField(blank=True,
                                        default="",
                                        help_text="""\
Custom script that will be run via sbuild's ``--chroot-setup-command``. Be sure to provide a proper shebang like ``#!/bin/sh -e``.

Note that some predefined script blocks to choose from are available (see 'Extra Options'), which might suite your needs already.

Example:
---
#!/bin/sh -e

echo "I: Foo bar..."
---
""")
    sbuildrc_snippet = UnixTextField(blank=True,
                                     default="",
                                     help_text=("Custom perl snippet to be added to ``.sbuildrc`` for each build.\n"
                                                "\n"
                                                "Note that some predefined config blocks to choose from are available (see 'Extra Options'), which might suite your needs already.\n"
                                                "\n"
                                                "You may use these placeholders:\n"
                                                "\n"
                                                "* ``%LIBDIR%``: Per-chroot persistent dir; may be used for data that should persist between builds (caches, ...).\n"
                                                "\n"
                                                "Example:\n"
                                                "---\n"
                                                "$build_environment = { 'MY_FOO_BAR' => '%LIBDIR%/my-foo-bar' };\n"
                                                "---\n"))

    class Meta(StatusModel.Meta):
        ordering = ["base_source"]

    class Admin(Model.Admin):
        inlines = (ArchitectureOptionInline,)
        filter_horizontal = ("extra_sources", "components",)

    def mbd_is_active(self):
        for s in [self.base_source] + [s.source for s in self.extra_sources.all()]:
            if not s.mbd_is_active():
                return False
        return True

    def __str__(self):
        """
         Note: Appends 'id' only to make multiple Distributions for same base source distinguishable for the user

        .. attention:: Distributions are not unique for base source (historic fluke)
        """
        return f"Distribution for {self.base_source} [pk={self.pk}]"

    def mbd_setup(self):
        return f"{self.base_source.codename}:{'+'.join((a.name for a in self.architectures.all()))}"

    def mbd_get_components(self):
        return dist.sorted_components([c.name for c in self.components.all()])

    def mbd_get_archall_architectures(self):
        return [a.architecture.name for a in self.architectureoption_set.all() if a.build_architecture_all]

    def mbd_get_mandatory_architectures(self):
        return [a.architecture.name for a in self.architectureoption_set.all() if not a.optional]

    def mbd_get_reverse_dependencies(self):
        """When the distribution changes, all repos that use that distribution also change"""
        return list(self.repository_set.all())

    __hash__ = Model.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.base_source == other.base_source
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return self.base_source < other.base_source
        return NotImplemented

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        if self.chroot_setup_script:
            try:
                files.File.shebang_from_data(self.chroot_setup_script)
            except BaseException as e:
                raise django.core.exceptions.ValidationError({"chroot_setup_script": "Missing shebang"}) from e

        try:
            sbuild.CONFIG_BLOCKS.validate(self.mbd_get_extra_option(sbuild.CONFIG_BLOCKS.name, ""))
            sbuild.SETUP_BLOCKS.validate(self.mbd_get_extra_option(sbuild.SETUP_BLOCKS.name, ""))
        except util.HTTPError as e:
            raise django.core.exceptions.ValidationError({"extra_options": f"{e}"})


class EmailAddress(Model):
    address = django.db.models.EmailField(primary_key=True, max_length=255)
    name = django.db.models.CharField(blank=True, max_length=255)

    class Meta(Model.Meta):
        verbose_name_plural = "Email addresses"

    def __str__(self):
        return f"EmailAddress '{self.name} <{self.address}>'"


class Repository(StatusModel):
    identity = django.db.models.CharField(primary_key=True, max_length=50, default="test",
                                          help_text=("The id of the reprepro repository, placed in\n"
                                                     "'repositories/<ID>'. It can also be used in 'version enforcement\n"
                                                     "string' (true for the default layout) -- in this context, it\n"
                                                     "plays the same role as the well-known 'bpo' version string from\n"
                                                     "Debian backports."))

    layout = django.db.models.ForeignKey(Layout,
                                         on_delete=django.db.models.CASCADE)
    distributions = django.db.models.ManyToManyField(Distribution)

    allow_unauthenticated_uploads = django.db.models.BooleanField(default=False,
                                                                  help_text="Allow unauthenticated user uploads.")

    extra_uploader_keyrings = UnixTextField(blank=True,
                                            help_text=("Extra keyrings, line by line, to be allowed as uploaders (in addition to configured django users).\n"
                                                       "\n"
                                                       "Example:\n"
                                                       "---\n"
                                                       "# Allow Debian maintainers (must install the 'debian-keyring' package)\n"
                                                       "/usr/share/keyrings/debian-keyring.gpg\n"
                                                       "# Allow from some local keyring file\n"
                                                       "/etc/my-schlingels.gpg\n"
                                                       "---"))

    notify = django.db.models.ManyToManyField(EmailAddress,
                                              blank=True,
                                              help_text="Addresses that get all notification emails unconditionally.")
    notify_changed_by = django.db.models.BooleanField(default=False,
                                                      help_text="Notify the address in the 'Changed-By' field of the uploaded changes file.")
    notify_maintainer = django.db.models.BooleanField(default=False,
                                                      help_text="Notify the address in the 'Maintainer' field of the uploaded changes file.")

    reprepro_morguedir = django.db.models.BooleanField(default=False,
                                                       help_text="Move files deleted from repo pool to 'morguedir' (see reprepro).")

    external_home_url = django.db.models.URLField(blank=True)

    LETHAL_DEPENDENCIES = False

    class Meta(StatusModel.Meta):
        verbose_name_plural = "Repositories"

    class Admin(StatusModel.Admin):
        readonly_fields = []
        filter_horizontal = ("distributions", "notify",)

        def get_readonly_fields(self, _request, obj=None):
            """Forbid change identity on existing repository"""
            fields = copy.copy(self.readonly_fields)
            if obj:
                fields.append("identity")
            return fields

    def __str__(self):
        return f"Repository '{self.identity}' with distributions for {','.join([d.base_source.codename + ('' if d.mbd_is_active() else '*') for d in self.mbd_sorted_distributions()])}"

    def mbd_setup(self, with_dists=True):
        return f"{self.identity}:{self.layout.name}" + (f":{'+'.join((d.base_source.codename for d in self.distributions.all()))}" if with_dists else "")

    def mbd_json(self):
        return {
            "codenames": [d.base_source.codename for d in self.distributions.all()],
            "layout": {
                s.suite.name: {
                    "uploadable": s.uploadable,
                    "experimental": s.experimental,
                    "migrates_to": s.migrates_to.suite.name if s.migrates_to else None,
                } for s in self.layout.suiteoption_set.all()},
        }

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.mbd_path = config.ROUTES["repositories"].path.join(self.identity)
        self.mbd_reprepro = reprepro.Reprepro(basedir=self.mbd_path)

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        self.mbd_validate_regex(r"^[a-z0-9]+$", self.identity, "identity")

    def mbd_get_diststr(self, distribution, suite, rollback=None):
        diststr = f"{distribution.base_source.codename}-{self.identity}-{suite.suite.name}"

        if rollback is not None:
            if rollback not in list(range(suite.rollback)):
                raise util.HTTPBadRequest(f"{diststr}: Rollback number out of range: {rollback} ({list(range(suite.rollback))})")
            diststr += f"-rollback{rollback}"

        return diststr

    def mbd_get_diststrs(self, frollbacks=None, **suiteoption_filter):
        """List of all distribution strings (except rollbacks) in this repo, optionally matching a suite options filter (unstable, experimental,...)"""
        diststrs = []
        for d in self.distributions.all():
            for s in self.layout.suiteoption_set.filter(**suiteoption_filter):
                diststrs.append(self.mbd_get_diststr(d, s))
                if frollbacks is not None:
                    for r in frollbacks(s):
                        diststrs.append(self.mbd_get_diststr(d, s, rollback=r))
        return diststrs

    def mbd_get_meta_distributions(self, distribution, suite_option):
        try:
            result = []
            for p in self.layout.mbd_get_extra_option("Meta-Distributions", "").split():
                meta, d = p.split("=")
                diststr = f"{d.split('-')[0]}-{self.identity}-{d.split('-')[1]}"
                if diststr == self.mbd_get_diststr(distribution, suite_option):
                    result.append(meta)
            return result
        except BaseException as e:
            raise util.HTTPBadRequest(f"Please fix syntax error in extra option 'Meta-Distributions' in layout '{self.layout}'") from e

    def mbd_get_apt_pin(self, distribution, suite):
        return f"release n={self.mbd_get_diststr(distribution, suite)}, o={util.daemon().model.mbd_get_archive_origin()}"

    def mbd_get_apt_preferences(self, distribution, suite, prio=1):
        return f"Package: *\nPin: {self.mbd_get_apt_pin(distribution, suite)}\nPin-Priority: {prio}\n"

    @classmethod
    def mbd_get_apt_keys(cls, distribution):
        result = util.daemon().gnupg.pub_key
        for e in distribution.extra_sources.all():
            for k in e.source.apt_keys.all():
                result += k.key
        return result

    def mbd_get_internal_suite_dependencies(self, suite_option):
        result = []

        # Add ourselves
        result.append(suite_option)

        if suite_option.experimental:
            # Add all non-experimental suites
            for s in self.layout.suiteoption_set.all().filter(experimental=False):
                result.append(s)
        else:
            # Add all suites that we migrate to
            s = suite_option.migrates_to
            while s:
                result.append(s)
                s = s.migrates_to

        return result

    @classmethod
    def __mbd_subst_placeholders(cls, value, repository, distribution):
        return util.subst_placeholders(
            value,
            {"IDENTITY": repository.identity,
             "CODEVERSION": distribution.base_source.codeversion})

    def mbd_get_mandatory_version_regex(self, distribution, suite_option):
        return self.__mbd_subst_placeholders(
            self.layout.experimental_mandatory_version_regex if suite_option.experimental else self.layout.mandatory_version_regex,
            self, distribution)

    def mbd_get_default_version(self, distribution, suite_option):
        return self.__mbd_subst_placeholders(
            self.layout.experimental_default_version if suite_option.experimental else self.layout.default_version,
            self, distribution)

    def mbd_get_apt_line(self, distribution, suite_option, rollback=None, snapshot=None):
        diststr = self.mbd_get_diststr(distribution, suite_option, rollback=rollback)
        if snapshot:
            snapshots = self.mbd_reprepro.get_snapshots(diststr)
            if snapshot not in snapshots:
                raise util.HTTPBadRequest(f"No such snapshot for {diststr}: '{snapshot}' (available: {','.join(snapshots)})")
            diststr += os.path.join("/", "snapshots", snapshot)

        return files.AptLine(config.URIS["repositories"]["static"].url_join(),
                             self.identity,
                             diststr,
                             dist.sorted_components([c.name for c in distribution.components.all()]),
                             comment=f"{util.daemon().model.mbd_get_archive_origin()} '{diststr}': {self.mbd_get_apt_pin(distribution, suite_option)}")

    def mbd_get_apt_build_sources_list(self, distribution, suite_option):
        sources_list = files.SourcesList()

        sources_list.append(distribution.base_source.mbd_get_apt_line(limit_components=distribution.components.all()))
        for e in distribution.extra_sources.all():
            sources_list.append(e.source.mbd_get_apt_line(limit_components=distribution.components.all()))
        for s in self.mbd_get_internal_suite_dependencies(suite_option):
            sources_list.append(self.mbd_get_apt_line(distribution, s))

        return sources_list

    def mbd_get_apt_build_preferences(self, distribution, suite_option, internal_apt_priority_override=None):
        result = ""

        # Get preferences for all extra (prioritized) sources
        for e in distribution.extra_sources.all():
            result += e.mbd_get_apt_preferences() + "\n"

        # Get preferences for all internal sources
        internal_prio = internal_apt_priority_override if internal_apt_priority_override else distribution.mbd_get_extra_option("Internal-APT-Priority", 1)
        for s in self.mbd_get_internal_suite_dependencies(suite_option):
            result += self.mbd_get_apt_preferences(distribution, s, prio=internal_prio) + "\n"

        return result

    def _mbd_reprepro_conf_options(self):
        return files.File("options",
                          snippet=(f"gnupghome {config.ROUTES['home'].path.join('.gnupg')}\n"
                                   f"{'morguedir +b/morguedir' if self.reprepro_morguedir else ''}\n"))

    def _mbd_reprepro_conf_distributions(self):
        dist_template = ("Codename: {distribution}\n"
                         "Suite: {distribution}\n"
                         "Label: {distribution}\n"
                         "AlsoAcceptFor: {meta_distributions}\n"
                         "Origin: {origin}\n"
                         "Components: {components}\n"
                         "UDebComponents: {components}\n"
                         "Architectures: source {architectures}\n"
                         "Description: {desc}\n"
                         "SignWith: default\n"
                         "NotAutomatic: {na}\n"
                         "ButAutomaticUpgrades: {bau}\n"
                         "DebIndices: Packages Release . .gz .xz\n"
                         "DscIndices: Sources Release . .gz .xz\n"
                         "Contents: .gz\n\n")

        result = ""
        for d in self.distributions.all():
            for s in self.layout.suiteoption_set.all():
                result += dist_template.format(
                    distribution=self.mbd_get_diststr(d, s),
                    meta_distributions=" ".join(self.mbd_get_meta_distributions(d, s)),
                    origin=util.daemon().model.mbd_get_archive_origin(),
                    components=" ".join(d.mbd_get_components()),
                    architectures=" ".join([x.name for x in d.architectures.all()]),
                    desc=self.mbd_get_diststr(d, s),
                    na="yes" if s.not_automatic else "no",
                    bau="yes" if s.but_automatic_upgrades else "no")

                for r in range(s.rollback):
                    result += dist_template.format(
                        distribution=self.mbd_get_diststr(d, s, r),
                        meta_distributions="",
                        origin=util.daemon().model.mbd_get_archive_origin(),
                        components=" ".join(d.mbd_get_components()),
                        architectures=" ".join([x.name for x in d.architectures.all()]),
                        desc=self.mbd_get_diststr(d, s, r),
                        na="yes",
                        bau="no")

        return files.File("distributions", snippet=result)

    def mbd_reprepro_update_config(self):
        if files.Dir(os.path.join(self.mbd_path, "conf")).add(self._mbd_reprepro_conf_options()) \
                                                         .add(self._mbd_reprepro_conf_distributions()).update():
            self.mbd_reprepro.reindex(self.mbd_get_diststrs())

    def mbd_dsc_pool_path(self, source, version, raise_exception=True):
        """Get DSC pool path of an installed source (``<repo>/pool/...``)"""
        repositories_path = config.ROUTES["repositories"].path
        dscs = glob.glob(repositories_path.join(self.identity, "pool", "*", "*", source, files.DebianName(source, version).dsc()))

        LOG.debug(f"Repository {self.identity}: Found pool DSCs for '{source}-{version}': {dscs}")

        if len(dscs) > 1:  # This should not really ever happen. Pool-wise it could however with different components (main, contrib, etc)
            LOG.warning(f"Found multiple DSCs for '{source}-{version}' in pool (only the 1st found will be used)")
            for d in dscs:
                LOG.warning(f"↳ {d}")

        if len(dscs) < 1:
            msg = f"Can't find DSC for '{source}-{version}' in pool of repository '{self.identity}'"
            if raise_exception:
                raise util.HTTPBadRequest(msg)
            LOG.warning(msg)
            return None

        return repositories_path.removeprefix(dscs[0])

    def _mbd_package_shift_rollbacks(self, distribution, suite_option, package_name):
        diststr = self.mbd_get_diststr(distribution, suite_option)
        ls = self.mbd_reprepro.ls(package_name)
        if ls.filter_rollbacks(version=ls[diststr]["version"]):
            LOG.info(f"{package_name}_{ls[diststr]['version']}: Already in rollbacks, skipping rollback")
        else:
            for r in range(suite_option.rollback - 1, -1, -1):
                src = self.mbd_get_diststr(distribution, suite_option, None if r == 0 else r - 1)
                dst = self.mbd_get_diststr(distribution, suite_option, r)
                LOG.debug(f"Rollback: Moving {package_name}: {src} to {dst}")
                try:
                    self.mbd_reprepro.migrate(package_name, src, dst)
                except BaseException as e:
                    util.log_exception(LOG, "Rollback failed (ignoring)", e)

    def _mbd_package_migrate(self, package, distribution, suite, rollback=None, version=None, log_event=True):
        src_diststr = self.mbd_get_diststr(distribution, suite)
        dst_diststr = None
        pkg_ls = self.mbd_reprepro.ls(package)

        if rollback is not None:
            dst_diststr = src_diststr
            LOG.info(f"Rollback restore of '{package}' from rollback {rollback} to '{dst_diststr}'")

            pkg_ls.filter(diststr=dst_diststr, raise_if_found=f"Package '{package}' exists in '{dst_diststr}': Remove first to restore rollback")
            rob_diststr = self.mbd_get_diststr(distribution, suite, rollback=rollback)
            pkg_ls.filter(diststr=rob_diststr, version=version, raise_if_not_found="No such rollback source")

            # Actually migrate package in reprepro
            self.mbd_reprepro.migrate(package, rob_diststr, dst_diststr, version)
        else:
            # Get src and dst dist strings, and check we are configured to migrate
            if not suite.migrates_to:
                raise util.HTTPBadRequest(f"You can't migrate from '{src_diststr}'")
            dst_diststr = self.mbd_get_diststr(distribution, suite.migrates_to)

            # Check if package is in src_dst (&& update 'version' if None)
            version = pkg_ls.filter(diststr=src_diststr, version=version, raise_if_not_found="No such source")[src_diststr]["version"]

            # Check that version is not already migrated
            if pkg_ls.filter(diststr=dst_diststr, version=version):
                raise util.HTTPBadRequest(f"Version '{version}' already migrated to '{dst_diststr}'")

            # Shift rollbacks in the destination distributions
            if pkg_ls.filter(diststr=dst_diststr):
                self._mbd_package_shift_rollbacks(distribution, suite.migrates_to, package)

            # Actually migrate package in reprepro
            self.mbd_reprepro.migrate(package, src_diststr, dst_diststr, version)

        if log_event:
            pkg_ls = self.mbd_reprepro.ls(package)
            util.daemon().events.log(events.Type.MIGRATED, pkg_ls.changes(dst_diststr, None).create_events())

    def mbd_package_migrate(self, package, distribution, suite, full=False, rollback=None, version=None):
        if full:
            while suite.migrates_to is not None:
                self._mbd_package_migrate(package, distribution, suite, rollback=rollback, version=version)
                suite = suite.migrates_to
        else:
            self._mbd_package_migrate(package, distribution, suite, rollback=rollback, version=version)

    def mbd_package_remove(self, package, distribution, suite, rollback=None, version=None, without_rollback=False):
        diststr = self.mbd_get_diststr(distribution, suite, rollback)
        src_pkg = self.mbd_reprepro.ls(package)
        src_pkg.filter(diststr=diststr, version=version, raise_if_not_found="No such source")

        if rollback is None:
            if not without_rollback:
                self._mbd_package_shift_rollbacks(distribution, suite, package)
            # Remove package
            self.mbd_reprepro.remove(package, diststr, version)
        else:
            # Rollback removal
            self.mbd_reprepro.remove(package, diststr, version)

            # Fix up empty rollback dist
            for r in range(rollback, suite.rollback - 1):
                src_diststr = self.mbd_get_diststr(distribution, suite, r + 1)
                dst_diststr = self.mbd_get_diststr(distribution, suite, r)
                try:
                    self.mbd_reprepro.migrate(package, src_diststr, dst_diststr)
                    self.mbd_reprepro.remove(package, src_diststr)
                except BaseException as e:
                    util.log_exception(LOG, f"Rollback: Moving '{package}' from '{src_diststr}' to '{dst_diststr}' FAILED (ignoring)", e, logging.WARN)

        # Notify
        util.daemon().events.log(events.Type.REMOVED, src_pkg.changes(diststr, None).create_events())

    def mbd_package_precheck(self, distribution, suite_option, package, version):
        # 1st, check that the given version matches the distribution's version restrictions
        mandatory_regex = self.mbd_get_mandatory_version_regex(distribution, suite_option)
        if not re.compile(mandatory_regex).search(version):
            raise util.HTTPBadRequest(f"Version restrictions failed for suite '{suite_option.suite.name}': '{mandatory_regex}' not in '{version}'")

        pkg_ls = self.mbd_reprepro.ls(package)
        diststr = self.mbd_get_diststr(distribution, suite_option)

        # 2nd: Check whether the very same version is already in any distribution
        pkg_ls.filter(version=version, raise_if_found="Source already installed")

        # 3rd: Check that distribution's current version is smaller than the to-be installed version
        pkg_in_dist = pkg_ls.filter(diststr=diststr)
        if pkg_in_dist:
            pkg_in_dist_version = pkg_in_dist[diststr]["version"]
            if debian.debian_support.Version(version) < debian.debian_support.Version(pkg_in_dist_version):
                raise util.HTTPBadRequest(f"Package '{package}' has greater version '{pkg_in_dist_version}' installed in '{diststr}'")

    def _mbd_buildresult_install(self, buildresult, diststr):
        # Don't try install if skipped
        if buildresult.cget("Sbuild-Status") == "skipped":
            LOG.debug(f"Skipped: {buildresult}")
        else:
            with util.tmp_dir(prefix="buildresult-untar-") as tmpdir:
                buildresult.untar(tmpdir)
                self.mbd_reprepro.install(" ".join(glob.glob(os.path.join(tmpdir, "*.changes"))), diststr)
                LOG.info(f"Installed: {buildresult}")

    def mbd_package_install(self, distribution, suite_option, changes, buildresults):
        """Install a dict arch:buildresult of successful build results"""
        # Get the full distribution str
        diststr = self.mbd_get_diststr(distribution, suite_option)

        # Check that all mandatory archs are present
        missing_mandatory_archs = [arch for arch in distribution.mbd_get_mandatory_architectures() if arch not in buildresults]
        if missing_mandatory_archs:
            raise util.HTTPBadRequest(f"{len(missing_mandatory_archs)} mandatory architecture(s) missing: {' '.join(missing_mandatory_archs)}")

        # Get the (source) package name
        package = changes["Source"]
        LOG.debug(f"Package install: Package={package}")

        # Shift current package up in the rollback distributions (unless this is the initial install)
        is_installed = self.mbd_reprepro.ls(package).filter(diststr=diststr)
        if is_installed:
            self._mbd_package_shift_rollbacks(distribution, suite_option, package)

        # First, install the dsc
        self.mbd_reprepro.install_dsc(changes.dsc_file_path(), diststr)

        # Second, install all build results; if any buildresult fails to install, rollback changes to repository
        try:
            for buildresult in list(buildresults.values()):
                self._mbd_buildresult_install(buildresult, diststr)
        except Exception as e:
            self.mbd_reprepro.remove(package, diststr)
            if is_installed:
                self._mbd_package_migrate(package, distribution, suite_option, rollback=0, log_event=False)
            util.log_exception(LOG, "Binary install failed", e)
            raise util.HTTPInternal(f"Binary install failed for {package}: {util.e2http(e)}")

    def mbd_sorted_distributions(self):
        return sorted(self.distributions.all())

    def mbd_icodenames(self):
        for d in self.mbd_sorted_distributions():
            yield d.base_source.codename

    def mbd_prepare(self):
        """Idempotent repository preparation. This may be used as-is as mbd_sync"""
        # Architecture sanity checks
        for d in self.mbd_sorted_distributions():
            if not d.architectureoption_set.all().filter(optional=False):
                raise util.HTTPBadRequest(f"{d}: There must be at least one mandatory architecture")
            if len(d.architectureoption_set.all().filter(optional=False, build_architecture_all=True)) != 1:
                raise util.HTTPBadRequest(f"{d}: There must be exactly one one arch-all architecture")

        # Check that the codenames of the distribution are unique
        codenames = []
        for d in self.distributions.all():
            if d.base_source.codename in codenames:
                raise util.HTTPBadRequest(f"Multiple distribution codename in: {d}")
            codenames.append(d.base_source.codename)

            # Check for mandatory component "main"
            if not d.components.all().filter(name="main"):
                raise util.HTTPBadRequest(f"Mandatory component 'main' missing in: {d}")

        self.mbd_reprepro_update_config()

    def mbd_sync(self):
        self.mbd_prepare()

    def mbd_remove(self):
        if os.path.exists(self.mbd_path):
            shutil.rmtree(self.mbd_path)

    def mbd_check(self):
        self.mbd_reprepro_update_config()
        self.mbd_reprepro.check()

        # Check for ambiguity with other repos in meta distribution maps
        get_meta_distribution_map()

    def mbd_get_dependencies(self):
        result = []
        for d in self.distributions.all():
            result.append(d.base_source)
            result += [e.source for e in d.extra_sources.all()]
        return result


def get_meta_distribution_map():
    """Get a dict of the meta distributions: meta -> actual"""
    result = {}
    for r in Repository.objects.all():
        for d in r.distributions.all():
            for s in r.layout.suiteoption_set.all():
                for m in r.mbd_get_meta_distributions(d, s):
                    diststr = r.mbd_get_diststr(d, s)
                    if m in result:
                        raise util.HTTPInternal(f"Ambiguous Meta-Distributions ({m}={diststr} or {result[m]}). "
                                                f"Please check Repositories and Layouts (see Layouts/Meta-Distributions in Administrators Manual)")
                    result[m] = diststr

    LOG.debug(f"Got meta distribution map: {result}")
    return result


def map_distribution(diststr):
    """Map incoming distribution to internal"""
    return get_meta_distribution_map().get(diststr, diststr)


def get_repository(identity):
    """Get repository object with user error handling"""
    repository = Repository.objects.filter(pk=identity).first()
    if repository is None:
        raise util.HTTPBadRequest(f"No such repository: {identity}")
    return repository


def parse_dist(_dist, check_uploadable=False):
    repository = get_repository(_dist.repository)

    distribution = repository.distributions.all().filter(base_source__codename__exact=_dist.codename).first()
    if distribution is None:
        raise util.HTTPBadRequest(f"No distribution for codename '{_dist.codename}' in repository '{repository.identity}'")

    suite = repository.layout.suiteoption_set.filter(suite__name=_dist.suite).first()
    if suite is None:
        raise util.HTTPBadRequest(f"No distribution for suite '{_dist.suite}' in repository '{repository.identity}'")

    if check_uploadable:
        if _dist.rollback_no is not None or not suite.uploadable:
            raise util.HTTPBadRequest(f"Distribution '{_dist.get()}' not uploadable")

        if not distribution.mbd_is_active():
            raise util.HTTPUnavailable(f"Distribution '{_dist.get()}' not active")

        if not repository.mbd_is_active():
            raise util.HTTPUnavailable(f"Repository '{repository.identity}' not active")

    return repository, distribution, suite


def parse_diststr(diststr, check_uploadable=False):
    return parse_dist(dist.Dist(diststr), check_uploadable=check_uploadable)


if util.VERSIONS.has("django", "3.2"):
    class DaemonEmailField(django.db.models.EmailField):
        default_validators = [django.core.validators.EmailValidator(allowlist=["localhost", config.HOSTNAME_FQDN])]
else:
    class DaemonEmailField(django.db.models.EmailField):
        default_validators = [django.core.validators.EmailValidator(whitelist=["localhost", config.HOSTNAME_FQDN])]


class Daemon(StatusModel):
    MBD_HELP_EXTRA_OPTIONS = """
X-Archive-Origin: <origin>
  Custom APT "Origin" field for repository indices.
"""

    MBD_HELP_NEEDS_RESTART = "[Daemon must be restarted to activate this setting.]\n\n"

    # Basics
    identity = django.db.models.CharField(max_length=50,
                                          default=config.HOSTNAME,
                                          help_text=("Will be used to identify this mini-buildd instance in various places:\n"
                                                     "\n"
                                                     "* 'Name-Real' part of the GnuPG key.\n"
                                                     "* name of the automated keyring packages.\n"
                                                     "* the 'Origin' tag of repository indices (but this may also be customized via 'X-Archive-Origin' extra option, see ``Extra options`` below).\n"
                                                     "* to identify remotes.\n"))

    hostname = django.db.models.CharField(
        max_length=200,
        blank=True,
        help_text="Obsoleted: Use command line arg '--hostname' instead.")

    email_address = DaemonEmailField(max_length=255, default=f"mini-buildd@{config.HOSTNAME_FQDN}",
                                     help_text=("Will be used in:\n"
                                                "\n"
                                                "* 'Name-Email' part of the GnuPG key.\n"
                                                "* Sender for notification email.\n"))
    # GnuPG options
    gnupg_template = UnixTextField(default=("Key-Type: RSA\n"
                                            "Key-Length: 4096\n"
                                            "Expire-Date: 0"),
                                   help_text=("Template as accepted by 'gpg --batch --gen-key' (see 'man gpg').\n"
                                              "\n"
                                              "You should not give 'Name-Real' and 'Name-Email', as these are\n"
                                              "automatically added.\n"
                                              "\n"
                                              "Prepare/Remove actions will generate/remove the GnuPG key.\n"))

    gnupg_keyserver = django.db.models.CharField(
        max_length=200,
        default="keyserver.ubuntu.com",
        help_text="GnuPG keyserver to use (as fill-in helper).")

    ftpd_bind = django.db.models.CharField(
        max_length=200,
        default="tcp:port=8067",
        help_text=MBD_HELP_NEEDS_RESTART + f"FTP Server Endpoint: {inspect.getdoc(net.ServerEndpoint)}")

    ftpd_options = django.db.models.CharField(max_length=255,
                                              default="",
                                              blank=True,
                                              help_text=MBD_HELP_NEEDS_RESTART + ftpd.FtpD.HANDLER_OPTIONS_USAGE)

    # Load options
    build_queue_size = django.db.models.IntegerField(
        default=util.get_cpus(),
        help_text=MBD_HELP_NEEDS_RESTART + "Max Parallel Builds: Maximum number of parallel builds.")

    sbuild_jobs = django.db.models.IntegerField(
        default=1,
        help_text="DEPRECATED/UNUSED (since 1.99.17). See Developer Manual, 'Is package building parallel?'")

    # EMail options
    # DEPRECATED/UNUSED: With the switch to django mail framework, this is now configured via the --smtp command line argument.
    smtp_server = django.db.models.CharField(
        max_length=254,
        default="localhost:25",
        help_text="DEPRECATED/UNUSED: Replaced by '--smtp' command line option.")

    notify = django.db.models.ManyToManyField(EmailAddress,
                                              blank=True,
                                              help_text="Addresses that get all notification emails unconditionally.")
    allow_emails_to = django.db.models.CharField(
        max_length=254,
        default=f".*@{config.HOSTNAME_FQDN}",
        help_text=("Regex to allow sending emails to automatically computed\n"
                   "addresses (currently the 'Changed-By' or 'Maintainer' addresses\n"
                   "when this feature is enabled for the resp. repository).\n"
                   "\n"
                   "Use '.*' to allow all -- it's however recommended to put this to\n"
                   "s.th. like '.*@myemail.domain', to avoid original package\n"
                   "maintainers to be accidentally spammed.\n"))

    custom_hooks_directory = django.db.models.CharField(max_length=255, default="", blank=True, help_text="DEPRECATED, never used (there are now event hooks in predefined path ~/etc/event-hooks).")

    show_last_packages = django.db.models.IntegerField(
        default=500,
        help_text=MBD_HELP_NEEDS_RESTART + "Event queue size: Max number of items that can be stored in the events queue.")
    show_last_builds = django.db.models.IntegerField(
        default=100,
        help_text="Event queue view size: Max number of items visible on home page (this essentially pimps its load time).")

    wait_for_build_results = django.db.models.IntegerField(
        default=5,
        help_text="Future use: How many days to wait for build results until finishing a package.")
    keep_build_results = django.db.models.IntegerField(
        default=5,
        help_text="Future use: How many days to keep build results that cannot be uploaded.")

    LETHAL_DEPENDENCIES = False

    class Meta(StatusModel.Meta):
        verbose_name_plural = "Daemon"

    class Admin(StatusModel.Admin):
        exclude = ("hostname", "custom_hooks_directory", "sbuild_jobs", "smtp_server", "wait_for_build_results", "keep_build_results")

        filter_horizontal = ("notify",)

        @classmethod
        def _mbd_on_deactivation(cls, obj):
            util.daemon().mbd_stop()

    def save(self, *args, **kwargs):
        """Save model && sync with Daemon()"""
        super().save(*args, **kwargs)
        util.daemon().sync()

    def __str__(self):
        return (f"Daemon '{self.identity}' serving {len(Repository.mbd_get_active())} repositories, "
                f"entertaining {len(Chroot.mbd_get_active())} chroots, "
                f"using {len(Remote.mbd_get_active())} remotes")

    def mbd_get_ftp_endpoint(self):
        return net.ServerEndpoint(self.ftpd_bind, protocol="ftp")

    def mbd_fullname(self):
        return f"mini-buildd archive {self.identity}"

    def mbd_gnupg(self):
        return gnupg.GnuPG(self.gnupg_template, self.mbd_fullname(), self.email_address)

    def clean(self, *args, **kwargs):
        super().clean(*args, **kwargs)
        self.mbd_validate_regex(r"^[a-zA-Z0-9\-]+$", self.identity, "identity")

        try:
            ftpd.FtpD.parse_handler_options(self.ftpd_options)
        except BaseException as e:
            raise django.core.exceptions.ValidationError({"ftpd_options": f"Can't parse '{self.ftpd_options}': {util.e2http(e)}"})

        try:
            net.ServerEndpoint(self.ftpd_bind, protocol="ftp")
        except BaseException as e:
            raise django.core.exceptions.ValidationError({"ftpd_bind": f"Can't parse '{self.ftpd_bind}': {util.e2http(e)}"})

        if Daemon.objects.count() > 0 and self.pk != Daemon.objects.get().pk:
            raise django.core.exceptions.ValidationError("You can only create one Daemon instance")

    def mbd_prepare(self):
        _gnupg = self.mbd_gnupg()
        _gnupg.prepare()
        util.daemon().gnupg = _gnupg
        LOG.debug("Daemon GnuPG prepared.")

    @classmethod
    def mbd_sync(cls):
        LOG.warning("The GnuPG key will never be updated automatically. Explicitly run remove+prepare to achieve this.")

    def mbd_remove(self):
        self.mbd_gnupg().remove()
        LOG.debug("Daemon GnuPG removed.")

    def mbd_get_dependencies(self):
        """All active or to-be active repositories, remotes and chroots"""
        result = []
        for model in [Repository, Chroot, Remote]:
            for o in model.mbd_get_active_or_auto_reactivate():
                result.append(o)
        return result

    @classmethod
    def mbd_check(cls):
        """Just warn in case there are no repos and no chroots"""
        if not Repository.mbd_get_active() and not Chroot.mbd_get_active():
            LOG.warning("No active chroot or repository.")

    def mbd_get_archive_origin(self):
        return self.mbd_get_extra_option("X-Archive-Origin", f"Mini-Buildd archive {self.identity} on {config.HOSTNAME_FQDN}")

    def mbd_get_dput_conf(self):
        ftp = self.mbd_get_ftp_endpoint()
        return (f"[mini-buildd-{self.identity}]\n"
                f"method   = {'ftps' if ftp.is_ssl() else 'ftp'}\n"
                f"fqdn     = {ftp.hopo()}\n"
                "login    = anonymous\n"
                "incoming = /incoming\n")

    def _mbd_notify_signature(self, typ):
        reason = {"daemon": "Your address is configured to get any notifications (contact administrators if you don't want this).",
                  "repository": "Your address is configured to get any notifications for this repository (contact administrators if you don't want this).",
                  "changed-by": "Your address is the uploader of the package ('Changed-By' in changes).",
                  "maintainer": "Your address is the maintainer of the package ('Maintainer' in changes).",
                  "subscriber": "Your user account has a matching subscription.",
                  }.get(typ, "Unknown")

        return (f"--\N{SPACE}\n"
                f"mini-buildd instance '{self.identity}' at {config.HOSTNAME} <{self.email_address}>\n"
                f"Reason for this mail: {reason}\n"
                f"Visit mini-buildd   : {util.http_endpoint().geturl()}\n")

    def mbd_notify_event(self, event):
        repository, distribution = None, None
        try:
            repository, distribution, _suite = parse_diststr(event.distribution)
        except BaseException:
            pass

        try:
            subject = f"{event}"
            body = util.json_pretty(event.to_json()) + "\n"

            # Old mbd_notify() code:
            subject_prefix = f"[mini-buildd-{self.identity}] "
            m_to = []
            m_to_raw = []
            m_automatic_to_allow = re.compile(self.allow_emails_to)

            def add_to(address, typ, is_automatic):
                address_raw = email.utils.parseaddr(address)[1]
                if not is_automatic or m_automatic_to_allow.search(address_raw):
                    if address_raw in m_to_raw:
                        LOG.debug(f"Notify: Skipping {typ} address: {address}: Duplicate")
                    else:
                        m_to.append((subject_prefix + subject, body + self._mbd_notify_signature(typ), self.email_address, [address]))
                        m_to_raw.append(address_raw)
                        LOG.debug(f"Notify: Adding {typ} address: {address}")
                else:
                    LOG.info(f"Notify: Skipping {typ} address: {address}: Not allowed (only '{self.allow_emails_to}')")

            def get_subscriptions():
                source = "" if event.source is None else event.source
                real_distribution = distribution
                if real_distribution is None:
                    # If distribution was not given explicitly, try from changes, resolving meta dists if needed.
                    changes_dist = "" if event.distribution is None else event.distribution
                    real_distribution = map_distribution(changes_dist)
                return Subscription.objects.filter(package__in=[source, ""], distribution__in=[real_distribution, ""])

            # Add hardcoded addresses from daemon
            for m in self.notify.all():
                add_to(str(m), "daemon", is_automatic=False)

            # Add hardcoded addresses from repository
            if repository:
                for m in repository.notify.all():
                    add_to(str(m), "repository", is_automatic=False)

            # Add package uploader (Changed-By): Add even if we do not have a repo, so uploader is informed on such error cases too.
            if event.extra.get("changed_by") and (repository is None or repository.notify_changed_by):
                add_to(event.extra.get("changed_by"), "changed-by", is_automatic=True)

            # Add package maintainer: Add only when we have a repo, and it's configured to do so
            if event.extra.get("maintainer") and repository and repository.notify_maintainer:
                add_to(event.extra.get("maintainer"), "maintainer", is_automatic=True)

            # Add user subscriptions
            for s in get_subscriptions():
                address = f"{s.subscriber.get_full_name()} <{s.subscriber.email}>"
                if s.subscriber.is_active:
                    add_to(address, "subscriber", is_automatic=False)
                else:
                    LOG.debug(f"Notify: Skipping subscription address: {address}: Account disabled")

            django.core.mail.send_mass_mail(m_to)
            LOG.debug(f"EMail notification for event ({event}) sent.")
        except BaseException as e:
            util.log_exception(LOG, f"EMail notification for event ({event}) FAILED", e)


def get_daemon():
    model, created = Daemon.objects.get_or_create(id=1, defaults={"ftpd_bind": config.default_ftp_endpoint()})
    if created:
        LOG.info("New Daemon model instance created")
    return model


class Chroot(StatusModel):
    PERSONALITIES = {'i386': 'linux32'}

    MBD_HELP_EXTRA_OPTIONS = """
Debootstrap-Command: <ALT_COMMAND>: Alternate command to run instead of standard debootstrap

  For example, ``Debootstrap-Command: /usr/sbin/mini-buildd-debootstrap-uname-2.6`` may be used as workaround for urold (``<= lenny``) distributions.

Debootstrap-Extra-Options: <EXTRA_OPTIONS>: Additional debootrap options

  For example, ``--include=foo,bar`` may be used to have these packages pre-installed in build chroots.
"""

    source = django.db.models.ForeignKey(Source,
                                         on_delete=django.db.models.CASCADE,
                                         help_text=("Base source to create the chroot from; its codename must be 'debootstrapable'.\n"
                                                    "\n"
                                                    "Examples: Debian 'squeeze', Debian 'wheezy', Ubuntu 'hardy'.\n"))
    architecture = django.db.models.ForeignKey(Architecture,
                                               on_delete=django.db.models.CASCADE,
                                               help_text=("Chroot's architecture; using the same arch as the host system\n"
                                                          "will always work, other architectures may work if supported. An\n"
                                                          "'amd64' host, for example, will also allow for architecture\n"
                                                          "'i386'.\n"))
    personality = django.db.models.CharField(max_length=50, editable=False, blank=True, default="",
                                             help_text=("Schroot 'personality' value (see 'schroot'); for 32bit chroots\n"
                                                        "running on a 64bit host, this must be 'linux32'.\n"))
    personality_override = django.db.models.CharField(max_length=50, blank=True, default="",
                                                      help_text=("Leave empty unless you want to override the automated way (via\n"
                                                                 "an internal mapping). Please report manual overrides so it can\n"
                                                                 "go to the default mapping.\n"))

    class Meta(StatusModel.Meta):
        unique_together = ("source", "architecture")
        ordering = ["source", "architecture"]

    class Admin(StatusModel.Admin):
        search_fields = ["source__codename", "architecture__name"]
        readonly_fields = ["personality"]

        def get_readonly_fields(self, _request, obj=None):
            """Forbid change source/arch on existing chroot (we would loose the path to the associated data)"""
            fields = copy.copy(self.readonly_fields)
            if obj:
                fields.append("source")
                fields.append("architecture")
            return fields

    def mbd_key(self):
        return f"{self.source.codename}:{self.architecture.name}"

    def __str__(self):
        return f"{self.__class__.__name__} for {self.source} with {self.architecture}"

    @property
    def mbd_path(self):
        # Note: Property as this code must not be in __init__(): self.source, self.architecture may not be initialized (error on "add" in django admin).
        return config.ROUTES["chroots"].path.join(self.source.codename, self.architecture.name)

    def mbd_get_backend(self):
        """Get actual class of (saved) instance (even if operating on base class ``Chroot`` only)"""
        for cls in ["filechroot", "dirchroot", "looplvmchroot", "lvmchroot", "btrfssnapshotchroot"]:
            if hasattr(self, cls):
                return getattr(self, cls)
        raise util.HTTPBadRequest("No chroot backend found")

    def mbd_schroot_name(self):
        """
        Schroot name w/o namespace (see ``schroot --list``)

        Must produce same as :func:`~changes.Buildrequest.schroot_name`.
        """
        return f"mini-buildd-{self.source.codename}-{self.architecture.name}"

    def mbd_get_tmp_dir(self):
        return os.path.join(self.mbd_path, "tmp")

    def mbd_get_schroot_conf_file(self):
        return os.path.join(self.mbd_path, "schroot.conf")

    def mbd_get_keyring_file(self):
        """Get keyring file path. Holds all keys from the source to verify the release via debootstrap's --keyring option"""
        return os.path.join(self.mbd_path, "keyring.gpg")

    def mbd_get_system_schroot_conf_file(self):
        return os.path.join("/etc/schroot/chroot.d", self.mbd_schroot_name() + ".conf")

    def mbd_get_pre_sequence(self):
        """Get preliminary sequence. Subclasses may implement this to do define an extra preliminary sequence"""
        LOG.debug(f"{self}: No pre-sequence defined.")
        return []

    def mbd_get_sequence(self):
        # In case a source was removed, we might not be able to get an archive URL for debootstrap.
        # We should be able to assemble the sequence anyway to be able to remove that chroot.
        try:
            debootstrap_url = self.source.mbd_get_archive().url
        except BaseException as e:
            LOG.warning(f"{self}: Can't get archive URL from source (source removed?): {e}")
            debootstrap_url = "no_archive_url_found_maybe_source_is_removed"

        debootstrap_extra_options = shlex.split(self.mbd_get_extra_option("Debootstrap-Extra-Options", ""))

        return [
            (["/bin/mkdir", "--verbose", self.mbd_get_tmp_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_tmp_dir()])] + self.mbd_get_backend().mbd_get_pre_sequence() + [
                 ([self.mbd_get_extra_option("Debootstrap-Command", "/usr/sbin/debootstrap"),
                   "--variant", "buildd",
                   "--keyring", self.mbd_get_keyring_file(),
                   "--arch", self.architecture.name,
                   *debootstrap_extra_options,
                   self.source.codename,
                   self.mbd_get_tmp_dir(),
                   debootstrap_url],
                  ["/bin/umount", "-v", os.path.join(self.mbd_get_tmp_dir(), "proc"), os.path.join(self.mbd_get_tmp_dir(), "sys")])] + self.mbd_get_backend().mbd_get_post_sequence() + [
                      (["/bin/cp", "--verbose", self.mbd_get_schroot_conf_file(), self.mbd_get_system_schroot_conf_file()],
                       ["/bin/rm", "--verbose", self.mbd_get_system_schroot_conf_file()])]

    def mbd_prepare(self):
        os.makedirs(self.mbd_path, exist_ok=True)

        # Set personality
        self.personality = self.personality_override if self.personality_override else self.PERSONALITIES.get(self.architecture.name, "linux")

        files.File("schroot.conf",
                   snippet=(f"[{self.mbd_schroot_name()}]\n"
                            f"description=Mini-Buildd chroot {self.mbd_schroot_name()}\n"
                            f"setup.fstab=mini-buildd/fstab\n"
                            f"groups=sbuild\n"
                            f"users=mini-buildd\n"
                            f"root-groups=sbuild\n"
                            f"root-users=mini-buildd\n"
                            f"source-root-users=mini-buildd\n"
                            f"personality={self.personality}\n"
                            f"\n"
                            f"# Backend specific config\n"
                            f"{self.mbd_get_backend().mbd_get_schroot_conf()}\n")).save_as(self.mbd_get_schroot_conf_file())

        # Gen keyring file to use with debootstrap
        with contextlib.closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-debootstrap-"})) as gpg:
            for k in self.source.apt_keys.all():
                gpg.add_pub_key(k.key)
            gpg.export(self.mbd_get_keyring_file())

        call.call_sequence(self.mbd_get_sequence(), run_as_root=True)

    def mbd_remove(self):
        call.call_sequence(self.mbd_get_sequence(), rollback_only=True, run_as_root=True)
        util.rmdirs(self.mbd_path)

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_backend_check(self):
        """Run backend check. Subclasses may implement this to do extra backend-specific checks"""
        LOG.debug(f"{self}: No backend check implemented.")

    def mbd_apt_line(self):
        return self.source.mbd_get_apt_line(limit_components=[Component(name="main")]).get()

    def mbd_check(self):
        with closing(schroot.Session(self.mbd_schroot_name(), "source")) as session:
            # Check for the old sudo workaround (chroots created by mini-buildd <= 1.0.4)
            session.check_sudo_workaround()

            # Basic function checks
            session.info()

            # Globally configure debconf frontend to noninteractive.
            # This should avoid possible stalls due to debconf interaction on apt commands on the chroot.
            # (While rare, this definitely occasionally happened with rolling distributions)
            # There is no good way via schroot to convey environment (like DEBIAN_FRONTEND) for specific commands.
            # Also, a generic approach via schroot.conf seems unwise (would need --preserve-environment, and then to manually maintain 'environment-filter' regex).
            session.set_debconf("debconf/frontend", "noninteractive")

            # Backend checks
            self.mbd_get_backend().mbd_backend_check()

            # Update base apt line (source might have switched to another archive).
            session.update_file("/etc/apt/sources.list", self.mbd_apt_line())

            # "apt update/upgrade" check
            for args, fatal in [(["update"], True),
                                (["--ignore-missing", "dist-upgrade"], True),
                                (["--purge", "autoremove"], False),
                                (["clean"], True)]:
                try:
                    session.run(
                        [
                            "--directory", "/",
                            "--",
                        ] + shlex.split(sbuild.APT_GET) + args)
                except BaseException as e:
                    LOG.warning(f"'apt-get {' '.join(args)}' not supported in this chroot: {e}")
                    if fatal:
                        raise

    def mbd_get_dependencies(self):
        return [self.source]


class DirChroot(Chroot):
    """Directory chroot backend"""

    UNION_AUFS = 0
    UNION_OVERLAYFS = 1
    UNION_UNIONFS = 2
    UNION_OVERLAY = 3
    UNION_CHOICES = (
        (UNION_AUFS, "aufs"),
        (UNION_OVERLAYFS, "overlayfs"),
        (UNION_UNIONFS, "unionfs"),
        (UNION_OVERLAY, "overlay"))
    union_type = django.db.models.IntegerField(choices=UNION_CHOICES,
                                               default=util.guess_default_dirchroot_backend(overlay=UNION_OVERLAY, aufs=UNION_AUFS),
                                               help_text="""\
See 'man 5 schroot.conf'
""")

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    def mbd_backend_flavor(self):
        return self.get_union_type_display()

    def mbd_get_chroot_dir(self):
        return os.path.join(self.mbd_path, "source")

    def mbd_get_schroot_conf(self):
        return (f"type=directory\n"
                f"directory={self.mbd_get_chroot_dir()}\n"
                f"union-type={self.get_union_type_display()}\n")

    def mbd_get_post_sequence(self):
        return [
            (["/bin/mv",
              "--verbose",
              self.mbd_get_tmp_dir(),
              self.mbd_get_chroot_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_chroot_dir()])]


class FileChroot(Chroot):
    """File chroot backend"""

    COMPRESSION_NONE = 0
    COMPRESSION_GZIP = 1
    COMPRESSION_BZIP2 = 2
    COMPRESSION_XZ = 3
    COMPRESSION_CHOICES = (
        (COMPRESSION_NONE, "no compression"),
        (COMPRESSION_GZIP, "gzip"),
        (COMPRESSION_BZIP2, "bzip2"),
        (COMPRESSION_XZ, "xz"))

    compression = django.db.models.IntegerField(choices=COMPRESSION_CHOICES, default=COMPRESSION_NONE)

    TAR_ARGS = {
        COMPRESSION_NONE: [],
        COMPRESSION_GZIP: ["--gzip"],
        COMPRESSION_BZIP2: ["--bzip2"],
        COMPRESSION_XZ: ["--xz"]}
    TAR_SUFFIX = {
        COMPRESSION_NONE: "tar",
        COMPRESSION_GZIP: "tar.gz",
        COMPRESSION_BZIP2: "tar.bz2",
        COMPRESSION_XZ: "tar.xz"}

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    def mbd_backend_flavor(self):
        return self.TAR_SUFFIX[self.compression]

    def mbd_get_tar_file(self):
        return os.path.join(self.mbd_path, "source." + self.TAR_SUFFIX[self.compression])

    def mbd_get_schroot_conf(self):
        return (f"type=file\n"
                f"file={self.mbd_get_tar_file()}\n")

    def mbd_get_post_sequence(self):
        return [
            (["/bin/tar",
              "--create",
              "--directory", self.mbd_get_tmp_dir(),
              "--file", self.mbd_get_tar_file()] + self.TAR_ARGS[self.compression] + ["."],
             []),
            (["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_tmp_dir()],
             [])]


class LVMChroot(Chroot):
    """LVM chroot backend"""

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    volume_group = django.db.models.CharField(max_length=80, default="auto",
                                              help_text="Give a pre-existing LVM volume group name. Just leave it on 'auto' for loop lvm chroots.")
    filesystem = django.db.models.CharField(max_length=10, default="ext2")
    snapshot_size = django.db.models.IntegerField(default=4,
                                                  help_text="Snapshot device file size in GB.")

    def mbd_backend_flavor(self):
        return f"lvm={self.volume_group}/{self.filesystem}/{self.snapshot_size}G"

    def mbd_get_volume_group(self):
        try:
            return self.looplvmchroot.mbd_get_volume_group()
        except BaseException:
            return self.volume_group

    def mbd_get_lvm_device(self):
        return f"/dev/{self.mbd_get_volume_group()}/{self.mbd_schroot_name()}"

    def mbd_get_schroot_conf(self):
        return (f"type=lvm-snapshot\n"
                f"device={self.mbd_get_lvm_device()}\n"
                f"mount-options=-t {self.filesystem} -o noatime,user_xattr\n"
                f"lvm-snapshot-options=--size {self.snapshot_size}G\n")

    def mbd_get_pre_sequence(self):
        if not os.path.exists("/sbin/lvcreate"):
            raise util.HTTPUnavailable("LVM not installed")

        return [
            (["/sbin/lvcreate", "--size", f"{self.snapshot_size}G", "--name", self.mbd_schroot_name(), self.mbd_get_volume_group()],
             ["/sbin/lvremove", "--verbose", "--force", self.mbd_get_lvm_device()]),

            (["/sbin/mkfs", "-t", self.filesystem, self.mbd_get_lvm_device()],
             []),

            (["/bin/mount", "-v", "-t", self.filesystem, self.mbd_get_lvm_device(), self.mbd_get_tmp_dir()],
             ["/bin/umount", "-v", self.mbd_get_tmp_dir()])]

    def mbd_get_post_sequence(self):
        return [(["/bin/umount", "-v", self.mbd_get_tmp_dir()], [])]

    def mbd_backend_check(self):
        call.Call(["/sbin/fsck", "-a", "-t", self.filesystem, self.mbd_get_lvm_device()],
                  run_as_root=True).check()


class LoopLVMChroot(LVMChroot):
    """Loop LVM chroot backend"""

    class Meta(LVMChroot.Meta):
        pass

    class Admin(LVMChroot.Admin):
        pass

    loop_size = django.db.models.IntegerField(default=100,
                                              help_text="Loop device file size in GB.")

    def mbd_backend_flavor(self):
        return f"{self.loop_size}G loop: {super().mbd_backend_flavor()}"

    def mbd_get_volume_group(self):
        return f"mini-buildd-loop-{self.source.codename}-{self.architecture.name}"

    def mbd_get_backing_file(self):
        return os.path.join(self.mbd_path, "lvmloop.image")

    def mbd_get_loop_device(self):
        for f in glob.glob("/sys/block/loop[0-9]*/loop/backing_file"):
            with util.fopen(f) as ld:
                if os.path.realpath(ld.read().strip()) == os.path.realpath(self.mbd_get_backing_file()):
                    return "/dev/" + f.split("/")[3]
        LOG.debug(f"No existing loop device for {self.mbd_get_backing_file()}, searching for free device")
        return call.Call(["/sbin/losetup", "--find"], run_as_root=True).check().stdout.rstrip()

    def mbd_get_pre_sequence(self):
        loop_device = self.mbd_get_loop_device()
        LOG.debug(f"Acting on loop device: {loop_device}")
        return [
            (["/bin/dd",
              "if=/dev/zero", f"of={self.mbd_get_backing_file()}",
              f"bs={self.loop_size}M",
              "seek=1024", "count=0"],
             ["/bin/rm", "--verbose", self.mbd_get_backing_file()]),

            (["/sbin/losetup", "--verbose", loop_device, self.mbd_get_backing_file()],
             ["/sbin/losetup", "--verbose", "--detach", loop_device]),

            (["/sbin/pvcreate", "--verbose", loop_device],
             ["/sbin/pvremove", "--verbose", loop_device]),

            (["/sbin/vgcreate", "--verbose", self.mbd_get_volume_group(), loop_device],
             ["/sbin/vgremove", "--verbose", "--force", self.mbd_get_volume_group()])] + super().mbd_get_pre_sequence()


class BtrfsSnapshotChroot(Chroot):
    """Btrfs Snapshot chroot backend"""

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    @classmethod
    def mbd_backend_flavor(cls):
        return "btrfs_snapshot"

    def mbd_get_chroot_dir(self):
        return os.path.join(self.mbd_path, "source")

    def mbd_get_snapshot_dir(self):
        return os.path.join(self.mbd_path, "snapshot")

    def mbd_get_schroot_conf(self):
        return (f"type=btrfs-snapshot\n"
                f"btrfs-source-subvolume={self.mbd_get_chroot_dir()}\n"
                f"btrfs-snapshot-directory={self.mbd_get_snapshot_dir()}\n")

    def mbd_get_pre_sequence(self):
        return [
            (["/bin/btrfs", "subvolume", "create", self.mbd_get_chroot_dir()],
             ["/bin/btrfs", "subvolume", "delete", self.mbd_get_chroot_dir()]),

            (["/bin/mount", "-v", "-o", "bind", self.mbd_get_chroot_dir(), self.mbd_get_tmp_dir()],
             ["/bin/umount", "-v", self.mbd_get_tmp_dir()]),

            (["/bin/mkdir", "--verbose", self.mbd_get_snapshot_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_snapshot_dir()])]

    def mbd_get_post_sequence(self):
        return [(["/bin/umount", "-v", self.mbd_get_tmp_dir()], [])]


class Subscription(Model):
    subscriber = django.db.models.ForeignKey(django.contrib.auth.models.User,
                                             on_delete=django.db.models.CASCADE)
    package = django.db.models.CharField(max_length=100, blank=True)
    distribution = django.db.models.CharField(max_length=100, blank=True)

    def __str__(self):
        return f"Subscription for '{self.subscriber}' (subscribes to '{self.package if self.package else 'any package'}' in '{self.distribution if self.distribution else 'any distribution'}')"
