import dataclasses
import functools
import itertools
import operator
import os.path
import re
import textwrap
from abc import ABC
from typing import (
    FrozenSet,
    Optional,
    cast,
    Mapping,
    Iterable,
    List,
    Generic,
    TypeVar,
    Union,
    Callable,
    Tuple,
    Any,
    Set,
    TYPE_CHECKING,
    Sequence,
    Dict,
)

from debian.debian_support import DpkgArchTable, Version

from debputy.filesystem_scan import VirtualPathBase
from debputy.linting.lint_util import LintState
from debputy.lsp.diagnostics import DiagnosticData
from debputy.lsp.lsp_reference_keyword import (
    ALL_PUBLIC_NAMED_STYLES,
    Keyword,
    allowed_values,
    format_comp_item_synopsis_doc,
    UsageHint,
    LSP_DATA_DOMAIN,
)
from debputy.lsp.quickfixes import (
    propose_correct_text_quick_fix,
    propose_remove_line_quick_fix,
    propose_remove_range_quick_fix,
)
from debputy.lsp.ref_models.deb822_reference_parse_models import (
    Deb822ReferenceData,
    DEB822_REFERENCE_DATA_PARSER,
    FieldValueClass,
    StaticValue,
    Deb822Field,
)
from debputy.lsp.text_edit import apply_text_edits
from debputy.lsp.text_util import (
    normalize_dctrl_field_name,
    LintCapablePositionCodec,
    te_range_to_lsp,
    trim_end_of_line_whitespace,
)
from debputy.lsp.vendoring._deb822_repro.parsing import (
    Deb822KeyValuePairElement,
    LIST_SPACE_SEPARATED_INTERPRETATION,
    Deb822ParagraphElement,
    Deb822FileElement,
    Interpretation,
    parse_deb822_file,
    Deb822ParsedTokenList,
    Deb822ValueLineElement,
)
from debputy.lsp.vendoring._deb822_repro.tokens import (
    Deb822FieldNameToken,
)
from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback
from debputy.lsp.vendoring._deb822_repro.types import TE
from debputy.lsp.vendoring.wrap_and_sort import _sort_packages_key
from debputy.lsprotocol.types import (
    DiagnosticSeverity,
    Diagnostic,
    DiagnosticTag,
    Range,
    TextEdit,
    Position,
    CompletionItem,
    MarkupContent,
    CompletionItemTag,
    MarkupKind,
    CompletionItemKind,
    CompletionItemLabelDetails,
)
from debputy.manifest_parser.util import AttributePath
from debputy.path_matcher import BasenameGlobMatch
from debputy.plugin.api import VirtualPath
from debputy.util import PKGNAME_REGEX, _info, detect_possible_typo, _error
from debputy.yaml import MANIFEST_YAML

try:
    from debputy.lsp.vendoring._deb822_repro.locatable import (
        Position as TEPosition,
        Range as TERange,
        START_POSITION,
    )
except ImportError:
    pass


if TYPE_CHECKING:
    from debputy.lsp.maint_prefs import EffectiveFormattingPreference
    from debputy.lsp.debputy_ls import DebputyLanguageServer


F = TypeVar("F", bound="Deb822KnownField", covariant=True)
S = TypeVar("S", bound="StanzaMetadata")


_RE_SYNOPSIS_STARTS_WITH_ARTICLE = re.compile(r"^\s*(an?|the)(?:\s|$)", re.I)
_RE_SV = re.compile(r"(\d+[.]\d+[.]\d+)([.]\d+)?")
_RE_SYNOPSIS_IS_TEMPLATE = re.compile(
    r"^\s*(missing|<insert up to \d+ chars description>)$"
)
_RE_SYNOPSIS_IS_TOO_SHORT = re.compile(r"^\s*(\S+)$")
CURRENT_STANDARDS_VERSION = Version("4.7.0")


CustomFieldCheck = Callable[
    [
        "F",
        Deb822FileElement,
        Deb822KeyValuePairElement,
        "TERange",
        "TERange",
        Deb822ParagraphElement,
        "TEPosition",
        LintState,
    ],
    Iterable[Diagnostic],
]


@functools.lru_cache
def all_package_relationship_fields() -> Mapping[str, str]:
    # TODO: Pull from `dpkg-dev` when possible fallback only to the static list.
    return {
        f.lower(): f
        for f in (
            "Pre-Depends",
            "Depends",
            "Recommends",
            "Suggests",
            "Enhances",
            "Conflicts",
            "Breaks",
            "Replaces",
            "Provides",
            "Built-Using",
            "Static-Built-Using",
        )
    }


@functools.lru_cache
def all_source_relationship_fields() -> Mapping[str, str]:
    # TODO: Pull from `dpkg-dev` when possible fallback only to the static list.
    return {
        f.lower(): f
        for f in (
            "Build-Depends",
            "Build-Depends-Arch",
            "Build-Depends-Indep",
            "Build-Conflicts",
            "Build-Conflicts-Arch",
            "Build-Conflicts-Indep",
        )
    }


ALL_SECTIONS_WITHOUT_COMPONENT = frozenset(
    [
        "admin",
        "cli-mono",
        "comm",
        "database",
        "debian-installer",
        "debug",
        "devel",
        "doc",
        "editors",
        "education",
        "electronics",
        "embedded",
        "fonts",
        "games",
        "gnome",
        "gnu-r",
        "gnustep",
        "golang",
        "graphics",
        "hamradio",
        "haskell",
        "httpd",
        "interpreters",
        "introspection",
        "java",
        "javascript",
        "kde",
        "kernel",
        "libdevel",
        "libs",
        "lisp",
        "localization",
        "mail",
        "math",
        "metapackages",
        "misc",
        "net",
        "news",
        "ocaml",
        "oldlibs",
        "otherosfs",
        "perl",
        "php",
        "python",
        "ruby",
        "rust",
        "science",
        "shells",
        "sound",
        "tasks",
        "tex",
        "text",
        "utils",
        "vcs",
        "video",
        "virtual",
        "web",
        "x11",
        "xfce",
        "zope",
    ]
)

ALL_COMPONENTS = frozenset(
    [
        "main",
        "restricted",  # Ubuntu
        "non-free",
        "non-free-firmware",
        "contrib",
    ]
)


def _fields(*fields: F) -> Mapping[str, F]:
    return {normalize_dctrl_field_name(f.name.lower()): f for f in fields}


ALL_SECTIONS = allowed_values(
    *[
        s if c is None else f"{c}/{s}"
        for c, s in itertools.product(
            itertools.chain(cast("Iterable[Optional[str]]", [None]), ALL_COMPONENTS),
            ALL_SECTIONS_WITHOUT_COMPONENT,
        )
    ]
)

ALL_PRIORITIES = allowed_values(
    Keyword(
        "required",
        usage_hint="rare",
        synopsis="Package is Essential or an Essential package needs it (and is not a library)",
        long_description=textwrap.dedent(
            """\
            The package is necessary for the proper functioning of the system (read: dpkg needs it).

            Applicable if dpkg *needs* this package to function and it is not a library.

            No two packages that both have a priority of *standard* or higher may conflict with
            each other.
        """
        ),
    ),
    Keyword(
        "important",
        usage_hint="rare",
        synopsis="Bare minimum of commonly-expected and necessary tools",
        long_description=textwrap.dedent(
            """\
            The *important* packages are a bare minimum of commonly-expected and necessary tools.

            Applicable if 99% of all users in the distribution needs this package and it is not a library.

            No two packages that both have a priority of *standard* or higher may conflict with
            each other.
        """
        ),
    ),
    Keyword(
        "standard",
        usage_hint="rare",
        synopsis="If your distribution installer would install this by default (not for libraries)",
        long_description=textwrap.dedent(
            """\
            These packages provide a reasonable small but not too limited character-mode system.  This is
            what will be installed by default (by the debian-installer) if the user does not select anything
            else.  This does not include many large applications.

            Applicable if your distribution installer will install this package by default on a new system
            and it is not a library.

            No two packages that both have a priority of *standard* or higher may conflict with
            each other.
        """
        ),
    ),
    Keyword(
        "optional",
        synopsis="The default priority.",
        sort_text="aa-optional",
        long_description="This is the default priority and used by the majority of all packages"
        " in the Debian archive",
    ),
    Keyword(
        "extra",
        is_obsolete=True,
        replaced_by="optional",
        sort_text="zz-extra",
        synopsis="Obsolete alias of `optional`",
        long_description="Obsolete alias of `optional`.",
    ),
)


def all_architectures_and_wildcards(
    arch2table, *, allow_negations: bool = False
) -> Iterable[Union[str, Keyword]]:
    wildcards = set()
    yield Keyword(
        "any",
        is_exclusive=True,
        synopsis="Build once per machine architecture (native code, such as C/C++, interpreter to C bindings)",
        long_description=textwrap.dedent(
            """\
            The package is an architecture dependent package and need to be compiled for each and every
            architecture it.

            The name `any` refers to the fact that this is an architecture *wildcard* matching
            *any machine architecture* supported by dpkg.
        """
        ),
    )
    yield Keyword(
        "all",
        is_exclusive=True,
        synopsis="Independent of machine architecture (scripts, Java without JNI, data or documentation)",
        long_description=textwrap.dedent(
            """\
            The package is an architecture independent package.  This is typically fitting for packages containing
            only scripts, data or documentation.

            This name `all` refers to the fact that the package can be used for *all* architectures at the same.
            Though note that it is still subject to the rules of the `Multi-Arch` field.
        """
        ),
    )
    for arch_name, quad_tuple in arch2table.items():
        yield arch_name
        if allow_negations:
            yield f"!{arch_name}"
        cpu_wc = "any-" + quad_tuple.cpu_name
        os_wc = quad_tuple.os_name + "-any"
        if cpu_wc not in wildcards:
            yield cpu_wc
            if allow_negations:
                yield f"!{cpu_wc}"
            wildcards.add(cpu_wc)
        if os_wc not in wildcards:
            yield os_wc
            if allow_negations:
                yield f"!{os_wc}"
            wildcards.add(os_wc)
        # Add the remaining wildcards


@functools.lru_cache
def dpkg_arch_and_wildcards(*, allow_negations=False) -> FrozenSet[Union[str, Keyword]]:
    dpkg_arch_table = DpkgArchTable.load_arch_table()
    return frozenset(
        all_architectures_and_wildcards(
            dpkg_arch_table._arch2table,
            allow_negations=allow_negations,
        )
    )


def extract_first_value_and_position(
    kvpair: Deb822KeyValuePairElement,
    stanza_pos: "TEPosition",
    lint_state: LintState,
    *,
    interpretation: Interpretation[
        Deb822ParsedTokenList[Any, Any]
    ] = LIST_SPACE_SEPARATED_INTERPRETATION,
) -> Tuple[Optional[str], Optional[Range]]:
    kvpair_pos = kvpair.position_in_parent().relative_to(stanza_pos)
    value_element_pos = kvpair.value_element.position_in_parent().relative_to(
        kvpair_pos
    )
    for value_ref in kvpair.interpret_as(interpretation).iter_value_references():
        v = value_ref.value
        section_value_loc = value_ref.locatable
        value_range_te = section_value_loc.range_in_parent().relative_to(
            value_element_pos
        )
        section_range_server_units = te_range_to_lsp(value_range_te)
        section_range = lint_state.position_codec.range_to_client_units(
            lint_state.lines,
            section_range_server_units,
        )
        return v, section_range
    return None, None


def _sv_field_validation(
    _known_field: "F",
    _deb822_file: Deb822FileElement,
    kvpair: Deb822KeyValuePairElement,
    _kvpair_range: "TERange",
    _field_name_range_te: "TERange",
    _stanza: Deb822ParagraphElement,
    stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    sv_value, sv_value_range = extract_first_value_and_position(
        kvpair,
        stanza_position,
        lint_state,
    )
    m = _RE_SV.fullmatch(sv_value)
    if m is None:
        yield Diagnostic(
            sv_value_range,
            f'Not a valid standards version. Current version is "{CURRENT_STANDARDS_VERSION}"',
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )
        return

    sv_version = Version(sv_value)
    if sv_version < CURRENT_STANDARDS_VERSION:
        yield Diagnostic(
            sv_value_range,
            f"Latest Standards-Version is {CURRENT_STANDARDS_VERSION}",
            severity=DiagnosticSeverity.Information,
            source="debputy",
        )
        return
    extra = m.group(2)
    if extra:
        extra_len = lint_state.position_codec.client_num_units(extra)
        yield Diagnostic(
            Range(
                Position(
                    sv_value_range.end.line,
                    sv_value_range.end.character - extra_len,
                ),
                sv_value_range.end,
            ),
            "Unnecessary version segment. This part of the version is only used for editorial changes",
            severity=DiagnosticSeverity.Information,
            source="debputy",
            data=DiagnosticData(
                quickfixes=[
                    propose_remove_range_quick_fix(
                        proposed_title="Remove unnecessary version part"
                    )
                ]
            ),
        )


def _dctrl_ma_field_validation(
    _known_field: "F",
    _deb822_file: Deb822FileElement,
    _kvpair: Deb822KeyValuePairElement,
    _kvpair_range: "TERange",
    _field_name_range: "TERange",
    stanza: Deb822ParagraphElement,
    stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    ma_kvpair = stanza.get_kvpair_element(("Multi-Arch", 0), use_get=True)
    arch = stanza.get("Architecture", "any")
    if arch == "all" and ma_kvpair is not None:
        ma_value, ma_value_range = extract_first_value_and_position(
            ma_kvpair,
            stanza_position,
            lint_state,
        )
        if ma_value == "same":
            yield Diagnostic(
                ma_value_range,
                "Multi-Arch: same is not valid for Architecture: all packages. Maybe you want foreign?",
                severity=DiagnosticSeverity.Error,
                source="debputy",
            )


def _udeb_only_field_validation(
    known_field: "F",
    _deb822_file: Deb822FileElement,
    _kvpair: Deb822KeyValuePairElement,
    _kvpair_range: "TERange",
    field_name_range: "TERange",
    stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    package_type = stanza.get("Package-Type")
    if package_type != "udeb":
        field_range_server_units = te_range_to_lsp(field_name_range)
        field_range = lint_state.position_codec.range_to_client_units(
            lint_state.lines,
            field_range_server_units,
        )
        yield Diagnostic(
            field_range,
            f"The {known_field.name} field is only applicable to udeb packages (`Package-Type: udeb`)",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )


def _complete_only_in_arch_dep_pkgs(
    stanza_parts: Iterable[Deb822ParagraphElement],
) -> bool:
    for stanza in stanza_parts:
        arch = stanza.get("Architecture")
        if arch is None:
            continue
        archs = arch.split()
        return "all" not in archs
    return False


def _complete_only_for_udeb_pkgs(
    stanza_parts: Iterable[Deb822ParagraphElement],
) -> bool:
    for stanza in stanza_parts:
        for option in ("Package-Type", "XC-Package-Type"):
            pkg_type = stanza.get(option)
            if pkg_type is not None:
                return pkg_type == "udeb"
    return False


def _arch_not_all_only_field_validation(
    known_field: "F",
    _deb822_file: Deb822FileElement,
    _kvpair: Deb822KeyValuePairElement,
    _kvpair_range_te: "TERange",
    field_name_range_te: "TERange",
    stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    architecture = stanza.get("Architecture")
    if architecture == "all":
        field_range_server_units = te_range_to_lsp(field_name_range_te)
        field_range = lint_state.position_codec.range_to_client_units(
            lint_state.lines,
            field_range_server_units,
        )
        yield Diagnostic(
            field_range,
            f"The {known_field.name} field is not applicable to arch:all packages (`Architecture: all`)",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )


def _single_line_span_to_client_range(
    span: Tuple[int, int],
    relative_to: "TEPosition",
    lint_state: LintState,
) -> Range:
    range_server_units = Range(
        Position(
            relative_to.line_position,
            relative_to.cursor_position + span[0],
        ),
        Position(
            relative_to.line_position,
            relative_to.cursor_position + span[1],
        ),
    )
    return lint_state.position_codec.range_to_client_units(
        lint_state.lines,
        range_server_units,
    )


def _check_extended_description_line(
    description_value_line: Deb822ValueLineElement,
    description_line_range_te: "TERange",
    package: Optional[str],
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    if description_value_line.comment_element is not None:
        # TODO: Fix this limitation (we get the content and the range wrong with comments.
        # They are rare inside a Description, so this is a 80:20 trade off
        return
    description_line_with_leading_space = (
        description_value_line.convert_to_text().rstrip()
    )
    try:
        idx = description_line_with_leading_space.index(
            "<insert long description, indented with spaces>"
        )
    except ValueError:
        pass
    else:
        template_span = idx, idx + len(
            "<insert long description, indented with spaces>"
        )
        yield Diagnostic(
            _single_line_span_to_client_range(
                template_span,
                description_line_range_te.start_pos,
                lint_state,
            ),
            "Unfilled or left-over template from dh_make",
            severity=DiagnosticSeverity.Error,
            source="debputy",
        )
    if len(description_line_with_leading_space) >= 80:
        # Policy says nothing here, but lintian has 80 characters as hard limit and that
        # probably matches the limitation of package manager UIs (TUIs/GUIs) somewhere.
        span = 79, len(description_line_with_leading_space)
        yield Diagnostic(
            _single_line_span_to_client_range(
                span,
                description_line_range_te.start_pos,
                lint_state,
            ),
            "Package description line is too long; please line wrap it.",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )


def _check_synopsis(
    synopsis_value_line: Deb822ValueLineElement,
    synopsis_range_te: "TERange",
    field_name_range_te: "TERange",
    package: Optional[str],
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    # This function would compute range would be wrong if there is a comment
    assert synopsis_value_line.comment_element is None
    synopsis_text_with_leading_space = synopsis_value_line.convert_to_text().rstrip()
    if not synopsis_text_with_leading_space:
        yield Diagnostic(
            lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                te_range_to_lsp(field_name_range_te),
            ),
            "Package synopsis is missing",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )
        return
    synopsis_text_trimmed = synopsis_text_with_leading_space.lstrip()
    synopsis_offset = len(synopsis_text_with_leading_space) - len(synopsis_text_trimmed)
    starts_with_article = _RE_SYNOPSIS_STARTS_WITH_ARTICLE.search(
        synopsis_text_with_leading_space
    )
    # TODO: Handle ${...} expansion
    if starts_with_article:
        yield Diagnostic(
            _single_line_span_to_client_range(
                starts_with_article.span(1),
                synopsis_range_te.start_pos,
                lint_state,
            ),
            "Package synopsis starts with an article (a/an/the).",
            severity=DiagnosticSeverity.Warning,
            source="DevRef 6.2.2",
        )
    if len(synopsis_text_trimmed) >= 80:
        # Policy says `certainly under 80 characters.`, so exactly 80 characters is considered bad too.
        span = synopsis_offset + 79, len(synopsis_text_with_leading_space)
        yield Diagnostic(
            _single_line_span_to_client_range(
                span,
                synopsis_range_te.start_pos,
                lint_state,
            ),
            "Package synopsis is too long.",
            severity=DiagnosticSeverity.Warning,
            source="Policy 3.4.1",
        )
    if template_match := _RE_SYNOPSIS_IS_TEMPLATE.match(
        synopsis_text_with_leading_space
    ):
        yield Diagnostic(
            _single_line_span_to_client_range(
                template_match.span(1),
                synopsis_range_te.start_pos,
                lint_state,
            ),
            "Package synopsis is a placeholder",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )
    elif too_short_match := _RE_SYNOPSIS_IS_TOO_SHORT.match(
        synopsis_text_with_leading_space
    ):
        yield Diagnostic(
            _single_line_span_to_client_range(
                too_short_match.span(1),
                synopsis_range_te.start_pos,
                lint_state,
            ),
            "Package synopsis is too short",
            severity=DiagnosticSeverity.Warning,
            source="debputy",
        )


def dctrl_description_validator(
    _known_field: "F",
    _deb822_file: Deb822FileElement,
    kvpair: Deb822KeyValuePairElement,
    kvpair_range_te: "TERange",
    _field_name_range: "TERange",
    stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    value_lines = kvpair.value_element.value_lines
    if not value_lines:
        return
    package = stanza.get("Package")
    synopsis_value_line = value_lines[0]
    value_range_te = kvpair.value_element.range_in_parent().relative_to(
        kvpair_range_te.start_pos
    )
    synopsis_line_range_te = synopsis_value_line.range_in_parent().relative_to(
        value_range_te.start_pos
    )
    if synopsis_value_line.continuation_line_token is None:
        field_name_range_te = kvpair.field_token.range_in_parent().relative_to(
            kvpair_range_te.start_pos
        )
        yield from _check_synopsis(
            synopsis_value_line,
            synopsis_line_range_te,
            field_name_range_te,
            package,
            lint_state,
        )
        description_lines = value_lines[1:]
    else:
        description_lines = value_lines
    for description_line in description_lines:
        description_line_range_te = description_line.range_in_parent().relative_to(
            value_range_te.start_pos
        )
        yield from _check_extended_description_line(
            description_line,
            description_line_range_te,
            package,
            lint_state,
        )


def _has_packaging_expected_file(
    name: str,
    msg: str,
    severity: DiagnosticSeverity = DiagnosticSeverity.Error,
) -> CustomFieldCheck:

    def _impl(
        _known_field: "F",
        _deb822_file: Deb822FileElement,
        _kvpair: Deb822KeyValuePairElement,
        kvpair_range_te: "TERange",
        _field_name_range_te: "TERange",
        _stanza: Deb822ParagraphElement,
        _stanza_position: "TEPosition",
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        debian_dir = lint_state.debian_dir
        if debian_dir is None:
            return
        cpy = debian_dir.lookup(name)
        if not cpy:
            field_range = lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                te_range_to_lsp(kvpair_range_te),
            )
            yield Diagnostic(
                field_range,
                msg,
                severity=severity,
                source="debputy",
                data=DiagnosticData(
                    report_for_related_file=f"debian/{name}",
                ),
            )

    return _impl


_check_missing_debian_rules = _has_packaging_expected_file(
    "rules",
    'Missing debian/rules when the Build-Driver is unset or set to "debian-rules"',
)


def _has_build_instructions(
    known_field: "F",
    deb822_file: Deb822FileElement,
    kvpair: Deb822KeyValuePairElement,
    kvpair_range_te: "TERange",
    field_name_range_te: "TERange",
    stanza: Deb822ParagraphElement,
    stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    if stanza.get("Build-Driver", "debian-rules").lower() != "debian-rules":
        return ()

    return _check_missing_debian_rules(
        known_field,
        deb822_file,
        kvpair,
        kvpair_range_te,
        field_name_range_te,
        stanza,
        stanza_position,
        lint_state,
    )


def _each_value_match_regex_validation(
    regex: re.Pattern,
    *,
    diagnostic_severity: DiagnosticSeverity = DiagnosticSeverity.Error,
) -> CustomFieldCheck:

    def _validator(
        _known_field: "F",
        _deb822_file: Deb822FileElement,
        kvpair: Deb822KeyValuePairElement,
        kvpair_range_te: "TERange",
        _field_name_range_te: "TERange",
        _stanza: Deb822ParagraphElement,
        _stanza_position: "TEPosition",
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:

        value_element_pos = kvpair.value_element.position_in_parent().relative_to(
            kvpair_range_te.start_pos
        )
        for value_ref in kvpair.interpret_as(
            LIST_SPACE_SEPARATED_INTERPRETATION
        ).iter_value_references():
            v = value_ref.value
            m = regex.fullmatch(v)
            if m is not None:
                continue

            if "${" in v:
                # Ignore substvars
                continue

            section_value_loc = value_ref.locatable
            value_range_te = section_value_loc.range_in_parent().relative_to(
                value_element_pos
            )
            value_range_server_units = te_range_to_lsp(value_range_te)
            value_range = lint_state.position_codec.range_to_client_units(
                lint_state.lines, value_range_server_units
            )
            yield Diagnostic(
                value_range,
                f'The value "{v}" does not match the regex {regex.pattern}.',
                severity=diagnostic_severity,
                source="debputy",
            )

    return _validator


_DEP_OR_RELATION = re.compile(r"[|]")
_DEP_RELATION_CLAUSE = re.compile(
    r"""
    ^
    \s*
    (?P<name_arch_qual>[-+.a-zA-Z0-9${}:]{2,})
    \s*
    (?: [(] \s* (?P<operator>>>|>=|=|<=|<<) \s* (?P<version> [^)]+) \s* [)] \s* )?
    (?: \[ (?P<arch_restriction> [\s!\w\-]+) ] \s*)?
    (?: < (?P<build_profile_restriction> .+ ) > \s*)?
    ((?P<garbage>\S.*)\s*)?
    $
""",
    re.VERBOSE | re.MULTILINE,
)


def _span_to_te_range(
    text: str,
    start_pos: int,
    end_pos: int,
) -> TERange:
    prefix = text[0:start_pos]
    prefix_plus_text = text[0:end_pos]

    start_line = prefix.count("\n")
    if start_line:
        start_newline_offset = prefix.rindex("\n")
        # +1 to skip past the newline
        start_cursor_pos = start_pos - (start_newline_offset + 1)
    else:
        start_cursor_pos = start_pos

    end_line = prefix_plus_text.count("\n")
    if end_line == start_line:
        end_cursor_pos = start_cursor_pos + (end_pos - start_pos)
    else:
        end_newline_offset = prefix_plus_text.rindex("\n")
        end_cursor_pos = end_pos - (end_newline_offset + 1)

    return TERange(
        TEPosition(
            start_line,
            start_cursor_pos,
        ),
        TEPosition(
            end_line,
            end_cursor_pos,
        ),
    )


def _split_w_spans(
    v: str,
    sep: str,
    *,
    offset: int = 0,
) -> Sequence[Tuple[str, int, int]]:
    separator_size = len(sep)
    parts = v.split(sep)
    for part in parts:
        size = len(part)
        end_offset = offset + size
        yield part, offset, end_offset
        offset = end_offset + separator_size


_COLLAPSE_WHITESPACE = re.compile(r"\s+")


def _cleanup_rel(rel: str) -> str:
    return _COLLAPSE_WHITESPACE.sub(" ", rel.strip())


def _text_to_te_position(text: str) -> "TEPosition":
    newlines = text.count("\n")
    if not newlines:
        return TEPosition(
            newlines,
            len(text),
        )
    last_newline_offset = text.rindex("\n")
    line_offset = len(text) - (last_newline_offset + 1)
    return TEPosition(
        newlines,
        line_offset,
    )


def _dctrl_validate_dep(
    known_field: "F",
    _deb822_file: Deb822FileElement,
    kvpair: Deb822KeyValuePairElement,
    kvpair_range_te: "TERange",
    _field_name_range: "TERange",
    _stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    value_element_pos = kvpair.value_element.position_in_parent().relative_to(
        kvpair_range_te.start_pos
    )
    raw_value_with_comments = kvpair.value_element.convert_to_text()
    raw_value_masked_comments = "".join(
        (line if not line.startswith("#") else (" " * (len(line) - 1)) + "\n")
        for line in raw_value_with_comments.splitlines(keepends=True)
    )
    if isinstance(known_field, DctrlRelationshipKnownField):
        version_operators = known_field.allowed_version_operators
        supports_or_relation = known_field.supports_or_relation
    else:
        version_operators = frozenset()
        supports_or_relation = True

    for rel, rel_offset, rel_end_offset in _split_w_spans(
        raw_value_masked_comments, ","
    ):
        seen_relation = False
        for or_rel, offset, end_offset in _split_w_spans(rel, "|", offset=rel_offset):
            if or_rel.isspace():
                continue
            if seen_relation and not supports_or_relation:
                separator_range_te = TERange(
                    _text_to_te_position(raw_value_masked_comments[: offset - 1]),
                    _text_to_te_position(raw_value_masked_comments[:offset]),
                ).relative_to(value_element_pos)
                separator_range = lint_state.position_codec.range_to_client_units(
                    lint_state.lines,
                    te_range_to_lsp(separator_range_te),
                )
                yield Diagnostic(
                    lint_state.position_codec.range_to_client_units(
                        lint_state.lines,
                        separator_range,
                    ),
                    f'The field {known_field.name} does not support "|" (OR) in relations.',
                    DiagnosticSeverity.Error,
                    source="debputy",
                )
            seen_relation = True
            m = _DEP_RELATION_CLAUSE.fullmatch(or_rel)

            if m is not None:
                garbage = m.group("garbage")
                version_operator = m.group("operator")
                if (
                    version_operators
                    and version_operator is not None
                    and version_operator not in version_operators
                ):
                    operator_span = m.span("operator")
                    v_start_offset = offset + operator_span[0]
                    v_end_offset = offset + operator_span[1]
                    version_problem_range_te = TERange(
                        _text_to_te_position(
                            raw_value_masked_comments[:v_start_offset]
                        ),
                        _text_to_te_position(raw_value_masked_comments[:v_end_offset]),
                    ).relative_to(value_element_pos)

                    version_problem_range = (
                        lint_state.position_codec.range_to_client_units(
                            lint_state.lines,
                            te_range_to_lsp(version_problem_range_te),
                        )
                    )
                    sorted_version_operators = sorted(version_operators)
                    yield Diagnostic(
                        lint_state.position_codec.range_to_client_units(
                            lint_state.lines,
                            version_problem_range,
                        ),
                        f'The version operator "{version_operator}" is not allowed in {known_field.name}',
                        DiagnosticSeverity.Error,
                        source="debputy",
                        data=DiagnosticData(
                            quickfixes=[
                                propose_correct_text_quick_fix(n)
                                for n in sorted_version_operators
                            ]
                        ),
                    )
            else:
                garbage = None

            if m is not None and not garbage:
                continue
            if m is not None:
                garbage_span = m.span("garbage")
                garbage_start, garbage_end = garbage_span
                error_start_offset = offset + garbage_start
                error_end_offset = offset + garbage_end
                garbage_part = raw_value_masked_comments[
                    error_start_offset:error_end_offset
                ]
            else:
                garbage_part = None
                error_start_offset = offset
                error_end_offset = end_offset

            problem_range_te = TERange(
                _text_to_te_position(raw_value_masked_comments[:error_start_offset]),
                _text_to_te_position(raw_value_masked_comments[:error_end_offset]),
            ).relative_to(value_element_pos)

            problem_range = lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                te_range_to_lsp(problem_range_te),
            )
            if garbage_part is not None:
                if _DEP_RELATION_CLAUSE.fullmatch(garbage_part) is not None:
                    msg = (
                        "Trailing data after a relationship that might be a second relationship."
                        " Is a separator missing before this part?"
                    )
                else:
                    msg = "Parse error of the relationship. Either a syntax error or a missing separator somewhere."
                yield Diagnostic(
                    lint_state.position_codec.range_to_client_units(
                        lint_state.lines,
                        problem_range,
                    ),
                    msg,
                    DiagnosticSeverity.Error,
                    source="debputy",
                )
            else:
                dep = _cleanup_rel(
                    raw_value_masked_comments[error_start_offset:error_end_offset]
                )
                yield Diagnostic(
                    lint_state.position_codec.range_to_client_units(
                        lint_state.lines,
                        problem_range,
                    ),
                    f'Could not parse "{dep}" as a dependency relation.',
                    DiagnosticSeverity.Error,
                    source="debputy",
                )


def _rrr_build_driver_mismatch(
    _known_field: "F",
    _deb822_file: Deb822FileElement,
    _kvpair: Deb822KeyValuePairElement,
    kvpair_range_te: "TERange",
    _field_name_range: "TERange",
    stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    dr = stanza.get("Build-Driver", "debian-rules")
    if dr != "debian-rules":
        yield Diagnostic(
            lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                te_range_to_lsp(kvpair_range_te),
            ),
            f"The Rules-Requires-Root field is irrelevant for the `Build-Driver` `{dr}`.",
            DiagnosticSeverity.Warning,
            source="debputy",
            data=DiagnosticData(
                quickfixes=[
                    propose_remove_range_quick_fix(
                        proposed_title="Remove Rules-Requires-Root"
                    )
                ]
            ),
        )


class Dep5Matcher(BasenameGlobMatch):
    def __init__(self, basename_glob: str) -> None:
        super().__init__(
            basename_glob,
            only_when_in_directory=None,
            path_type=None,
            recursive_match=False,
        )


def _match_dep5_segment(
    current_dir: VirtualPathBase, basename_glob: str
) -> Iterable[VirtualPathBase]:
    if "*" in basename_glob or "?" in basename_glob:
        return Dep5Matcher(basename_glob).finditer(current_dir)
    else:
        res = current_dir.get(basename_glob)
        if res is None:
            return tuple()
        return (res,)


_RE_SLASHES = re.compile(r"//+")


def _dep5_unnecessary_symbols(
    value: str,
    value_range: TERange,
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    slash_check_index = 0
    if value.startswith(("./", "/")):
        prefix_len = 1 if value[0] == "/" else 2
        if value[prefix_len - 1 : prefix_len + 2].startswith("//"):
            _, slashes_end = _RE_SLASHES.search(value).span()
            prefix_len = slashes_end

        slash_check_index = prefix_len
        prefix_range = te_range_to_lsp(
            TERange(
                value_range.start_pos,
                TEPosition(
                    value_range.start_pos.line_position,
                    value_range.start_pos.cursor_position + prefix_len,
                ),
            )
        )
        yield Diagnostic(
            lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                prefix_range,
            ),
            f'Unnecessary prefix "{value[0:prefix_len]}"',
            DiagnosticSeverity.Warning,
            source="debputy",
            data=DiagnosticData(
                quickfixes=[
                    propose_remove_range_quick_fix(
                        proposed_title=f'Delete "{value[0:prefix_len]}"'
                    )
                ]
            ),
        )

    for m in _RE_SLASHES.finditer(value, slash_check_index):
        m_start, m_end = m.span(0)

        prefix_range = te_range_to_lsp(
            TERange(
                TEPosition(
                    value_range.start_pos.line_position,
                    value_range.start_pos.cursor_position + m_start,
                ),
                TEPosition(
                    value_range.start_pos.line_position,
                    value_range.start_pos.cursor_position + m_end,
                ),
            )
        )
        yield Diagnostic(
            lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                prefix_range,
            ),
            'Simplify to a single "/"',
            DiagnosticSeverity.Warning,
            source="debputy",
            data=DiagnosticData(quickfixes=[propose_correct_text_quick_fix("/")]),
        )


def _dep5_files_check(
    known_field: "F",
    _deb822_file: Deb822FileElement,
    kvpair: Deb822KeyValuePairElement,
    kvpair_range_te: "TERange",
    _field_name_range: "TERange",
    _stanza: Deb822ParagraphElement,
    _stanza_position: "TEPosition",
    lint_state: LintState,
) -> Iterable[Diagnostic]:
    interpreter = known_field.field_value_class.interpreter()
    assert interpreter is not None
    full_value_range = kvpair.value_element.range_in_parent().relative_to(
        kvpair_range_te.start_pos
    )
    values_with_ranges = []
    for value_ref in kvpair.interpret_as(interpreter).iter_value_references():
        value_range = value_ref.locatable.range_in_parent().relative_to(
            full_value_range.start_pos
        )
        value = value_ref.value
        values_with_ranges.append((value_ref.value, value_range))
        yield from _dep5_unnecessary_symbols(value, value_range, lint_state)

    source_root = lint_state.source_root
    if source_root is None:
        return
    i = 0
    limit = len(values_with_ranges)
    while i < limit:
        value, value_range = values_with_ranges[i]
        i += 1


def _combined_custom_field_check(*checks: CustomFieldCheck) -> CustomFieldCheck:
    def _validator(
        known_field: "F",
        deb822_file: Deb822FileElement,
        kvpair: Deb822KeyValuePairElement,
        kvpair_range_te: "TERange",
        field_name_range_te: "TERange",
        stanza: Deb822ParagraphElement,
        stanza_position: "TEPosition",
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        for check in checks:
            yield from check(
                known_field,
                deb822_file,
                kvpair,
                kvpair_range_te,
                field_name_range_te,
                stanza,
                stanza_position,
                lint_state,
            )

    return _validator


@dataclasses.dataclass(slots=True, frozen=True)
class PackageNameSectionRule:
    section: str
    check: Callable[[str], bool]


def _package_name_section_rule(
    section: str,
    check: Union[Callable[[str], bool], re.Pattern],
    *,
    confirm_re: Optional[re.Pattern] = None,
) -> PackageNameSectionRule:
    if confirm_re is not None:
        assert callable(check)

        def _impl(v: str) -> bool:
            return check(v) and confirm_re.search(v)

    elif isinstance(check, re.Pattern):

        def _impl(v: str) -> bool:
            return check.search(v) is not None

    else:
        _impl = check

    return PackageNameSectionRule(section, _impl)


# rules: order is important (first match wins in case of a conflict)
_PKGNAME_VS_SECTION_RULES = [
    _package_name_section_rule("debian-installer", lambda n: n.endswith("-udeb")),
    _package_name_section_rule("doc", lambda n: n.endswith(("-doc", "-docs"))),
    _package_name_section_rule("debug", lambda n: n.endswith(("-dbg", "-dbgsym"))),
    _package_name_section_rule(
        "httpd",
        lambda n: n.startswith(("lighttpd-mod", "libapache2-mod-", "libnginx-mod-")),
    ),
    _package_name_section_rule("gnustep", lambda n: n.startswith("gnustep-")),
    _package_name_section_rule(
        "gnustep",
        lambda n: n.endswith(
            (
                ".framework",
                ".framework-common",
                ".tool",
                ".tool-common",
                ".app",
                ".app-common",
            )
        ),
    ),
    _package_name_section_rule("embedded", lambda n: n.startswith("moblin-")),
    _package_name_section_rule("javascript", lambda n: n.startswith("node-")),
    _package_name_section_rule(
        "zope",
        lambda n: n.startswith(("python-zope", "python3-zope", "zope")),
    ),
    _package_name_section_rule(
        "python",
        lambda n: n.startswith(("python-", "python3-")),
    ),
    _package_name_section_rule(
        "gnu-r",
        lambda n: n.startswith(("r-cran-", "r-bioc-", "r-other-")),
    ),
    _package_name_section_rule("editors", lambda n: n.startswith("elpa-")),
    _package_name_section_rule("lisp", lambda n: n.startswith("cl-")),
    _package_name_section_rule(
        "lisp",
        lambda n: "-elisp-" in n or n.endswith("-elisp"),
    ),
    _package_name_section_rule(
        "lisp",
        lambda n: n.startswith("lib") and n.endswith("-guile"),
    ),
    _package_name_section_rule("lisp", lambda n: n.startswith("guile-")),
    _package_name_section_rule("golang", lambda n: n.startswith("golang-")),
    _package_name_section_rule(
        "perl",
        lambda n: n.startswith("lib") and n.endswith("-perl"),
    ),
    _package_name_section_rule(
        "cli-mono",
        lambda n: n.startswith("lib") and n.endswith(("-cil", "-cil-dev")),
    ),
    _package_name_section_rule(
        "java",
        lambda n: n.startswith("lib") and n.endswith(("-java", "-gcj", "-jni")),
    ),
    _package_name_section_rule(
        "php",
        lambda n: n.startswith(("libphp", "php")),
        confirm_re=re.compile(r"^(?:lib)?php(?:\d(?:\.\d)?)?-"),
    ),
    _package_name_section_rule(
        "php", lambda n: n.startswith("lib-") and n.endswith("-php")
    ),
    _package_name_section_rule(
        "haskell",
        lambda n: n.startswith(("haskell-", "libhugs-", "libghc-", "libghc6-")),
    ),
    _package_name_section_rule(
        "ruby",
        lambda n: "-ruby" in n,
        confirm_re=re.compile(r"^lib.*-ruby(?:1\.\d)?$"),
    ),
    _package_name_section_rule("ruby", lambda n: n.startswith("ruby-")),
    _package_name_section_rule(
        "rust",
        lambda n: n.startswith("librust-") and n.endswith("-dev"),
    ),
    _package_name_section_rule("rust", lambda n: n.startswith("rust-")),
    _package_name_section_rule(
        "ocaml",
        lambda n: n.startswith("lib-") and n.endswith(("-ocaml-dev", "-camlp4-dev")),
    ),
    _package_name_section_rule("javascript", lambda n: n.startswith("libjs-")),
    _package_name_section_rule(
        "interpreters",
        lambda n: n.startswith("lib-") and n.endswith(("-tcl", "-lua", "-gst")),
    ),
    _package_name_section_rule(
        "introspection",
        lambda n: n.startswith("gir-"),
        confirm_re=re.compile(r"^gir\d+\.\d+-.*-\d+\.\d+$"),
    ),
    _package_name_section_rule(
        "fonts",
        lambda n: n.startswith(("xfonts-", "fonts-", "ttf-")),
    ),
    _package_name_section_rule("admin", lambda n: n.startswith(("libnss-", "libpam-"))),
    _package_name_section_rule(
        "localization",
        lambda n: n.startswith(
            (
                "aspell-",
                "hunspell-",
                "myspell-",
                "mythes-",
                "dict-freedict-",
                "gcompris-sound-",
            )
        ),
    ),
    _package_name_section_rule(
        "localization",
        lambda n: n.startswith("hyphen-"),
        confirm_re=re.compile(r"^hyphen-[a-z]{2}(?:-[a-z]{2})?$"),
    ),
    _package_name_section_rule(
        "localization",
        lambda n: "-l10n-" in n or n.endswith("-l10n"),
    ),
    _package_name_section_rule("kernel", lambda n: n.endswith(("-dkms", "-firmware"))),
    _package_name_section_rule(
        "libdevel",
        lambda n: n.startswith("lib") and n.endswith(("-dev", "-headers")),
    ),
    _package_name_section_rule(
        "libs",
        lambda n: n.startswith("lib"),
        confirm_re=re.compile(r"^lib.*\d[ad]?$"),
    ),
]


# Fiddling with the package name can cause a lot of changes (diagnostic scans), so we have an upper bound
# on the cache. The number is currently just taken out of a hat.
@functools.lru_cache(64)
def package_name_to_section(name: str) -> Optional[str]:
    for rule in _PKGNAME_VS_SECTION_RULES:
        if rule.check(name):
            return rule.section
    return None


def _unknown_value_check(
    field_name: str,
    value: str,
    known_values: Mapping[str, Keyword],
    unknown_value_severity: Optional[DiagnosticSeverity],
) -> Tuple[
    Optional[Keyword], Optional[str], Optional[DiagnosticSeverity], Optional[Any]
]:
    known_value = known_values.get(value)
    message = None
    severity = unknown_value_severity
    fix_data = None
    if known_value is None:
        candidates = detect_possible_typo(
            value,
            known_values,
        )
        if len(known_values) < 5:
            values = ", ".join(sorted(known_values))
            hint_text = f" Known values for this field: {values}"
        else:
            hint_text = ""
        fix_data = None
        severity = unknown_value_severity
        fix_text = hint_text
        if candidates:
            match = candidates[0]
            if len(candidates) == 1:
                known_value = known_values[match]
            fix_text = (
                f' It is possible that the value is a typo of "{match}".{fix_text}'
            )
            fix_data = [propose_correct_text_quick_fix(m) for m in candidates]
        elif severity is None:
            return None, None, None, None
        if severity is None:
            severity = DiagnosticSeverity.Warning
            # It always has leading whitespace
            message = fix_text.strip()
        else:
            message = f'The value "{value}" is not supported in {field_name}.{fix_text}'
    return known_value, message, severity, fix_data


def _dep5_escape_path(path: str) -> str:
    return path.replace(" ", "?")


def _noop_escape_path(path: str) -> str:
    return path


def _should_ignore_dir(
    path: VirtualPath,
    *,
    supports_dir_match: bool = False,
    match_non_persistent_paths: bool = False,
) -> bool:
    if not supports_dir_match and not any(path.iterdir):
        return True
    cachedir_tag = path.get("CACHEDIR.TAG")
    if (
        not match_non_persistent_paths
        and cachedir_tag is not None
        and cachedir_tag.is_file
    ):
        # https://bford.info/cachedir/
        with cachedir_tag.open(byte_io=True, buffering=64) as fd:
            start = fd.read(43)
        if start == b"Signature: 8a477f597d28d172789f06886806bc55":
            return True
    return False


@dataclasses.dataclass(slots=True)
class Deb822KnownField:
    name: str
    field_value_class: FieldValueClass
    warn_if_default: bool = True
    replaced_by: Optional[str] = None
    deprecated_with_no_replacement: bool = False
    missing_field_severity: Optional[DiagnosticSeverity] = None
    default_value: Optional[str] = None
    known_values: Optional[Mapping[str, Keyword]] = None
    unknown_value_diagnostic_severity: Optional[DiagnosticSeverity] = (
        DiagnosticSeverity.Error
    )
    translation_context: str = ""
    # One-line description for space-constrained docs (such as completion docs)
    synopsis: Optional[str] = None
    usage_hint: Optional[UsageHint] = None
    long_description: Optional[str] = None
    spellcheck_value: bool = False
    custom_field_check: Optional[CustomFieldCheck] = None
    can_complete_field_in_stanza: Optional[
        Callable[[Iterable[Deb822ParagraphElement]], bool]
    ] = None

    def synopsis_translated(
        self, translation_provider: Union["DebputyLanguageServer", "LintState"]
    ) -> str:
        return translation_provider.translation(LSP_DATA_DOMAIN).pgettext(
            self.translation_context,
            self.synopsis,
        )

    def long_description_translated(
        self, translation_provider: Union["DebputyLanguageServer", "LintState"]
    ) -> str:
        return translation_provider.translation(LSP_DATA_DOMAIN).pgettext(
            self.translation_context,
            self.long_description,
        )

    def _can_complete_field_in_stanza(
        self,
        stanza_parts: Sequence[Deb822ParagraphElement],
    ) -> bool:
        return (
            self.can_complete_field_in_stanza is None
            or self.can_complete_field_in_stanza(stanza_parts)
        )

    def complete_field(
        self,
        lint_state: LintState,
        stanza_parts: Sequence[Deb822ParagraphElement],
        markdown_kind: MarkupKind,
    ) -> Optional[CompletionItem]:
        if not self._can_complete_field_in_stanza(stanza_parts):
            return None
        name = self.name
        complete_as = name + ": "
        options = self.value_options_for_completer(
            lint_state,
            stanza_parts,
            "",
            markdown_kind,
            is_completion_for_field=True,
        )
        if options is not None and len(options) == 1:
            value = options[0].insert_text
            if value is not None:
                complete_as += value
        tags = []
        is_deprecated = False
        if self.replaced_by or self.deprecated_with_no_replacement:
            is_deprecated = True
            tags.append(CompletionItemTag.Deprecated)

        doc = self.long_description
        if doc:
            doc = MarkupContent(
                value=doc,
                kind=markdown_kind,
            )
        else:
            doc = None

        return CompletionItem(
            name,
            insert_text=complete_as,
            deprecated=is_deprecated,
            tags=tags,
            detail=format_comp_item_synopsis_doc(
                self.usage_hint,
                self.synopsis_translated(lint_state),
                is_deprecated,
            ),
            documentation=doc,
        )

    def _complete_files(
        self,
        base_dir: Optional[VirtualPathBase],
        value_being_completed: str,
        *,
        is_dep5_file_list: bool = False,
        supports_dir_match: bool = False,
        supports_spaces_in_filename: bool = False,
        match_non_persistent_paths: bool = False,
    ) -> Optional[Sequence[CompletionItem]]:
        _info(f"_complete_files: {base_dir.fs_path} - {value_being_completed!r}")
        if base_dir is None or not base_dir.is_dir:
            return None

        if is_dep5_file_list:
            supports_spaces_in_filename = True
            supports_dir_match = False
            match_non_persistent_paths = False

        if value_being_completed == "":
            current_dir = base_dir
            unmatched_parts: Sequence[str] = ()
        else:
            current_dir, unmatched_parts = base_dir.attempt_lookup(
                value_being_completed
            )

        if len(unmatched_parts) > 1:
            # Unknown directory part / glob, and we currently do not deal with that.
            return None
        if len(unmatched_parts) == 1 and unmatched_parts[0] == "*":
            # Avoid convincing the client to remove the star (seen with emacs)
            return None
        items = []

        path_escaper = _dep5_escape_path if is_dep5_file_list else _noop_escape_path

        for child in current_dir.iterdir:
            if child.is_symlink and is_dep5_file_list:
                continue
            if not supports_spaces_in_filename and (
                " " in child.name or "\t" in child.name
            ):
                continue
            sort_text = (
                f"z-{child.name}" if child.name.startswith(".") else f"a-{child.name}"
            )
            if child.is_dir:
                if _should_ignore_dir(
                    child,
                    supports_dir_match=supports_dir_match,
                    match_non_persistent_paths=match_non_persistent_paths,
                ):
                    continue
                items.append(
                    CompletionItem(
                        f"{child.path}/",
                        label_details=CompletionItemLabelDetails(
                            description=child.path,
                        ),
                        insert_text=path_escaper(f"{child.path}/"),
                        filter_text=f"{child.path}/",
                        sort_text=sort_text,
                        kind=CompletionItemKind.Folder,
                    )
                )
            else:
                items.append(
                    CompletionItem(
                        child.path,
                        label_details=CompletionItemLabelDetails(
                            description=child.path,
                        ),
                        insert_text=path_escaper(child.path),
                        filter_text=child.path,
                        sort_text=sort_text,
                        kind=CompletionItemKind.File,
                    )
                )
        return items

    def value_options_for_completer(
        self,
        lint_state: LintState,
        stanza_parts: Sequence[Deb822ParagraphElement],
        value_being_completed: str,
        markdown_kind: MarkupKind,
        *,
        is_completion_for_field: bool = False,
    ) -> Optional[Sequence[CompletionItem]]:
        known_values = self.known_values
        if self.field_value_class == FieldValueClass.DEP5_FILE_LIST:
            if is_completion_for_field:
                return None
            return self._complete_files(
                lint_state.source_root,
                value_being_completed,
                is_dep5_file_list=True,
            )

        if known_values is None:
            return None
        if is_completion_for_field and (
            len(known_values) == 1
            or (
                len(known_values) == 2
                and self.warn_if_default
                and self.default_value is not None
            )
        ):
            value = next(
                iter(v for v in self.known_values if v != self.default_value),
                None,
            )
            if value is None:
                return None
            return [CompletionItem(value, insert_text=value)]
        return [
            CompletionItem(
                keyword.value,
                insert_text=keyword.value,
                sort_text=keyword.sort_text,
                detail=format_comp_item_synopsis_doc(
                    keyword.usage_hint,
                    keyword.synopsis_translated(lint_state),
                    keyword.is_deprecated,
                ),
                deprecated=keyword.is_deprecated,
                tags=[CompletionItemTag.Deprecated] if keyword.is_deprecated else None,
                documentation=(
                    MarkupContent(value=keyword.long_description, kind=markdown_kind)
                    if keyword.long_description
                    else None
                ),
            )
            for keyword in known_values.values()
            if keyword.is_keyword_valid_completion_in_stanza(stanza_parts)
        ]

    def field_omitted_diagnostics(
        self,
        deb822_file: Deb822FileElement,
        representation_field_range: Range,
        stanza: Deb822ParagraphElement,
        stanza_position: "TEPosition",
        header_stanza: Optional[Deb822FileElement],
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        missing_field_severity = self.missing_field_severity
        if missing_field_severity is None:
            return

        yield Diagnostic(
            representation_field_range,
            f"Stanza is missing field {self.name}",
            severity=missing_field_severity,
            source="debputy",
        )

    def field_diagnostics(
        self,
        deb822_file: Deb822FileElement,
        kvpair: Deb822KeyValuePairElement,
        stanza: Deb822ParagraphElement,
        stanza_position: "TEPosition",
        kvpair_range_te: "TERange",
        lint_state: LintState,
        *,
        field_name_typo_reported: bool = False,
    ) -> Iterable[Diagnostic]:
        field_name_token = kvpair.field_token
        field_name_range_te = kvpair.field_token.range_in_parent().relative_to(
            kvpair_range_te.start_pos
        )
        yield from self._diagnostics_for_field_name(
            field_name_token,
            field_name_range_te,
            field_name_typo_reported,
            lint_state,
        )
        if self.custom_field_check is not None:
            yield from self.custom_field_check(
                self,
                deb822_file,
                kvpair,
                kvpair_range_te,
                field_name_range_te,
                stanza,
                stanza_position,
                lint_state,
            )
        yield from self._dep5_file_list_diagnostics(
            kvpair, kvpair_range_te.start_pos, lint_state
        )
        if not self.spellcheck_value:
            yield from self._known_value_diagnostics(
                kvpair,
                kvpair_range_te.start_pos,
                lint_state,
            )

    def _diagnostics_for_field_name(
        self,
        token: Deb822FieldNameToken,
        token_range: "TERange",
        typo_detected: bool,
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        field_name = token.text
        # Defeat the case-insensitivity from python-debian
        field_name_cased = str(field_name)
        token_range_server_units = te_range_to_lsp(token_range)
        token_range = lint_state.position_codec.range_to_client_units(
            lint_state.lines,
            token_range_server_units,
        )
        if self.deprecated_with_no_replacement:
            yield Diagnostic(
                token_range,
                f"{field_name_cased} is deprecated and no longer used",
                severity=DiagnosticSeverity.Warning,
                source="debputy",
                tags=[DiagnosticTag.Deprecated],
                data=DiagnosticData(quickfixes=[propose_remove_line_quick_fix()]),
            )
        elif self.replaced_by is not None:
            yield Diagnostic(
                token_range,
                f"{field_name_cased} is a deprecated name for {self.replaced_by}",
                severity=DiagnosticSeverity.Warning,
                source="debputy",
                tags=[DiagnosticTag.Deprecated],
                data=DiagnosticData(
                    quickfixes=[propose_correct_text_quick_fix(self.replaced_by)],
                ),
            )

        if not typo_detected and field_name_cased != self.name:
            yield Diagnostic(
                token_range,
                f"Non-canonical spelling of {self.name}",
                severity=DiagnosticSeverity.Information,
                source="debputy",
                data=DiagnosticData(
                    quickfixes=[propose_correct_text_quick_fix(self.name)]
                ),
            )

    def _dep5_file_list_diagnostics(
        self,
        kvpair: Deb822KeyValuePairElement,
        kvpair_position: "TEPosition",
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        source_root = lint_state.source_root
        if (
            self.field_value_class != FieldValueClass.DEP5_FILE_LIST
            or source_root is None
        ):
            return
        interpreter = self.field_value_class.interpreter()
        values = kvpair.interpret_as(interpreter)
        value_off = kvpair.value_element.position_in_parent().relative_to(
            kvpair_position
        )

        assert interpreter is not None

        for token in values.iter_parts():
            if token.is_whitespace:
                continue
            text = token.convert_to_text()
            if "?" in text or "*" in text:
                # TODO: We should validate these as well
                continue
            matched_path, missing_part = source_root.attempt_lookup(text)
            # It is common practice to delete "dirty" files during clean. This causes files listed
            # in `debian/copyright` to go missing and as a consequence, we do not validate whether
            # they are present (that would require us to check the `.orig.tar`, which we could but
            # do not have the infrastructure for).
            if not missing_part and matched_path.is_dir:
                path_range_te = token.range_in_parent().relative_to(value_off)
                path_range = lint_state.position_codec.range_to_client_units(
                    lint_state.lines,
                    te_range_to_lsp(path_range_te),
                )
                yield Diagnostic(
                    path_range,
                    "Directories cannot be a match. Use `dir/*` to match everything in it",
                    severity=DiagnosticSeverity.Warning,
                    source="debputy",
                    data=DiagnosticData(
                        quickfixes=[
                            propose_correct_text_quick_fix(f"{matched_path.path}/*")
                        ]
                    ),
                )

    def _known_value_diagnostics(
        self,
        kvpair: Deb822KeyValuePairElement,
        kvpair_position: "TEPosition",
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        unknown_value_severity = self.unknown_value_diagnostic_severity
        interpreter = self.field_value_class.interpreter()
        if interpreter is None:
            return
        values = kvpair.interpret_as(interpreter)
        value_off = kvpair.value_element.position_in_parent().relative_to(
            kvpair_position
        )

        last_token_non_ws_sep_token: Optional[TE] = None
        for token in values.iter_parts():
            if token.is_whitespace:
                continue
            if not token.is_separator:
                last_token_non_ws_sep_token = None
                continue
            if last_token_non_ws_sep_token is not None:
                sep_range_te = token.range_in_parent().relative_to(value_off)
                value_range = lint_state.position_codec.range_to_client_units(
                    lint_state.lines,
                    te_range_to_lsp(sep_range_te),
                )
                yield Diagnostic(
                    value_range,
                    "Duplicate separator",
                    severity=DiagnosticSeverity.Error,
                    source="debputy",
                )
            last_token_non_ws_sep_token = token

        allowed_values = self.known_values
        if not allowed_values:
            return

        first_value = None
        first_exclusive_value_ref = None
        first_exclusive_value = None
        has_emitted_for_exclusive = False

        for value_ref in values.iter_value_references():
            value = value_ref.value
            if (
                first_value is not None
                and self.field_value_class == FieldValueClass.SINGLE_VALUE
            ):
                value_loc = value_ref.locatable
                range_position_te = value_loc.range_in_parent().relative_to(value_off)
                value_range_in_server_units = te_range_to_lsp(range_position_te)
                value_range = lint_state.position_codec.range_to_client_units(
                    lint_state.lines,
                    value_range_in_server_units,
                )
                yield Diagnostic(
                    value_range,
                    f"The field {self.name} can only have exactly one value.",
                    severity=DiagnosticSeverity.Error,
                    source="debputy",
                )
                # TODO: Add quickfix if the value is also invalid
                continue

            if first_exclusive_value_ref is not None and not has_emitted_for_exclusive:
                assert first_exclusive_value is not None
                value_loc = first_exclusive_value_ref.locatable
                value_range_te = value_loc.range_in_parent().relative_to(value_off)
                value_range_in_server_units = te_range_to_lsp(value_range_te)
                value_range = lint_state.position_codec.range_to_client_units(
                    lint_state.lines,
                    value_range_in_server_units,
                )
                yield Diagnostic(
                    value_range,
                    f'The value "{first_exclusive_value}" cannot be used with other values.',
                    severity=DiagnosticSeverity.Error,
                    source="debputy",
                )

            known_value, unknown_value_message, unknown_severity, typo_fix_data = (
                _unknown_value_check(
                    self.name,
                    value,
                    self.known_values,
                    unknown_value_severity,
                )
            )

            issues = []

            if known_value and known_value.is_exclusive:
                first_exclusive_value = known_value.value  # In case of typos.
                first_exclusive_value_ref = value_ref
                if first_value is not None:
                    has_emitted_for_exclusive = True
                    issues.append(
                        {
                            "message": f'The value "{known_value.value}" cannot be used with other values.',
                            "severity": DiagnosticSeverity.Error,
                            "source": "debputy",
                        }
                    )

            if first_value is None:
                first_value = value

            if unknown_value_message is not None:
                assert unknown_severity is not None
                issues.append(
                    {
                        "message": unknown_value_message,
                        "severity": unknown_severity,
                        "source": "debputy",
                        "data": DiagnosticData(quickfixes=typo_fix_data),
                    }
                )

            if known_value is not None and known_value.is_obsolete:
                replacement = known_value.replaced_by
                if replacement is not None:
                    obsolete_value_message = (
                        f'The value "{value}" has been replaced by {replacement}'
                    )
                    obsolete_severity = DiagnosticSeverity.Warning
                    obsolete_fix_data = [propose_correct_text_quick_fix(replacement)]
                else:
                    obsolete_value_message = (
                        f'The value "{value}" is obsolete without a single replacement'
                    )
                    obsolete_severity = DiagnosticSeverity.Warning
                    obsolete_fix_data = None
                issues.append(
                    {
                        "message": obsolete_value_message,
                        "severity": obsolete_severity,
                        "source": "debputy",
                        "data": DiagnosticData(quickfixes=obsolete_fix_data),
                    }
                )

            if not issues:
                continue

            value_loc = value_ref.locatable
            value_range_te = value_loc.range_in_parent().relative_to(value_off)
            value_range_in_server_units = te_range_to_lsp(value_range_te)
            value_range = lint_state.position_codec.range_to_client_units(
                lint_state.lines,
                value_range_in_server_units,
            )
            yield from (Diagnostic(value_range, **issue_data) for issue_data in issues)

    def reformat_field(
        self,
        effective_preference: "EffectiveFormattingPreference",
        stanza_range: TERange,
        kvpair: Deb822KeyValuePairElement,
        formatter: FormatterCallback,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        kvpair_range = kvpair.range_in_parent().relative_to(stanza_range.start_pos)
        return trim_end_of_line_whitespace(
            position_codec,
            lines,
            line_range=range(
                kvpair_range.start_pos.line_position,
                kvpair_range.end_pos.line_position,
            ),
        )


@dataclasses.dataclass(slots=True)
class DctrlLikeKnownField(Deb822KnownField):

    def reformat_field(
        self,
        effective_preference: "EffectiveFormattingPreference",
        stanza_range: TERange,
        kvpair: Deb822KeyValuePairElement,
        formatter: FormatterCallback,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        interpretation = self.field_value_class.interpreter()
        if (
            not effective_preference.deb822_normalize_field_content
            or interpretation is None
        ):
            yield from super(DctrlLikeKnownField, self).reformat_field(
                effective_preference,
                stanza_range,
                kvpair,
                formatter,
                position_codec,
                lines,
            )
            return
        if not self.reformattable_field:
            yield from super(DctrlLikeKnownField, self).reformat_field(
                effective_preference,
                stanza_range,
                kvpair,
                formatter,
                position_codec,
                lines,
            )
            return
        seen: Set[str] = set()
        old_kvpair_range = kvpair.range_in_parent()
        sort = self.is_sortable_field

        # Avoid the context manager as we do not want to perform the change (it would contaminate future ranges)
        field_content = kvpair.interpret_as(interpretation)
        old_value = field_content.convert_to_text(with_field_name=True)
        for package_ref in field_content.iter_value_references():
            value = package_ref.value
            if self.is_relationship_field:
                new_value = " | ".join(x.strip() for x in value.split("|"))
            else:
                new_value = value
            if not sort or new_value not in seen:
                if new_value != value:
                    package_ref.value = new_value
                seen.add(new_value)
            else:
                package_ref.remove()
            if sort:
                field_content.sort(key=_sort_packages_key)
            field_content.value_formatter(formatter)
            field_content.reformat_when_finished()

        new_value = field_content.convert_to_text(with_field_name=True)
        if new_value != old_value:
            range_server_units = te_range_to_lsp(
                old_kvpair_range.relative_to(stanza_range.start_pos)
            )
            yield TextEdit(
                position_codec.range_to_client_units(lines, range_server_units),
                new_value,
            )

    @property
    def reformattable_field(self) -> bool:
        return self.is_relationship_field or self.is_sortable_field

    @property
    def is_relationship_field(self) -> bool:
        return False

    @property
    def is_sortable_field(self) -> bool:
        return self.is_relationship_field


@dataclasses.dataclass(slots=True)
class DTestsCtrlKnownField(DctrlLikeKnownField):
    @property
    def is_relationship_field(self) -> bool:
        return self.name == "Depends"

    @property
    def is_sortable_field(self) -> bool:
        return self.is_relationship_field or self.name in (
            "Features",
            "Restrictions",
            "Tests",
        )


@dataclasses.dataclass(slots=True)
class DctrlKnownField(DctrlLikeKnownField):
    inherits_from_source: bool = False

    def field_omitted_diagnostics(
        self,
        deb822_file: Deb822FileElement,
        representation_field_range: Range,
        stanza: Deb822ParagraphElement,
        stanza_position: "TEPosition",
        header_stanza: Optional[Deb822FileElement],
        lint_state: LintState,
    ) -> Iterable[Diagnostic]:
        missing_field_severity = self.missing_field_severity
        if missing_field_severity is None:
            return

        if (
            self.inherits_from_source
            and header_stanza is not None
            and self.name in header_stanza
        ):
            return

        if self.name == "Standards-Version":
            stanzas = list(deb822_file)[1:]
            if all(s.get("Package-Type") == "udeb" for s in stanzas):
                return

        yield Diagnostic(
            representation_field_range,
            f"Stanza is missing field {self.name}",
            severity=missing_field_severity,
            source="debputy",
        )

    def reformat_field(
        self,
        effective_preference: "EffectiveFormattingPreference",
        stanza_range: TERange,
        kvpair: Deb822KeyValuePairElement,
        formatter: FormatterCallback,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        if (
            self.name == "Architecture"
            and effective_preference.deb822_normalize_field_content
        ):
            interpretation = self.field_value_class.interpreter()
            assert interpretation is not None
            archs = list(kvpair.interpret_as(interpretation))
            # Sort, with wildcard entries (such as linux-any) first:
            archs = sorted(archs, key=lambda x: ("any" not in x, x))
            new_value = f"{kvpair.field_name}: {' '.join(archs)}\n"
            if new_value != kvpair.convert_to_text():
                kvpair_range = te_range_to_lsp(
                    kvpair.range_in_parent().relative_to(stanza_range.start_pos)
                )
                return [
                    TextEdit(
                        position_codec.range_to_client_units(lines, kvpair_range),
                        new_value,
                    )
                ]
            return tuple()

        return super(DctrlKnownField, self).reformat_field(
            effective_preference,
            stanza_range,
            kvpair,
            formatter,
            position_codec,
            lines,
        )

    @property
    def is_relationship_field(self) -> bool:
        name_lc = self.name.lower()
        return (
            name_lc in all_package_relationship_fields()
            or name_lc in all_source_relationship_fields()
        )

    @property
    def reformattable_field(self) -> bool:
        return self.is_relationship_field or self.name == "Uploaders"


@dataclasses.dataclass(slots=True)
class DctrlRelationshipKnownField(DctrlKnownField):
    allowed_version_operators: FrozenSet[str] = frozenset()
    supports_or_relation: bool = True

    @property
    def is_relationship_field(self) -> bool:
        return True


SOURCE_FIELDS = _fields(
    DctrlKnownField(
        "Source",
        FieldValueClass.SINGLE_VALUE,
        custom_field_check=_combined_custom_field_check(
            _each_value_match_regex_validation(PKGNAME_REGEX),
            _has_packaging_expected_file(
                "copyright",
                "No copyright file (package license)",
                severity=DiagnosticSeverity.Warning,
            ),
            _has_packaging_expected_file(
                "changelog",
                "No Debian changelog file",
                severity=DiagnosticSeverity.Error,
            ),
            _has_build_instructions,
        ),
        missing_field_severity=DiagnosticSeverity.Error,
    ),
    DctrlKnownField(
        "Standards-Version",
        FieldValueClass.SINGLE_VALUE,
        # Conditionally mandatory (special-cased by field_omitted_diagnostics)
        missing_field_severity=DiagnosticSeverity.Error,
        custom_field_check=_sv_field_validation,
    ),
    DctrlKnownField(
        "Section",
        FieldValueClass.SINGLE_VALUE,
        known_values=ALL_SECTIONS,
        unknown_value_diagnostic_severity=DiagnosticSeverity.Warning,
    ),
    DctrlKnownField(
        "Priority",
        FieldValueClass.SINGLE_VALUE,
        known_values=ALL_PRIORITIES,
    ),
    DctrlKnownField(
        "Maintainer",
        FieldValueClass.SINGLE_VALUE,
        missing_field_severity=DiagnosticSeverity.Error,
    ),
    DctrlKnownField(
        "Build-Depends",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Build-Depends-Arch",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Build-Depends-Indep",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlRelationshipKnownField(
        "Build-Conflicts",
        FieldValueClass.COMMA_SEPARATED_LIST,
        supports_or_relation=False,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlRelationshipKnownField(
        "Build-Conflicts-Arch",
        FieldValueClass.COMMA_SEPARATED_LIST,
        supports_or_relation=False,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlRelationshipKnownField(
        "Build-Conflicts-Indep",
        FieldValueClass.COMMA_SEPARATED_LIST,
        supports_or_relation=False,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Rules-Requires-Root",
        FieldValueClass.SPACE_SEPARATED_LIST,
        unknown_value_diagnostic_severity=None,
        custom_field_check=_rrr_build_driver_mismatch,
    ),
    DctrlKnownField(
        "X-Style",
        FieldValueClass.SINGLE_VALUE,
        known_values=ALL_PUBLIC_NAMED_STYLES,
        unknown_value_diagnostic_severity=DiagnosticSeverity.Warning,
    ),
)


BINARY_FIELDS = _fields(
    DctrlKnownField(
        "Package",
        FieldValueClass.SINGLE_VALUE,
        custom_field_check=_each_value_match_regex_validation(PKGNAME_REGEX),
        missing_field_severity=DiagnosticSeverity.Error,
    ),
    DctrlKnownField(
        "Architecture",
        FieldValueClass.SPACE_SEPARATED_LIST,
        missing_field_severity=DiagnosticSeverity.Error,
        unknown_value_diagnostic_severity=None,
        # FIXME: Specialize validation for architecture ("!foo" is not a "typo" and should have a better warning)
        known_values=allowed_values(*dpkg_arch_and_wildcards()),
    ),
    DctrlKnownField(
        "Pre-Depends",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Depends",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Recommends",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Suggests",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Enhances",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlRelationshipKnownField(
        "Provides",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
        supports_or_relation=False,
        allowed_version_operators=frozenset(["="]),
    ),
    DctrlRelationshipKnownField(
        "Conflicts",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
        supports_or_relation=False,
    ),
    DctrlRelationshipKnownField(
        "Breaks",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
        supports_or_relation=False,
    ),
    DctrlRelationshipKnownField(
        "Replaces",
        FieldValueClass.COMMA_SEPARATED_LIST,
        custom_field_check=_dctrl_validate_dep,
    ),
    DctrlKnownField(
        "Build-Profiles",
        FieldValueClass.BUILD_PROFILES_LIST,
    ),
    DctrlKnownField(
        "Section",
        FieldValueClass.SINGLE_VALUE,
        missing_field_severity=DiagnosticSeverity.Error,
        inherits_from_source=True,
        known_values=ALL_SECTIONS,
        unknown_value_diagnostic_severity=DiagnosticSeverity.Warning,
    ),
    DctrlKnownField(
        "Priority",
        FieldValueClass.SINGLE_VALUE,
        warn_if_default=False,
        missing_field_severity=DiagnosticSeverity.Error,
        inherits_from_source=True,
        known_values=ALL_PRIORITIES,
    ),
    DctrlKnownField(
        "Multi-Arch",
        FieldValueClass.SINGLE_VALUE,
        # Explicit "no" tends to be used as "someone reviewed this and concluded no", so we do
        # not warn about it being explicitly "no".
        warn_if_default=False,
        default_value="no",
        custom_field_check=_dctrl_ma_field_validation,
        known_values=allowed_values(
            Keyword(
                "no",
                # Show `no` after `foreign` and `same`. Often, you want `same` or `foreign`.
                sort_text="zz-no",
            ),
            Keyword(
                "same",
                can_complete_keyword_in_stanza=_complete_only_in_arch_dep_pkgs,
            ),
            Keyword(
                "allowed",
                # Never show `allowed` first, it is the absolute least likely candidate.
                sort_text="zzzz-allowed",
                usage_hint="rare",
            ),
        ),
    ),
    DctrlKnownField(
        "XB-Installer-Menu-Item",
        FieldValueClass.SINGLE_VALUE,
        can_complete_field_in_stanza=_complete_only_for_udeb_pkgs,
        custom_field_check=_combined_custom_field_check(
            _udeb_only_field_validation,
            _each_value_match_regex_validation(re.compile(r"^[1-9]\d{3,4}$")),
        ),
    ),
    DctrlKnownField(
        "X-DH-Build-For-Type",
        FieldValueClass.SINGLE_VALUE,
        custom_field_check=_arch_not_all_only_field_validation,
        can_complete_field_in_stanza=_complete_only_in_arch_dep_pkgs,
    ),
    DctrlKnownField(
        "X-Time64-Compat",
        FieldValueClass.SINGLE_VALUE,
        can_complete_field_in_stanza=_complete_only_in_arch_dep_pkgs,
        custom_field_check=_combined_custom_field_check(
            _each_value_match_regex_validation(PKGNAME_REGEX),
            _arch_not_all_only_field_validation,
        ),
    ),
    DctrlKnownField(
        "Description",
        FieldValueClass.FREE_TEXT_FIELD,
        # It will build just fine. But no one will know what it is for, so it probably won't be installed
        missing_field_severity=DiagnosticSeverity.Warning,
        custom_field_check=dctrl_description_validator,
    ),
    DctrlKnownField(
        "XB-Cnf-Visible-Pkgname",
        FieldValueClass.SINGLE_VALUE,
        custom_field_check=_each_value_match_regex_validation(PKGNAME_REGEX),
    ),
    DctrlKnownField(
        "X-DhRuby-Root",
        FieldValueClass.SINGLE_VALUE,
    ),
)
_DEP5_HEADER_FIELDS = _fields(
    Deb822KnownField(
        "Format",
        FieldValueClass.SINGLE_VALUE,
        missing_field_severity=DiagnosticSeverity.Error,
    ),
)
_DEP5_FILES_FIELDS = _fields(
    Deb822KnownField(
        "Files",
        FieldValueClass.DEP5_FILE_LIST,
        custom_field_check=_dep5_files_check,
        missing_field_severity=DiagnosticSeverity.Error,
    ),
    Deb822KnownField(
        "License",
        FieldValueClass.FREE_TEXT_FIELD,
        missing_field_severity=DiagnosticSeverity.Error,
    ),
    Deb822KnownField(
        "Comment",
        FieldValueClass.FREE_TEXT_FIELD,
    ),
)
_DEP5_LICENSE_FIELDS = _fields(
    Deb822KnownField(
        "License",
        FieldValueClass.FREE_TEXT_FIELD,
        missing_field_severity=DiagnosticSeverity.Error,
    ),
)

_DTESTSCTRL_FIELDS = _fields(
    DTestsCtrlKnownField(
        "Architecture",
        FieldValueClass.SPACE_SEPARATED_LIST,
        unknown_value_diagnostic_severity=None,
        # FIXME: Specialize validation for architecture ("!fou" to "foo" would be bad)
        known_values=allowed_values(*dpkg_arch_and_wildcards(allow_negations=True)),
    ),
    DTestsCtrlKnownField(
        "Restrictions",
        FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST,
        unknown_value_diagnostic_severity=DiagnosticSeverity.Warning,
    ),
)


@dataclasses.dataclass(slots=True)
class StanzaMetadata(Mapping[str, F], Generic[F], ABC):
    stanza_type_name: str
    stanza_fields: Mapping[str, F]

    def stanza_diagnostics(
        self,
        stanza: Deb822ParagraphElement,
        stanza_position_in_file: "TEPosition",
    ) -> Iterable[Diagnostic]:
        raise NotImplementedError

    def __getitem__(self, key: str) -> F:
        key_lc = key.lower()
        key_norm = normalize_dctrl_field_name(key_lc)
        return self.stanza_fields[key_norm]

    def __len__(self) -> int:
        return len(self.stanza_fields)

    def __iter__(self):
        return iter(self.stanza_fields.keys())

    def reformat_stanza(
        self,
        effective_preference: "EffectiveFormattingPreference",
        stanza: Deb822ParagraphElement,
        stanza_range: TERange,
        formatter: FormatterCallback,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        for known_field in self.stanza_fields.values():
            kvpair = stanza.get_kvpair_element((known_field.name, 0), use_get=True)
            if kvpair is None:
                continue
            yield from known_field.reformat_field(
                effective_preference,
                stanza_range,
                kvpair,
                formatter,
                position_codec,
                lines,
            )


@dataclasses.dataclass(slots=True)
class Dep5StanzaMetadata(StanzaMetadata[Deb822KnownField]):
    def stanza_diagnostics(
        self,
        stanza: Deb822ParagraphElement,
        stanza_position_in_file: "TEPosition",
    ) -> Iterable[Diagnostic]:
        pass


@dataclasses.dataclass(slots=True)
class DctrlStanzaMetadata(StanzaMetadata[DctrlKnownField]):

    def stanza_diagnostics(
        self,
        stanza: Deb822ParagraphElement,
        stanza_position_in_file: "TEPosition",
    ) -> Iterable[Diagnostic]:
        pass


@dataclasses.dataclass(slots=True)
class DTestsCtrlStanzaMetadata(StanzaMetadata[DTestsCtrlKnownField]):

    def stanza_diagnostics(
        self,
        stanza: Deb822ParagraphElement,
        stanza_position_in_file: "TEPosition",
    ) -> Iterable[Diagnostic]:
        pass


def lsp_reference_data_dir() -> str:
    return os.path.join(
        os.path.dirname(__file__),
        "data",
    )


class Deb822FileMetadata(Generic[S, F]):

    def __init__(self) -> None:
        self._is_initialized = False
        self._data: Optional[Deb822ReferenceData] = None

    @property
    def reference_data_basename(self) -> str:
        raise NotImplementedError

    def _new_field(
        self,
        name: str,
        field_value_type: FieldValueClass,
    ) -> F:
        raise NotImplementedError

    def _reference_data(self) -> Deb822ReferenceData:
        ref = self._data
        if ref is not None:
            return ref

        reference_data_file = os.path.join(
            lsp_reference_data_dir(),
            self.reference_data_basename,
        )

        with open(reference_data_file) as fd:
            raw = MANIFEST_YAML.load(fd)

        attr_path = AttributePath.root_path(raw)
        ref = DEB822_REFERENCE_DATA_PARSER.parse_input(raw, attr_path)
        self._data = ref
        return ref

    @property
    def is_initialized(self) -> bool:
        return self._is_initialized

    def ensure_initialized(self) -> None:
        if self.is_initialized:
            return
        # Enables us to use __getitem__
        self._is_initialized = True
        ref_data = self._reference_data()
        ref_defs = ref_data.get("definitions")
        variables = {}
        ref_variables = ref_defs.get("variables", []) if ref_defs else []
        for ref_variable in ref_variables:
            name = ref_variable["name"]
            fallback = ref_variable["fallback"]
            variables[name] = fallback

        def _resolve_doc(template: Optional[str]) -> Optional[str]:
            if template is None:
                return None
            try:
                return template.format(**variables)
            except ValueError as e:
                template_escaped = template.replace("\n", "\\r")
                _error(f"Bad template: {template_escaped}: {e}")

        for ref_stanza_type in ref_data["stanza_types"]:
            stanza_name = ref_stanza_type["stanza_name"]
            stanza = self[stanza_name]
            stanza_fields = dict(stanza.stanza_fields)
            stanza.stanza_fields = stanza_fields
            for ref_field in ref_stanza_type["fields"]:
                _resolve_field(
                    ref_field,
                    stanza_fields,
                    self._new_field,
                    _resolve_doc,
                    f"Stanza:{stanza.stanza_type_name}|Field:{ref_field['canonical_name']}",
                )

    def classify_stanza(self, stanza: Deb822ParagraphElement, stanza_idx: int) -> S:
        return self.guess_stanza_classification_by_idx(stanza_idx)

    def guess_stanza_classification_by_idx(self, stanza_idx: int) -> S:
        raise NotImplementedError

    def stanza_types(self) -> Iterable[S]:
        raise NotImplementedError

    def __getitem__(self, item: str) -> S:
        raise NotImplementedError

    def file_diagnostics(
        self,
        file: Deb822FileElement,
    ) -> Iterable[Diagnostic]:
        raise NotImplementedError

    def get(self, item: str) -> Optional[S]:
        try:
            return self[item]
        except KeyError:
            return None

    def reformat(
        self,
        effective_preference: "EffectiveFormattingPreference",
        deb822_file: Deb822FileElement,
        formatter: FormatterCallback,
        _content: str,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        stanza_idx = -1
        for token_or_element in deb822_file.iter_parts():
            if isinstance(token_or_element, Deb822ParagraphElement):
                stanza_range = token_or_element.range_in_parent()
                stanza_idx += 1
                stanza_metadata = self.classify_stanza(token_or_element, stanza_idx)
                yield from stanza_metadata.reformat_stanza(
                    effective_preference,
                    token_or_element,
                    stanza_range,
                    formatter,
                    position_codec,
                    lines,
                )
            else:
                token_range = token_or_element.range_in_parent()
                yield from trim_end_of_line_whitespace(
                    position_codec,
                    lines,
                    line_range=range(
                        token_range.start_pos.line_position,
                        token_range.end_pos.line_position,
                    ),
                )


_DCTRL_SOURCE_STANZA = DctrlStanzaMetadata(
    "Source",
    SOURCE_FIELDS,
)
_DCTRL_PACKAGE_STANZA = DctrlStanzaMetadata("Package", BINARY_FIELDS)

_DEP5_HEADER_STANZA = Dep5StanzaMetadata(
    "Header",
    _DEP5_HEADER_FIELDS,
)
_DEP5_FILES_STANZA = Dep5StanzaMetadata(
    "Files",
    _DEP5_FILES_FIELDS,
)
_DEP5_LICENSE_STANZA = Dep5StanzaMetadata(
    "License",
    _DEP5_LICENSE_FIELDS,
)

_DTESTSCTRL_STANZA = DTestsCtrlStanzaMetadata("Tests", _DTESTSCTRL_FIELDS)


class Dep5FileMetadata(Deb822FileMetadata[Dep5StanzaMetadata, Deb822KnownField]):

    @property
    def reference_data_basename(self) -> str:
        return "debian_copyright_reference_data.yaml"

    def _new_field(
        self,
        name: str,
        field_value_type: FieldValueClass,
    ) -> F:
        return Deb822KnownField(name, field_value_type)

    def classify_stanza(
        self,
        stanza: Deb822ParagraphElement,
        stanza_idx: int,
    ) -> Dep5StanzaMetadata:
        self.ensure_initialized()
        if stanza_idx == 0:
            return _DEP5_HEADER_STANZA
        if stanza_idx > 0:
            if "Files" in stanza:
                return _DEP5_FILES_STANZA
            return _DEP5_LICENSE_STANZA
        raise ValueError("The stanza_idx must be 0 or greater")

    def guess_stanza_classification_by_idx(self, stanza_idx: int) -> Dep5StanzaMetadata:
        self.ensure_initialized()
        if stanza_idx == 0:
            return _DEP5_HEADER_STANZA
        if stanza_idx > 0:
            return _DEP5_FILES_STANZA
        raise ValueError("The stanza_idx must be 0 or greater")

    def stanza_types(self) -> Iterable[Dep5StanzaMetadata]:
        self.ensure_initialized()
        # Order assumption made in the LSP code.
        yield _DEP5_HEADER_STANZA
        yield _DEP5_FILES_STANZA
        yield _DEP5_LICENSE_STANZA

    def __getitem__(self, item: str) -> Dep5StanzaMetadata:
        self.ensure_initialized()
        if item == "Header":
            return _DEP5_HEADER_STANZA
        if item == "Files":
            return _DEP5_FILES_STANZA
        if item == "License":
            return _DEP5_LICENSE_STANZA
        raise KeyError(item)


def _resolve_keyword(
    ref_value: StaticValue,
    known_values: Dict[str, Keyword],
    resolve_template: Callable[[Optional[str]], Optional[str]],
    translation_context: str,
) -> None:
    value_key = ref_value["value"]
    changes = {
        "translation_context": translation_context,
    }
    try:
        known_value = known_values[value_key]
    except KeyError:
        known_value = Keyword(value_key)
        known_values[value_key] = known_value
    value_doc = ref_value.get("documentation")
    if value_doc is not None:
        changes["synopsis"] = value_doc.get("synopsis")
        changes["long_description"] = resolve_template(
            value_doc.get("long_description")
        )
    if changes:
        known_value = known_value.replace(**changes)
        known_values[value_key] = known_value


def _resolve_field(
    ref_field: Deb822Field,
    stanza_fields: Dict[str, F],
    field_constructor: Callable[[str, FieldValueClass], F],
    resolve_template: Callable[[Optional[str]], Optional[str]],
    translation_context: str,
) -> None:
    field_name = ref_field["canonical_name"]
    field_value_type = FieldValueClass.from_key(ref_field["field_value_type"])
    doc = ref_field.get("documentation")
    ref_values = ref_field.get("values", [])
    norm_field_name = normalize_dctrl_field_name(field_name.lower())

    try:
        field = stanza_fields[norm_field_name]
    except KeyError:
        field = field_constructor(
            field_name,
            field_value_type,
        )
        stanza_fields[norm_field_name] = field
    else:
        if field.name != field_name:
            _error(
                f'Error in reference data: Code uses "{field.name}" as canonical name and the data file'
                f" uses {field_name}. Please ensure the data is correctly aligned."
            )
        if field.field_value_class != field_value_type:
            _error(
                f'Error in reference data for field "{field.name}": Code has'
                f" {field.field_value_class.key} and the data file uses {field_value_type.key}"
                f" for field-value-type. Please ensure the data is correctly aligned."
            )
    if doc is not None:
        field.synopsis = doc.get("synopsis")
        field.long_description = resolve_template(doc.get("long_description"))

    field.default_value = ref_field.get("default_value")
    field.spellcheck_value = ref_field.get("spellcheck_value", False)
    field.deprecated_with_no_replacement = ref_field.get(
        "is_obsolete_without_replacement", False
    )
    field.replaced_by = ref_field.get("replaced_by")
    field.translation_context = translation_context

    known_values = field.known_values
    if known_values is None:
        known_values = {}
    else:
        known_values = dict(known_values)

    for ref_value in ref_values:
        _resolve_keyword(ref_value, known_values, resolve_template, translation_context)

    if known_values:
        field.known_values = known_values


class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata, DctrlKnownField]):

    @property
    def reference_data_basename(self) -> str:
        return "debian_control_reference_data.yaml"

    def _new_field(
        self,
        name: str,
        field_value_type: FieldValueClass,
    ) -> F:
        return DctrlKnownField(name, field_value_type)

    def guess_stanza_classification_by_idx(
        self,
        stanza_idx: int,
    ) -> DctrlStanzaMetadata:
        self.ensure_initialized()
        if stanza_idx == 0:
            return _DCTRL_SOURCE_STANZA
        if stanza_idx > 0:
            return _DCTRL_PACKAGE_STANZA
        raise ValueError("The stanza_idx must be 0 or greater")

    def stanza_types(self) -> Iterable[DctrlStanzaMetadata]:
        self.ensure_initialized()
        # Order assumption made in the LSP code.
        yield _DCTRL_SOURCE_STANZA
        yield _DCTRL_PACKAGE_STANZA

    def __getitem__(self, item: str) -> DctrlStanzaMetadata:
        self.ensure_initialized()
        if item == "Source":
            return _DCTRL_SOURCE_STANZA
        if item == "Package":
            return _DCTRL_PACKAGE_STANZA
        raise KeyError(item)

    def reformat(
        self,
        effective_preference: "EffectiveFormattingPreference",
        deb822_file: Deb822FileElement,
        formatter: FormatterCallback,
        content: str,
        position_codec: LintCapablePositionCodec,
        lines: List[str],
    ) -> Iterable[TextEdit]:
        edits = list(
            super().reformat(
                effective_preference,
                deb822_file,
                formatter,
                content,
                position_codec,
                lines,
            )
        )

        if (
            not effective_preference.deb822_normalize_stanza_order
            or deb822_file.find_first_error_element() is not None
        ):
            return edits
        names = []
        for idx, stanza in enumerate(deb822_file):
            if idx < 2:
                continue
            name = stanza.get("Package")
            if name is None:
                return edits
            names.append(name)

        reordered = sorted(names)
        if names == reordered:
            return edits

        if edits:
            content = apply_text_edits(content, lines, edits)
            lines = content.splitlines(keepends=True)
            deb822_file = parse_deb822_file(
                lines,
                accept_files_with_duplicated_fields=True,
                accept_files_with_error_tokens=True,
            )

        stanzas = list(deb822_file)
        reordered_stanza = stanzas[:2] + sorted(
            stanzas[2:], key=operator.itemgetter("Package")
        )
        bits = []
        stanza_idx = 0
        for token_or_element in deb822_file.iter_parts():
            if isinstance(token_or_element, Deb822ParagraphElement):
                bits.append(reordered_stanza[stanza_idx].dump())
                stanza_idx += 1
            else:
                bits.append(token_or_element.convert_to_text())

        new_content = "".join(bits)

        return [
            TextEdit(
                Range(
                    Position(0, 0),
                    Position(len(lines) + 1, 0),
                ),
                new_content,
            )
        ]


class DTestsCtrlFileMetadata(
    Deb822FileMetadata[DTestsCtrlStanzaMetadata, DTestsCtrlKnownField]
):

    @property
    def reference_data_basename(self) -> str:
        return "debian_tests_control_reference_data.yaml"

    def _new_field(
        self,
        name: str,
        field_value_type: FieldValueClass,
    ) -> F:
        return DTestsCtrlKnownField(name, field_value_type)

    def guess_stanza_classification_by_idx(self, stanza_idx: int) -> S:
        if stanza_idx >= 0:
            self.ensure_initialized()
            return _DTESTSCTRL_STANZA
        raise ValueError("The stanza_idx must be 0 or greater")

    def stanza_types(self) -> Iterable[S]:
        self.ensure_initialized()
        yield _DTESTSCTRL_STANZA

    def __getitem__(self, item: str) -> S:
        self.ensure_initialized()
        if item == "Tests":
            return _DTESTSCTRL_STANZA
        raise KeyError(item)


TRANSLATABLE_DEB822_FILE_METADATA: Sequence[
    Callable[[], Deb822FileMetadata[Any, Any]]
] = [
    DctrlFileMetadata,
    Dep5FileMetadata,
    DTestsCtrlFileMetadata,
]
