import collections
import contextlib
import os
from typing import (
    Optional,
    Dict,
    Callable,
    List,
    Any,
    Union,
    Mapping,
    IO,
    Iterator,
    NotRequired,
    Literal,
    Annotated,
    TypedDict,
)

from debian.debian_support import DpkgArchTable
from ruamel.yaml import YAMLError

from debputy.highlevel_manifest import (
    HighLevelManifest,
    PackageTransformationDefinition,
    MutableYAMLManifest,
    MANIFEST_YAML,
)
from debputy.maintscript_snippet import (
    DpkgMaintscriptHelperCommand,
    MaintscriptSnippet,
    STD_CONTROL_SCRIPTS,
    MaintscriptSnippetContainer,
)
from debputy.packages import BinaryPackage
from debputy.path_matcher import (
    MatchRuleType,
    ExactFileSystemPath,
    MatchRule,
    MATCH_ANYTHING,
)
from debputy.substitution import Substitution, SUBST_VAR_RE, VariableNameState
from debputy.transformation_rules import (
    TransformationRule,
)
from debputy.util import (
    _normalize_path,
    escape_shell,
    _error,
)
from debputy.util import _warn, _info
from . import DEBPUTY_DOC_ROOT_DIR
from ._deb_options_profiles import DebBuildOptionsAndProfiles
from ._manifest_constants import *
from .architecture_support import DpkgArchitectureBuildProcessValuesTable
from .exceptions import DebputySubstitutionError
from .filesystem_scan import FSROOverlay
from .installations import InstallRule, PPFInstallRule
from .manifest_conditions import (
    ManifestCondition,
)
from .manifest_parser.base_types import (
    DebputyParsedContent,
    FileSystemExactMatchRule,
)
from .manifest_parser.declarative_parser import (
    parser_from_typed_dict,
    DebputyParseHint,
)
from .manifest_parser.exceptions import ManifestParseException
from .manifest_parser.parser_data import ParserContextData
from .manifest_parser.util import AttributePath
from .packager_provided_files import detect_all_packager_provided_files
from .plugin.api import VirtualPath, reference_documentation
from .plugin.api.impl_types import (
    PluginProvidedFeatureSet,
    TP,
    TTP,
    DispatchingParser,
)

try:
    from Levenshtein import distance
except ImportError:

    def _detect_possible_typo(
        _d,
        _key,
        _attribute_parent_path: AttributePath,
        required: bool,
    ) -> None:
        if required:
            _info(
                "Install python3-levenshtein to have debputy try to detect typos in the manifest."
            )

else:

    def _detect_possible_typo(
        d,
        key,
        _attribute_parent_path: AttributePath,
        _required: bool,
    ) -> None:
        k_len = len(key)
        for actual_key in d:
            if abs(k_len - len(actual_key)) > 2:
                continue
            d = distance(key, actual_key)
            if d > 2:
                continue
            path = _attribute_parent_path.path
            ref = f'at "{path}"' if path else "at the manifest root level"
            _warn(
                f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
            )


def _per_package_subst_variables(
    p: BinaryPackage,
    *,
    name: Optional[str] = None,
) -> Dict[str, str]:
    return {
        "PACKAGE": name if name is not None else p.name,
    }


ACCEPTABLE_CLEAN_ON_REMOVAL_FOR_GLOBS_AND_EXACT_MATCHES = frozenset(
    [
        "./var/log",
    ]
)


ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF = frozenset(
    [
        "./etc",
        "./run",
        "./var/lib",
        "./var/cache",
        "./var/backups",
        "./var/spool",
        # linux-image uses these paths with some `rm -f`
        "./usr/lib/modules",
        "./lib/modules",
        # udev special case
        "./lib/udev",
        "./usr/lib/udev",
        # pciutils deletes /usr/share/misc/pci.ids.<ext>
        "./usr/share/misc",
    ]
)


class ManifestRootRule(DebputyParsedContent):
    manifest_version: str
    definitions: NotRequired[Dict[str, Any]]
    installations: NotRequired[Optional[List[Any]]]
    packages: NotRequired[Dict[str, Any]]


class CleanAfterRemovalRuleSourceFormat(TypedDict):
    path: NotRequired[Annotated[str, DebputyParseHint.target_attribute("paths")]]
    paths: NotRequired[List[str]]
    delete_on: NotRequired[Literal["purge", "removal"]]
    recursive: NotRequired[bool]
    ignore_non_empty_dir: NotRequired[bool]


class CleanAfterRemovalRule(DebputyParsedContent):
    paths: List[str]
    delete_on: NotRequired[Literal["purge", "removal"]]
    recursive: NotRequired[bool]
    ignore_non_empty_dir: NotRequired[bool]


class BinaryPackageRule(DebputyParsedContent):
    binary_version: NotRequired[str]
    transformations: NotRequired[List[Any]]
    conffile_management: NotRequired[List[Any]]
    clean_after_removal: NotRequired[List[Any]]
    installation_search_dirs: NotRequired[List[FileSystemExactMatchRule]]


class DefinitionRule(DebputyParsedContent):
    variables: NotRequired[Dict[str, str]]


_CLEAN_AFTER_REMOVAL_RULE_PARSER = parser_from_typed_dict(
    CleanAfterRemovalRule,
    source_content=Union[CleanAfterRemovalRuleSourceFormat, str, List[str]],
    inline_reference_documentation=reference_documentation(
        reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md##remove-runtime-created-paths-on-purge-or-post-removal-clean-after-removal",
    ),
)
_BINARY_PACKAGE_RULE_PARSER = parser_from_typed_dict(
    BinaryPackageRule,
    inline_reference_documentation=reference_documentation(
        reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#binary-package-rules",
    ),
)
_DEFINITIONS_RULE_PARSER = parser_from_typed_dict(
    DefinitionRule,
    inline_reference_documentation=reference_documentation(
        reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#packager-provided-definitions",
    ),
)
_MANIFEST_ROOT_PARSER = parser_from_typed_dict(
    ManifestRootRule,
    allow_optional=True,
    inline_reference_documentation=reference_documentation(
        reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md",
    ),
)


class HighLevelManifestParser(ParserContextData):
    def __init__(
        self,
        manifest_path: str,
        binary_packages: Mapping[str, BinaryPackage],
        substitution: Substitution,
        dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
        dpkg_arch_query_table: DpkgArchTable,
        build_env: DebBuildOptionsAndProfiles,
        plugin_provided_feature_set: PluginProvidedFeatureSet,
        *,
        # Available for testing purposes only
        debian_dir: Union[str, VirtualPath] = "./debian",
    ):
        self.manifest_path = manifest_path
        self._binary_packages = binary_packages
        self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None
        # In source context, some variables are known to be unresolvable. Record this, so
        # we can give better error messages.
        self._substitution = substitution
        self._dpkg_architecture_variables = dpkg_architecture_variables
        self._dpkg_arch_query_table = dpkg_arch_query_table
        self._build_env = build_env
        self._package_state_stack: List[PackageTransformationDefinition] = []
        self._plugin_provided_feature_set = plugin_provided_feature_set
        self._declared_variables = {}

        if isinstance(debian_dir, str):
            debian_dir = FSROOverlay.create_root_dir("debian", debian_dir)

        self._debian_dir = debian_dir

        # Delayed initialized; we rely on this delay to parse the variables.
        self._all_package_states = None

        self._install_rules: Optional[List[InstallRule]] = None
        self._ownership_caches_loaded = False
        self._used = False

    def _ensure_package_states_is_initialized(self) -> None:
        if self._all_package_states is not None:
            return
        substitution = self._substitution
        binary_packages = self._binary_packages
        assert self._all_package_states is None

        self._all_package_states = {
            n: PackageTransformationDefinition(
                binary_package=p,
                substitution=substitution.with_extra_substitutions(
                    **_per_package_subst_variables(p)
                ),
                is_auto_generated_package=False,
                maintscript_snippets=collections.defaultdict(
                    MaintscriptSnippetContainer
                ),
            )
            for n, p in binary_packages.items()
        }
        for n, p in binary_packages.items():
            dbgsym_name = f"{n}-dbgsym"
            if dbgsym_name in self._all_package_states:
                continue
            self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
                binary_package=p,
                substitution=substitution.with_extra_substitutions(
                    **_per_package_subst_variables(p, name=dbgsym_name)
                ),
                is_auto_generated_package=True,
                maintscript_snippets=collections.defaultdict(
                    MaintscriptSnippetContainer
                ),
            )

    @property
    def binary_packages(self) -> Mapping[str, BinaryPackage]:
        return self._binary_packages

    @property
    def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
        assert self._all_package_states is not None
        return self._all_package_states

    @property
    def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
        return self._dpkg_architecture_variables

    @property
    def dpkg_arch_query_table(self) -> DpkgArchTable:
        return self._dpkg_arch_query_table

    @property
    def build_env(self) -> DebBuildOptionsAndProfiles:
        return self._build_env

    def build_manifest(self) -> HighLevelManifest:
        if self._used:
            raise TypeError("build_manifest can only be called once!")
        self._used = True
        self._ensure_package_states_is_initialized()
        for var, attribute_path in self._declared_variables.items():
            if not self.substitution.is_used(var):
                raise ManifestParseException(
                    f'The variable "{var}" is unused. Either use it or remove it.'
                    f" The variable was declared at {attribute_path.path}."
                )
        if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
            self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
        all_packager_provided_files = detect_all_packager_provided_files(
            self._plugin_provided_feature_set.packager_provided_files,
            self._debian_dir,
            self.binary_packages,
        )

        for package in self._package_states:
            with self.binary_package_context(package) as context:
                if not context.is_auto_generated_package:
                    ppf_result = all_packager_provided_files[package]
                    if ppf_result.auto_installable:
                        context.install_rules.append(
                            PPFInstallRule(
                                context.binary_package,
                                context.substitution,
                                ppf_result.auto_installable,
                            )
                        )
                    context.reserved_packager_provided_files.update(
                        ppf_result.reserved_only
                    )
                self._transform_dpkg_maintscript_helpers_to_snippets()

        return HighLevelManifest(
            self.manifest_path,
            self._mutable_yaml_manifest,
            self._install_rules,
            self.binary_packages,
            self.substitution,
            self._package_states,
            self._dpkg_architecture_variables,
            self._dpkg_arch_query_table,
            self._build_env,
            self._plugin_provided_feature_set,
            self._debian_dir,
        )

    @contextlib.contextmanager
    def binary_package_context(
        self, package_name: str
    ) -> Iterator[PackageTransformationDefinition]:
        if package_name not in self._package_states:
            self._error(
                f'The package "{package_name}" is not present in the debian/control file (could not find'
                f' "Package: {package_name}" in a binary paragraph) nor is it a -dbgsym package for one'
                " for a package in debian/control."
            )
        package_state = self._package_states[package_name]
        self._package_state_stack.append(package_state)
        ps_len = len(self._package_state_stack)
        yield package_state
        if ps_len != len(self._package_state_stack):
            raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
        self._package_state_stack.pop()

    def _dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingParser[TP]:
        t = self._plugin_provided_feature_set.dispatchable_parsers.get(rule_type)
        if t is None:
            raise AssertionError(
                f"Internal error: No dispatching parser for {rule_type.__name__}"
            )
        return t

    @property
    def substitution(self) -> Substitution:
        if self._package_state_stack:
            return self._package_state_stack[-1].substitution
        return self._substitution

    @property
    def current_binary_package_state(self) -> PackageTransformationDefinition:
        if not self._package_state_stack:
            raise RuntimeError("Invalid state: Not in a binary package context")
        return self._package_state_stack[-1]

    @property
    def is_in_binary_package_state(self) -> bool:
        return bool(self._package_state_stack)

    def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
        package_state = self.current_binary_package_state
        for dmh in package_state.dpkg_maintscript_helper_snippets:
            snippet = MaintscriptSnippet(
                definition_source=dmh.definition_source,
                snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
            )
            for script in STD_CONTROL_SCRIPTS:
                package_state.maintscript_snippets[script].append(snippet)

    def normalize_path(
        self,
        path: str,
        definition_source: AttributePath,
        *,
        allow_root_dir_match: bool = False,
    ) -> ExactFileSystemPath:
        try:
            normalized = _normalize_path(path)
        except ValueError:
            self._error(
                f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
                ' package and not use any ".." or "." segments.'
            )
        if normalized == "." and not allow_root_dir_match:
            self._error(
                "Manifests must not change the root directory of the deb file.  Please correct"
                f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
            )
        return ExactFileSystemPath(
            self.substitution.substitute(normalized, definition_source.path)
        )

    def parse_path_or_glob(
        self,
        path_or_glob: str,
        definition_source: AttributePath,
    ) -> MatchRule:
        match_rule = MatchRule.from_path_or_glob(
            path_or_glob, definition_source.path, substitution=self.substitution
        )
        # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
        # so there is no need to check for an exact match on "." like in normalize_path.
        if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
            self._error(
                f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
                f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
                f' something that matches "less" than everything.'
            )
        return match_rule

    def parse_manifest(self) -> HighLevelManifest:
        raise NotImplementedError


class YAMLManifestParser(HighLevelManifestParser):
    def _optional_key(
        self,
        d,
        key,
        attribute_parent_path: AttributePath,
        expected_type=None,
        default_value=None,
    ):
        v = d.get(key)
        if v is None:
            _detect_possible_typo(d, key, attribute_parent_path, False)
            return default_value
        if expected_type is not None:
            return self._ensure_value_is_type(
                v, expected_type, key, attribute_parent_path
            )
        return v

    def _required_key(
        self,
        d,
        key,
        attribute_parent_path: AttributePath,
        expected_type=None,
        extra: Optional[Union[str, Callable[[], str]]] = None,
    ):
        v = d.get(key)
        if v is None:
            _detect_possible_typo(d, key, attribute_parent_path, True)
            if extra is not None:
                msg = extra if isinstance(extra, str) else extra()
                extra_info = " " + msg
            else:
                extra_info = ""
            self._error(
                f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
                f"{extra_info}"
            )

        if expected_type is not None:
            return self._ensure_value_is_type(
                v, expected_type, key, attribute_parent_path
            )
        return v

    def _ensure_value_is_type(
        self,
        v,
        t,
        key: Union[str, int, AttributePath],
        attribute_parent_path: Optional[AttributePath],
    ):
        if v is None:
            return None
        if not isinstance(v, t):
            if isinstance(t, tuple):
                t_msg = "one of: " + ", ".join(x.__name__ for x in t)
            else:
                t_msg = f"a {t.__name__}"
            key_path = (
                key.path
                if isinstance(key, AttributePath)
                else attribute_parent_path[key].path
            )
            self._error(
                f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
            )
        return v

    def parse_manifest_condition(
        self,
        d: Union[str, Mapping[str, Any]],
        orig_definition_source: AttributePath,
    ) -> ManifestCondition:
        parser: DispatchingParser[ManifestCondition] = self._dispatch_parser_table_for(
            ManifestCondition
        )
        return parser.parse(d, orig_definition_source, parser_context=self)

    def _parse_yaml_installations(
        self,
        d: ManifestRootRule,
        orig_definition_source: AttributePath,
    ) -> None:
        raw_installations = d.get("installations")

        if raw_installations is None and MK_INSTALLATIONS not in d:
            # Explicitly empty vs. implicitly empty matters in corner cases, so preserve this
            # fact.
            return

        self._install_rules: List[InstallRule] = []
        parent_path = orig_definition_source[MK_INSTALLATIONS]
        install_rule_parser = self._dispatch_parser_table_for(InstallRule)

        for no, raw_installation in enumerate(raw_installations):
            definition_source = parent_path[no]
            installation_rule = install_rule_parser.parse(
                raw_installation,
                definition_source,
                parser_context=self,
            )
            self._install_rules.append(installation_rule)

    def _parse_yaml_transformations(
        self,
        d: BinaryPackageRule,
        transformations: List[TransformationRule],
        orig_definition_source: AttributePath,
    ) -> None:
        raw_transformations = d.get("transformations", [])
        parent_path = orig_definition_source["transformations"]
        transformation_rule_parser = self._dispatch_parser_table_for(TransformationRule)

        for no, raw_transformation in enumerate(raw_transformations):
            definition_source = parent_path[no]
            transformation = transformation_rule_parser.parse(
                raw_transformation,
                definition_source,
                parser_context=self,
            )
            transformations.append(transformation)

    def _parse_yaml_conffile_management(
        self,
        d: BinaryPackageRule,
        definition_source: AttributePath,
    ) -> None:
        entries = d.get("conffile_management", [])
        package_state = self.current_binary_package_state

        parent_path = definition_source[MK_CONFFILE_MANAGEMENT]
        rule_parser = self._dispatch_parser_table_for(DpkgMaintscriptHelperCommand)

        for no, entry in enumerate(entries):
            path_definition_source = parent_path[no]
            dpkg_script_snippet = rule_parser.parse(
                entry,
                path_definition_source,
                parser_context=self,
            )
            package_state.dpkg_maintscript_helper_snippets.append(dpkg_script_snippet)

    def _parse_clean_after_removal(
        self,
        d: BinaryPackageRule,
        orig_definition_source: AttributePath,
    ) -> None:
        raw_clean_after_removal = d.get("clean_after_removal", [])
        parent_path = orig_definition_source["clean-on-removal"]
        package_state = self.current_binary_package_state

        for no, raw_transformation in enumerate(raw_clean_after_removal):
            definition_source = parent_path[no]
            clean_after_removal_rules = _CLEAN_AFTER_REMOVAL_RULE_PARSER.parse_input(
                raw_transformation,
                definition_source,
                parser_context=self,
            )
            patterns = clean_after_removal_rules["paths"]
            if patterns:
                definition_source.path_hint = patterns[0]
            delete_on = clean_after_removal_rules.get("delete_on") or "purge"
            recurse = clean_after_removal_rules.get("recursive") or False
            ignore_non_empty_dir = (
                clean_after_removal_rules.get("ignore_non_empty_dir") or False
            )
            if delete_on == "purge":
                condition = '[ "$1" = "purge" ]'
            else:
                condition = '[ "$1" = "remove" ]'

            if ignore_non_empty_dir:
                if recurse:
                    self._error(
                        'The "recursive" and "ignore-non-empty-dir" options are mutually exclusive.'
                        f" Both were enabled at the same time in at {definition_source.path}"
                    )
                for pattern in patterns:
                    if not pattern.endswith("/"):
                        self._error(
                            'When ignore-non-empty-dir is True, then all patterns must end with a literal "/"'
                            f' to ensure they only apply to directories. The pattern "{pattern}" at'
                            f" {definition_source.path} did not."
                        )

            substitution = self.substitution
            match_rules = [
                MatchRule.from_path_or_glob(
                    p, definition_source.path, substitution=substitution
                )
                for p in patterns
            ]
            content_lines = [
                f"if {condition}; then\n",
            ]
            for idx, match_rule in enumerate(match_rules):
                original_pattern = patterns[idx]
                if match_rule is MATCH_ANYTHING:
                    self._error(
                        f'Using "{original_pattern}" in a clean rule would trash the system.'
                        f" Please restrict this pattern at {definition_source.path} considerably."
                    )
                is_subdir_match = False
                if isinstance(match_rule, ExactFileSystemPath):
                    matched_directory = (
                        os.path.dirname(match_rule.path)
                        if match_rule.path not in ("/", ".", "./")
                        else match_rule.path
                    )
                    is_subdir_match = True
                else:
                    matched_directory = getattr(match_rule, "directory", None)

                if matched_directory is None:
                    self._error(
                        f'The pattern "{original_pattern}" defined at {definition_source.path} is not'
                        f" trivially anchored in a specific directory. Cowardly refusing to use it"
                        f" in a clean rule as it may trash the system if the pattern is overreaching."
                        f" Please avoid glob characters in the top level directories."
                    )
                assert matched_directory.startswith("./") or matched_directory in (
                    ".",
                    "./",
                    "",
                )
                acceptable_directory = False
                would_have_allowed_direct_match = False
                while matched_directory not in (".", "./", ""):
                    # Our acceptable paths set includes "/var/lib" or "/etc".  We require that the
                    # pattern is either an exact match, in which case it may match directly inside
                    # the acceptable directory OR it is a pattern against a subdirectory of the
                    # acceptable path. As an example:
                    #
                    # /etc/inputrc <-- OK, exact match
                    # /etc/foo/*   <-- OK, subdir match
                    # /etc/*       <-- ERROR, glob directly in the accepted directory.
                    if is_subdir_match and (
                        matched_directory
                        in ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF
                    ):
                        acceptable_directory = True
                        break
                    if (
                        matched_directory
                        in ACCEPTABLE_CLEAN_ON_REMOVAL_FOR_GLOBS_AND_EXACT_MATCHES
                    ):
                        # Special-case: In some directories (such as /var/log), we allow globs directly.
                        # Notably, X11's log files are /var/log/Xorg.*.log
                        acceptable_directory = True
                        break
                    if (
                        matched_directory
                        in ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF
                    ):
                        would_have_allowed_direct_match = True
                        break
                    matched_directory = os.path.dirname(matched_directory)
                    is_subdir_match = True

                if would_have_allowed_direct_match and not acceptable_directory:
                    self._error(
                        f'The pattern "{original_pattern}" defined at {definition_source.path} seems to'
                        " be overreaching. If it has been a path (and not use a glob), the rule would"
                        " have been permitted."
                    )
                elif not acceptable_directory:
                    self._error(
                        f'The pattern or path "{original_pattern}" defined at {definition_source.path} seems to'
                        f' be overreaching or not limited to the set of "known acceptable" directories.'
                    )

                try:
                    shell_escaped_pattern = match_rule.shell_escape_pattern()
                except TypeError:
                    self._error(
                        f'Sorry, the pattern "{original_pattern}" defined at {definition_source.path}'
                        f" is unfortunately not supported by `debputy` for clean-after-removal rules."
                        f" If you can rewrite the rule to something like `/var/log/foo/*.log` or"
                        f' similar "trivial" patterns. You may have to rewrite the pattern the rule '
                        f" into multiple patterns to achieve this.  This restriction is to enable "
                        f' `debputy` to ensure the pattern is correctly executed plus catch "obvious'
                        f' system trashing" patterns. Apologies for the inconvenience.'
                    )

                if ignore_non_empty_dir:
                    cmd = f'    rmdir --ignore-fail-on-non-empty "${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
                elif recurse:
                    cmd = f'    rm -fr ${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
                elif original_pattern.endswith("/"):
                    cmd = f'    rmdir ${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
                else:
                    cmd = f'    rm -f ${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
                content_lines.append(cmd)
            content_lines.append("fi\n")

            snippet = MaintscriptSnippet(definition_source.path, "".join(content_lines))
            package_state.maintscript_snippets["postrm"].append(snippet)

    def _resolve_variables(
        self,
        definitions: DefinitionRule,
        attribute_path: AttributePath,
    ) -> None:
        variables = definitions.get("variables", {})
        if not variables:
            return
        variables_path = attribute_path["variables"]
        resolved_vars = {}
        substitution = self._substitution
        for key, value_raw in variables.items():
            key_path = variables_path[key]
            if not SUBST_VAR_RE.match("{{" + key + "}}"):
                self._error(
                    f"The variable at {key_path.path} has an invalid name and therefore cannot"
                    " be used."
                )
            if substitution.variable_state(key) != VariableNameState.UNDEFINED:
                self._error(
                    f'The variable "{key}" is already reserved/defined. Error triggered by'
                    f" {key_path.path}."
                )
            try:
                value = substitution.substitute(value_raw, key_path.path)
            except DebputySubstitutionError:
                if not resolved_vars:
                    raise
                # See if flushing the variables work
                substitution = substitution.with_extra_substitutions(**resolved_vars)
                resolved_vars = {}
                value = substitution.substitute(value_raw, key_path.path)
            self._declared_variables[key] = key_path
            resolved_vars[key] = value
        self._substitution = substitution.with_extra_substitutions(**resolved_vars)

    def _parse_definitions(
        self,
        d: ManifestRootRule,
        attribute_path: AttributePath,
    ) -> None:
        definitions_raw = d.get("definitions", {})
        definitions_path = attribute_path["definitions"]
        definitions = _DEFINITIONS_RULE_PARSER.parse_input(
            definitions_raw, definitions_path, parser_context=self
        )
        self._resolve_variables(definitions, definitions_path)

    def from_yaml_dict(self, yaml_data) -> "HighLevelManifest":
        attribute_path = AttributePath.root_path()
        parsed_data = _MANIFEST_ROOT_PARSER.parse_input(
            yaml_data, attribute_path, parser_context=self
        )
        manifest_version = parsed_data["manifest_version"]
        if manifest_version not in SUPPORTED_MANIFEST_VERSIONS:
            self._error(
                "Unsupported manifest-version.  This implementation supports the following versions:"
                f' {", ".join(repr(v) for v in SUPPORTED_MANIFEST_VERSIONS)}"'
            )

        self._parse_definitions(parsed_data, attribute_path)
        self._ensure_package_states_is_initialized()
        packages_dict = parsed_data.get("packages", {})
        self._parse_yaml_installations(parsed_data, attribute_path)
        packages_parent_path = attribute_path["packages"]
        for package_name_raw, v in packages_dict.items():
            definition_source = packages_parent_path[package_name_raw]
            package_name = package_name_raw
            if "{{" in package_name:
                package_name = self.substitution.substitute(
                    package_name_raw,
                    definition_source.path,
                )

            with self.binary_package_context(package_name) as package_state:
                if package_state.is_auto_generated_package:
                    # Maybe lift (part) of this restriction.
                    self._error(
                        f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
                        " auto-generated package."
                    )
                parsed = _BINARY_PACKAGE_RULE_PARSER.parse_input(
                    v, definition_source, parser_context=self
                )

                binary_version = parsed.get("binary_version")
                if binary_version is not None:
                    package_state.binary_version = (
                        package_state.substitution.substitute(
                            binary_version,
                            definition_source["binary-version"],
                        )
                    )
                search_dirs = parsed.get("installation_search_dirs")
                if search_dirs is not None:
                    package_state.search_dirs = search_dirs
                self._parse_yaml_transformations(
                    parsed,
                    package_state.transformations,
                    definition_source,
                )
                # Order between _parse_clean_on_removal and _parse_yaml_conffile_management is
                # important. We want the dpkg conffile management rules to happen before the
                # clean _parse_clean_on_removal rules.  Since the latter only affects `postrm`
                # and the order is reversed for `postrm` scripts (among other), we need do
                # _parse_clean_on_removal first to account for the reversing of order.
                self._parse_clean_after_removal(parsed, definition_source)
                self._parse_yaml_conffile_management(parsed, definition_source)
        return self.build_manifest()

    def _parse_manifest(self, fd: Union[IO[bytes], str]) -> HighLevelManifest:
        try:
            data = MANIFEST_YAML.load(fd)
        except YAMLError as e:
            msg = str(e)
            lines = msg.splitlines(keepends=True)
            i = -1
            for i, line in enumerate(lines):
                # Avoid an irrelevant "how do configure the YAML parser" message, which the
                # user cannot use.
                if line.startswith("To suppress this check"):
                    break
            if i > -1 and len(lines) > i + 1:
                lines = lines[:i]
                msg = "".join(lines)
            msg = msg.rstrip()
            msg += (
                f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
                " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
                " (such as indentation rules) in case that is of interest."
            )
            raise ManifestParseException(
                f"Could not parse {self.manifest_path} as a YAML document: {msg}"
            ) from e
        self._mutable_yaml_manifest = MutableYAMLManifest(data)
        return self.from_yaml_dict(data)

    def parse_manifest(
        self,
        *,
        fd: Optional[Union[IO[bytes], str]] = None,
    ) -> HighLevelManifest:
        if fd is None:
            with open(self.manifest_path, "rb") as fd:
                return self._parse_manifest(fd)
        else:
            return self._parse_manifest(fd)
