kopiur_kopia/

error.rs

//! Error types for kopia subprocess invocation and JSON parsing.
//!
//! A terminal [`KopiaError`] must carry enough structured detail for the mover
//! to build a `status.failure` block (ADR §4.10): exit code, the last lines of
//! stderr, and a best-effort *error class* derived from kopia's stderr so the
//! controller can decide whether a retry is worthwhile.

use std::fmt;

use crate::humanize::{exit_code_desc, humanize_tail};

/// How many trailing lines of stderr we retain on a failed invocation. Kopia
/// can print a lot of progress to stderr; the tail is where the actual error
/// message lands.
pub const STDERR_TAIL_LINES: usize = 20;

/// A best-effort classification of a kopia failure, derived by inspecting the
/// captured stderr. This is intentionally coarse — it exists to drive the
/// "should we retry?" decision in the mover, not to be exhaustive. Unknown
/// failures map to [`KopiaErrorClass::Unknown`] and are treated as
/// non-retryable by default.
///
/// Classification reads kopia's stderr; the class then drives the retry hint and
/// round-trips through its stable label:
///
/// ```
/// use kopiur_kopia::KopiaErrorClass;
///
/// // A backend down / unreachable error is transient → worth a retry.
/// let class = KopiaErrorClass::classify("ERROR error connecting to repository: dial tcp");
/// assert_eq!(class, KopiaErrorClass::RepositoryUnavailable);
/// assert!(class.is_retryable());
///
/// // A wrong repository password is not retryable without a config change.
/// let auth = KopiaErrorClass::classify("invalid repository password");
/// assert_eq!(auth, KopiaErrorClass::AuthFailure);
/// assert!(!auth.is_retryable());
///
/// // The stable label round-trips through from_label/as_str.
/// assert_eq!(class.as_str(), "RepositoryUnavailable");
/// assert_eq!(KopiaErrorClass::from_label("RepositoryUnavailable"), class);
/// // An unrecognized label degrades to Unknown.
/// assert_eq!(KopiaErrorClass::from_label("bogus"), KopiaErrorClass::Unknown);
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KopiaErrorClass {
    /// Repository could not be reached / opened (network, backend down,
    /// bad endpoint). Typically transient → retry.
    RepositoryUnavailable,
    /// Authentication / password / credential failure (wrong repository
    /// password). Not retryable without a config change.
    AuthFailure,
    /// The storage backend **denied access** to the bucket/container/object
    /// (e.g. S3/B2/GCS "Access Denied", HTTP 403). The credentials usually
    /// authenticate fine but lack permission — or the bucket/path doesn't exist
    /// and the backend masks that as access-denied (RustFS/S3 do this). Not
    /// retryable without a credentials/permission/bucket fix.
    AccessDenied,
    /// The repository **path is not writable by this process** — e.g. a
    /// filesystem repo whose PVC/NFS export is not writable by the operator's
    /// UID ("permission denied" / EACCES when connecting or creating). Not
    /// retryable without fixing ownership/mode.
    PermissionDenied,
    /// The requested source path / snapshot / target was not found.
    NotFound,
    /// A repository lock is held by another writer. Often transient → retry.
    Locked,
    /// Source filesystem error during upload (I/O, prepare failure).
    SourceError,
    /// Anything we could not classify.
    Unknown,
}

impl KopiaErrorClass {
    /// Stable string form for status fields / metrics labels.
    pub fn as_str(&self) -> &'static str {
        match self {
            KopiaErrorClass::RepositoryUnavailable => "RepositoryUnavailable",
            KopiaErrorClass::AuthFailure => "AuthFailure",
            KopiaErrorClass::AccessDenied => "AccessDenied",
            KopiaErrorClass::PermissionDenied => "PermissionDenied",
            KopiaErrorClass::NotFound => "NotFound",
            KopiaErrorClass::Locked => "Locked",
            KopiaErrorClass::SourceError => "SourceError",
            KopiaErrorClass::Unknown => "Unknown",
        }
    }

    /// Inverse of [`as_str`](Self::as_str): reconstruct the class from its stable
    /// label. Used when only the persisted string is available (the controller
    /// reads `result.failure.kopiaErrorClass` from a bootstrap Job's ConfigMap).
    /// An unrecognized label maps to [`KopiaErrorClass::Unknown`].
    pub fn from_label(s: &str) -> KopiaErrorClass {
        match s {
            "RepositoryUnavailable" => KopiaErrorClass::RepositoryUnavailable,
            "AuthFailure" => KopiaErrorClass::AuthFailure,
            "AccessDenied" => KopiaErrorClass::AccessDenied,
            "PermissionDenied" => KopiaErrorClass::PermissionDenied,
            "NotFound" => KopiaErrorClass::NotFound,
            "Locked" => KopiaErrorClass::Locked,
            "SourceError" => KopiaErrorClass::SourceError,
            _ => KopiaErrorClass::Unknown,
        }
    }

    /// A **stable**, volatile-free one-line summary of what this class means and
    /// how to fix it, suitable for a status *condition message*.
    ///
    /// Unlike `KopiaError::to_string` (which embeds the kopia stderr tail — and
    /// thus a per-attempt-random temp filename like `.shards.tmp.<hex>`), this is
    /// byte-identical across repeated failures of the same class. The controller
    /// uses it for the persisted condition so that re-writing an unchanged Failed
    /// status is a true no-op (no resourceVersion bump → no self-triggered
    /// reconcile). The full, volatile detail still goes to the Warning Event.
    pub fn summary(&self) -> &'static str {
        match self {
            KopiaErrorClass::RepositoryUnavailable => {
                "repository backend is unreachable; check the endpoint/network and retry"
            }
            KopiaErrorClass::AuthFailure => {
                "repository password was rejected; check the encryption password Secret \
                 (the KOPIA_PASSWORD key)"
            }
            KopiaErrorClass::AccessDenied => {
                "the storage backend denied access; check the credentials Secret and that the \
                 bucket/container/path exists"
            }
            KopiaErrorClass::PermissionDenied => {
                "repository path is not writable by the operator's UID; fix ownership/mode on the \
                 backing PVC or NFS export"
            }
            KopiaErrorClass::NotFound => {
                "the repository path, snapshot, or target was not found; check the backend \
                 path/prefix"
            }
            KopiaErrorClass::Locked => {
                "a repository lock is held by another writer; it usually clears on retry"
            }
            KopiaErrorClass::SourceError => {
                "could not read the source volume during upload; check the volume and the mover \
                 Job logs"
            }
            KopiaErrorClass::Unknown => {
                "unknown repository error; see the mover Job logs and status.failure"
            }
        }
    }

    /// Whether re-running the same operation later might succeed without any
    /// configuration change. This is the operator's default retry hint; the
    /// caller may override it with policy.
    pub fn is_retryable(&self) -> bool {
        matches!(
            self,
            KopiaErrorClass::RepositoryUnavailable
                | KopiaErrorClass::Locked
                | KopiaErrorClass::SourceError
        )
    }

    /// Best-effort classification from captured stderr text. Matches against
    /// substrings kopia is observed to emit (kopia 0.23). Order matters: more
    /// specific checks come first.
    pub fn classify(stderr: &str) -> KopiaErrorClass {
        let s = stderr.to_ascii_lowercase();
        if s.contains("invalid repository password")
            || s.contains("incorrect password")
            || s.contains("unable to derive")
        {
            KopiaErrorClass::AuthFailure
        } else if s.contains("access denied")
            || s.contains("accessdenied")
            || s.contains("forbidden")
            || s.contains("not authorized")
        {
            // Backend authorization (e.g. S3 "Access Denied"). Checked before the
            // generic permission/not-found arms because the backend phrasing is
            // specific and the fix (creds/bucket) is distinct.
            KopiaErrorClass::AccessDenied
        } else if s.contains("permission denied")
            || s.contains("operation not permitted")
            || s.contains("eacces")
        {
            // Local repo path not writable by our UID. Checked before SourceError
            // (which used to absorb "permission denied" and wrongly mark it
            // retryable) and before NotFound.
            KopiaErrorClass::PermissionDenied
        } else if s.contains("repository is locked")
            || s.contains("another process")
            || s.contains("lock")
        {
            KopiaErrorClass::Locked
        } else if s.contains("no such file or directory")
            || s.contains("not found")
            || s.contains("does not exist")
            || s.contains("unable to find snapshot")
            // kopia's `repo.ErrRepositoryNotInitialized` ("repository not initialized
            // in the provided storage") — an *empty* backend with no kopia repo at the
            // prefix. The CLI wraps it as `error connecting to repository: repository
            // not initialized ...`, so it MUST be matched here, ahead of the
            // RepositoryUnavailable arm, or an uninitialized repo is misread as an
            // unreachable backend. Classifying it `NotFound` is what lets the mover
            // surface the actionable `RepositoryNotInitialized` outcome (set
            // `spec.create.enabled: true`) instead of a misleading "backend
            // unreachable, retry".
            || s.contains("not initialized")
        {
            KopiaErrorClass::NotFound
        } else if s.contains("error connecting to repository")
            || s.contains("unable to open repository")
            // kopia's other open-failure phrasing, and the index/blob download
            // failures behind it: a remote backend dropping connections while
            // the index loads (issue #477). AFTER the AuthFailure/NotFound arms,
            // so a failed open with a specific cause (bad password, not
            // initialized) keeps its specific class.
            || s.contains("failed to open repository")
            || s.contains("error loading indexes")
            || s.contains("unexpected eof")
            || s.contains("connection refused")
            || s.contains("dial tcp")
            || s.contains("no route to host")
            || s.contains("timeout")
            // DNS resolution failure (Go's net resolver: `lookup <host> …: no
            // such host`). Deliberately AFTER the NotFound arm above: none of
            // its substrings ("no such file or directory", "not found", …)
            // match this phrasing, and keeping it here means it can never
            // shadow a genuine missing-path classification.
            || s.contains("no such host")
            // TLS / certificate failures reaching the backend (Go: `tls: …`,
            // `x509: certificate signed by unknown authority`, `certificate
            // has expired`). The endpoint is unreachable-as-configured — the
            // fix is the endpoint/CA/trust config, and the repository gate
            // should engage. AFTER the AuthFailure/AccessDenied arms above so
            // "certificate" can never capture a credential/authorization
            // message (those arms match their own specific phrasings first).
            || s.contains("tls:")
            || s.contains("x509")
            || s.contains("certificate")
        {
            KopiaErrorClass::RepositoryUnavailable
        } else if s.contains("upload error") || s.contains("failed to prepare source") {
            KopiaErrorClass::SourceError
        } else {
            KopiaErrorClass::Unknown
        }
    }
}

/// Whether a [`KopiaErrorClass::NotFound`] connect failure is the backend
/// reporting a *genuinely uninitialized* repository (kopia's
/// `ErrRepositoryNotInitialized`: "repository not initialized in the provided
/// storage") rather than a *missing path / mount* ("no such file or directory",
/// "does not exist").
///
/// Both phrasings classify as [`KopiaErrorClass::NotFound`] (so first-bootstrap
/// `create` still fires for either), but they mean very different things for an
/// already-`Ready` repository under the health probe:
///
/// * genuine "not initialized" ⇒ the backend answered and the kopia format blob
///   is gone — a candidate *vanished repository* (`RepositoryVanished`).
/// * a missing path / mount ⇒ the PVC isn't bound or the export moved — a
///   *backend/mount fault* (`BackendReachable=False`), NOT a wipe. Recreating
///   here would be catastrophic, so the two must never be conflated.
///
/// Returns `false` for any non-`NotFound` stderr; callers gate on the class first.
///
/// ```
/// use kopiur_kopia::notfound_is_uninitialized;
/// assert!(notfound_is_uninitialized("repository not initialized in the provided storage"));
/// assert!(!notfound_is_uninitialized("open /repo/kopia.repository: no such file or directory"));
/// ```
pub fn notfound_is_uninitialized(stderr: &str) -> bool {
    stderr.to_ascii_lowercase().contains("not initialized")
}

/// Whether a **successful** (exit 0) `snapshot create` that produced no JSON on
/// stdout was kopia deliberately declining to write a manifest because the
/// source is byte-identical to the previous snapshot.
///
/// kopia's message is:
///
/// ```text
///  Not saving snapshot because no files have been changed since previous snapshot
/// ```
///
/// This is gated by the **retention**-policy knob `ignoreIdenticalSnapshots`
/// (`*OptionalBool`, kopia default `false`). With it on, `snapshot create
/// --json` exits 0, writes **nothing** to stdout, and says why only on stderr —
/// which read as a hard `EmptyOutput` failure and terminally failed the
/// `Snapshot` CR (#351).
///
/// Matched on the stable middle of the sentence rather than the whole string:
/// kopia prefixes it with a leading space and has reworded the surrounding
/// phrasing across releases, but "no files have been changed" has been
/// constant. Case-insensitive for the same reason.
///
/// ```
/// use kopiur_kopia::snapshot_skipped_unchanged;
/// assert!(snapshot_skipped_unchanged(
///     " Not saving snapshot because no files have been changed since previous snapshot"
/// ));
/// assert!(!snapshot_skipped_unchanged("Snapshotting app@host:/pvc/data ..."));
/// assert!(!snapshot_skipped_unchanged(""));
/// ```
pub fn snapshot_skipped_unchanged(stderr: &str) -> bool {
    stderr
        .to_ascii_lowercase()
        .contains("no files have been changed")
}

impl fmt::Display for KopiaErrorClass {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.as_str())
    }
}

/// Errors produced while invoking kopia or parsing its `--json` output.
///
/// Each variant's `Display` is actionable — it names the operation, the failure,
/// and (for non-zero exits) the error class plus the stderr tail — so it can be
/// dropped straight into a `status.failure` block (ADR §4.10):
///
/// ```
/// use kopiur_kopia::{KopiaError, KopiaErrorClass};
///
/// let err = KopiaError::NonZeroExit {
///     args: "snapshot create".into(),
///     code: Some(1),
///     class: KopiaErrorClass::Locked,
///     stderr_tail: "repository is locked by another process".into(),
/// };
/// assert_eq!(
///     err.to_string(),
///     "kopia `snapshot create` failed (exit code 1, class Locked): \
///      repository is locked by another process",
/// );
/// // The class drives the retry decision; the stderr tail is recoverable.
/// assert_eq!(err.class(), KopiaErrorClass::Locked);
/// assert!(err.class().is_retryable());
/// assert_eq!(err.stderr_tail(), Some("repository is locked by another process"));
///
/// // A timeout names the args and elapsed seconds, and maps to a retryable class.
/// let to = KopiaError::Timeout { args: "maintenance run --full".into(), seconds: 3600 };
/// assert_eq!(to.to_string(), "kopia `maintenance run --full` timed out after 3600s");
/// assert_eq!(to.class(), KopiaErrorClass::RepositoryUnavailable);
/// ```
#[derive(thiserror::Error, Debug)]
pub enum KopiaError {
    /// Writing kopia's OUTPUT to the consumer failed. kopia spawned and ran
    /// fine; the sink — a `streamExec` restore's exec stdin — broke.
    ///
    /// Separate from [`KopiaError::Spawn`] because it used to BE `Spawn`, and
    /// that misattribution actively cost debugging time: a `streamExec` restore
    /// whose consumer pipe broke reported "failed to spawn kopia binary `kopia`:
    /// broken pipe", sending the reader after a missing binary that was in fact
    /// running. The three fallible halves of the streaming copy — the sink write,
    /// the stderr read, and the child wait — are not interchangeable and must not
    /// share one error.
    #[error("could not stream kopia `{args}` output into the consumer: {source}")]
    OutputSink {
        /// The kopia argv whose output was being streamed.
        args: String,
        /// Underlying I/O error from the SINK, not from kopia.
        #[source]
        source: std::io::Error,
    },

    /// The kopia binary could not be spawned at all (missing binary, not
    /// executable, fork failure). Carries the OS error.
    #[error("failed to spawn kopia binary `{binary}`: {source}")]
    Spawn {
        /// Path we attempted to execute.
        binary: String,
        /// Underlying I/O error.
        #[source]
        source: std::io::Error,
    },

    /// kopia ran but exited with a non-zero status. Carries everything needed
    /// to build a `status.failure` block.
    ///
    /// `Display` renders a clean exit code (no `Some(1)` `Debug` leak) and a
    /// **humanized** stderr extract (progress noise dropped, volatile temp-path
    /// fragments stripped) — this is what reaches Warning Events,
    /// `status.failure.message`, and logs. The full raw tail is preserved in the
    /// `stderr_tail` field (surfaced via `status.failure.stderrTail`) for
    /// debugging; read it with [`stderr_tail`](Self::stderr_tail).
    #[error(
        "kopia `{args}` failed ({}, class {class}): {}",
        exit_code_desc(.code),
        humanize_tail(.stderr_tail)
    )]
    NonZeroExit {
        /// The subcommand + args that were run (for diagnostics; secrets are
        /// passed via env, never argv).
        args: String,
        /// Process exit code, if one was reported (None if killed by signal).
        code: Option<i32>,
        /// Best-effort error classification from stderr.
        class: KopiaErrorClass,
        /// The last [`STDERR_TAIL_LINES`] lines of stderr, joined by newlines.
        stderr_tail: String,
    },

    /// kopia exited 0 (or produced output) but the JSON could not be parsed
    /// into the expected type — usually a kopia version skew.
    #[error("failed to parse kopia JSON output for `{context}`: {source}")]
    Json {
        /// What we were trying to parse (e.g. "snapshot create result").
        context: String,
        /// The serde error.
        #[source]
        source: serde_json::Error,
    },

    /// The process feeding kopia's stdin did not finish successfully, so the
    /// snapshot was deliberately aborted before kopia wrote any manifest.
    ///
    /// Distinct from [`KopiaError::NonZeroExit`]: kopia itself was fine — it was
    /// killed on purpose. Surfacing that difference is what tells an operator
    /// "your dump command failed" rather than "the backup tool failed".
    #[error(
        "snapshot aborted: {detail}{}",
        if stderr_tail.is_empty() { String::new() } else { format!(" (kopia stderr: {stderr_tail})") }
    )]
    StdinProducerFailed {
        /// What went wrong with the producer, already free of any streamed data.
        detail: String,
        /// Bounded tail of kopia's own stderr, for context.
        stderr_tail: String,
    },

    /// The stdin producer exited **successfully** having written ZERO bytes, so
    /// the snapshot was aborted before any manifest was written.
    ///
    /// Deliberately a failure rather than an empty snapshot. A producer's exit
    /// status cannot be trusted to catch this on its own: `pg_dump` exits 0
    /// against an instance it cannot see any databases in, `mysqldump` exits 0
    /// when its credentials grant nothing, and a `sh -c 'a | b'` pipeline reports
    /// only `b`'s status. Committing the result would put a zero-byte "restore
    /// point" into the repository that retention keeps and a restore would
    /// cheerfully write over a live database with.
    #[error(
        "snapshot aborted: the stdin producer exited 0 but wrote no data, so nothing was saved. \
         Fix: make sure `workloadExec.command` writes to stdout (a dump with no visible \
         databases can print nothing); in a `sh -c` pipeline add `set -o pipefail`. Args: \
         {args}{}",
        if stderr_tail.is_empty() { String::new() } else { format!(" (kopia stderr: {stderr_tail})") }
    )]
    StdinProducerWroteNothing {
        /// The kopia argv, for correlation with the Job log.
        args: String,
        /// Bounded tail of kopia's own stderr. NEVER the producer's stdout — that
        /// is the user's data and this crate offers no path by which it could
        /// reach an error message.
        stderr_tail: String,
    },

    /// We expected a JSON object/array on stdout but found none (kopia printed
    /// only progress / nothing) even though it exited **0**.
    ///
    /// `stderr_tail` is what makes this diagnosable. Without it the variant
    /// carried neither a reason nor an exit code, so a kopia that exited
    /// cleanly and explained itself on stderr was indistinguishable from a
    /// kopia that said nothing at all — the whole of #351.
    #[error("no JSON output found on stdout for `{context}`: {stderr_tail}")]
    EmptyOutput {
        /// What we were trying to parse.
        context: String,
        /// The trailing stderr lines, which is where kopia explains a silent
        /// success. Empty when kopia really did say nothing.
        stderr_tail: String,
    },

    /// The operation exceeded its configured timeout and was killed.
    #[error("kopia `{args}` timed out after {seconds}s")]
    Timeout {
        /// The subcommand + args that were run.
        args: String,
        /// The timeout that elapsed, in seconds.
        seconds: u64,
    },
}

impl KopiaError {
    /// The error class for this error, for retry decisions and metrics. Spawn,
    /// JSON-parse, empty-output, and timeout errors map to a fixed class;
    /// non-zero exits carry their own classification.
    pub fn class(&self) -> KopiaErrorClass {
        match self {
            KopiaError::NonZeroExit { class, .. } => *class,
            // A spawn failure is environmental (bad image / missing binary) —
            // retrying the same pod won't help, treat as Unknown/non-retryable.
            KopiaError::Spawn { .. } => KopiaErrorClass::Unknown,
            KopiaError::Json { .. } | KopiaError::EmptyOutput { .. } => KopiaErrorClass::Unknown,
            // The repository was never at fault — the consumer's pipe was. Re-running
            // re-runs the same consumer command, so this is terminal for the operator.
            KopiaError::OutputSink { .. } => KopiaErrorClass::Unknown,
            // Timeouts are usually a slow backend → worth a retry.
            KopiaError::Timeout { .. } => KopiaErrorClass::RepositoryUnavailable,
            // Both stdin-producer failures: the repository was never at fault —
            // the user's dump command was. Re-running the same command would
            // reproduce the same failure, or the same empty output, so the fix
            // is in the workload or the policy, never in another attempt.
            //
            // They share ONE arm deliberately. These two comments used to sit on
            // separate arms and drifted: one kept the honest scope below while
            // the other went on claiming outright that the failure is not
            // retried. Same class, same reasoning, one arm — so the claim cannot
            // rot on only one half again.
            //
            // NOTE what this class does and does NOT do. It makes the failure
            // TERMINAL for the operator: `MoverError::retry_recommended()` is
            // false, so `status.failure.retryRecommended` says so and the
            // controller does not treat the run as retryable. It does NOT stop
            // the Kubernetes Job from retrying — the mover exits non-zero and the
            // Job's own `backoffLimit` (default 2) may schedule replacement pods
            // that re-exec the producer's command. Preventing that needs the Job
            // DELETED, the way the wedged-pod path in `controller::snapshot`
            // does; nothing reads `retryRecommended` today. Tracked separately.
            KopiaError::StdinProducerFailed { .. }
            | KopiaError::StdinProducerWroteNothing { .. } => KopiaErrorClass::Unknown,
        }
    }

    /// The trailing stderr lines, if this error captured any.
    pub fn stderr_tail(&self) -> Option<&str> {
        match self {
            KopiaError::NonZeroExit { stderr_tail, .. } => Some(stderr_tail.as_str()),
            // An exit-0-with-no-JSON keeps its stderr too, so `status.failure`
            // shows kopia's own words instead of a bare "class Unknown".
            KopiaError::EmptyOutput { stderr_tail, .. } if !stderr_tail.is_empty() => {
                Some(stderr_tail.as_str())
            }
            // Bounded kopia stderr only. The producer's own diagnostics ride in
            // `detail`, and its STDOUT — the backup data — is never captured at all.
            KopiaError::StdinProducerFailed { stderr_tail, .. }
            | KopiaError::StdinProducerWroteNothing { stderr_tail, .. }
                if !stderr_tail.is_empty() =>
            {
                Some(stderr_tail.as_str())
            }
            _ => None,
        }
    }
}

/// Keep only the last `STDERR_TAIL_LINES` non-empty-trimmed lines of a stderr
/// blob, joined by newlines. Used when building a [`KopiaError::NonZeroExit`].
pub(crate) fn tail_lines(stderr: &str) -> String {
    let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
    let start = lines.len().saturating_sub(STDERR_TAIL_LINES);
    lines[start..].join("\n")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn classify_known_patterns() {
        assert_eq!(
            KopiaErrorClass::classify("ERROR error connecting to repository: dial tcp ..."),
            KopiaErrorClass::RepositoryUnavailable
        );
        assert_eq!(
            KopiaErrorClass::classify("invalid repository password"),
            KopiaErrorClass::AuthFailure
        );
        assert_eq!(
            KopiaErrorClass::classify("lstat /nope: no such file or directory"),
            KopiaErrorClass::NotFound
        );
        assert_eq!(
            KopiaErrorClass::classify("repository is locked by another process"),
            KopiaErrorClass::Locked
        );
        assert_eq!(
            KopiaErrorClass::classify("upload error: unsupported source"),
            KopiaErrorClass::SourceError
        );
        assert_eq!(
            KopiaErrorClass::classify("something totally unexpected"),
            KopiaErrorClass::Unknown
        );
    }

    /// Issue #477, verbatim: a remote backend dropping connections while the
    /// index loads. The repository failed to OPEN, a transient backend fault,
    /// so it must be retryable. As `Unknown` the batch-delete fallback would
    /// re-open the same struggling repository once per member.
    #[test]
    fn classify_index_load_failures_as_repository_unavailable() {
        let issue_477 = "failed to open repository: unable to create shared content manager: \
             error loading indexes: error downloading indexes: error loading index blob \
             xn45_abc: getContent: unable to complete GetBlob(xn45_abc,0,-1) despite 10 \
             retries: unexpected EOF";
        assert_eq!(
            KopiaErrorClass::classify(issue_477),
            KopiaErrorClass::RepositoryUnavailable
        );
        assert_eq!(
            KopiaErrorClass::classify("error reading blob: unexpected EOF"),
            KopiaErrorClass::RepositoryUnavailable
        );
        // The earlier arms still win on a failed open with a specific cause.
        assert_eq!(
            KopiaErrorClass::classify("failed to open repository: invalid repository password"),
            KopiaErrorClass::AuthFailure
        );
        assert_eq!(
            KopiaErrorClass::classify("failed to open repository: repository not initialized"),
            KopiaErrorClass::NotFound
        );
    }

    #[test]
    fn classify_dns_and_tls_failures_as_repository_unavailable() {
        // DNS resolution failure (Go's net resolver phrasing): the backend
        // hostname doesn't resolve — the backend is unreachable, the gate
        // should engage (#345).
        assert_eq!(
            KopiaErrorClass::classify(
                "unable to open repository: lookup minio.storage.svc on 10.96.0.10:53: \
                 no such host"
            ),
            KopiaErrorClass::RepositoryUnavailable
        );
        assert_eq!(
            KopiaErrorClass::classify("dial tcp: lookup s3.example.com: no such host"),
            KopiaErrorClass::RepositoryUnavailable
        );
        // TLS handshake / certificate trust failures: unreachable-as-configured.
        assert_eq!(
            KopiaErrorClass::classify("tls: failed to verify certificate"),
            KopiaErrorClass::RepositoryUnavailable
        );
        assert_eq!(
            KopiaErrorClass::classify("x509: certificate signed by unknown authority"),
            KopiaErrorClass::RepositoryUnavailable
        );
        assert_eq!(
            KopiaErrorClass::classify("certificate has expired or is not yet valid"),
            KopiaErrorClass::RepositoryUnavailable
        );
        // All of these are transient/config-external → retryable.
        assert!(KopiaErrorClass::RepositoryUnavailable.is_retryable());
    }

    #[test]
    fn widened_arms_do_not_shadow_existing_classifications() {
        // The new DNS/TLS substrings sit in the RepositoryUnavailable arm,
        // AFTER every more-specific arm — existing classifications must be
        // byte-for-byte unchanged.
        assert_eq!(
            KopiaErrorClass::classify("invalid repository password"),
            KopiaErrorClass::AuthFailure
        );
        assert_eq!(
            KopiaErrorClass::classify("access denied"),
            KopiaErrorClass::AccessDenied
        );
        assert_eq!(
            KopiaErrorClass::classify("no such file or directory"),
            KopiaErrorClass::NotFound
        );
        // A certificate-flavored message that ALSO carries an auth/authz
        // phrasing still classifies by the earlier, more specific arm.
        assert_eq!(
            KopiaErrorClass::classify("certificate auth: access denied"),
            KopiaErrorClass::AccessDenied
        );
        // "not initialized" (empty backend) keeps winning over the connect
        // prefix — the ordering the NotFound arm's comment documents.
        assert_eq!(
            KopiaErrorClass::classify(
                "error connecting to repository: repository not initialized in the \
                 provided storage"
            ),
            KopiaErrorClass::NotFound
        );
    }

    #[test]
    fn classify_uninitialized_repository_as_not_found() {
        // Regression: connecting to an empty backend (no kopia repo at the prefix)
        // makes kopia emit `repo.ErrRepositoryNotInitialized`, which the CLI wraps
        // with its generic connect prefix. That prefix matches the
        // RepositoryUnavailable arm, so without an explicit "not initialized" check
        // the empty-bucket case was misclassified as a transient unreachable backend
        // — and the mover's `not_initialized()` path (keyed on NotFound) never fired,
        // so the operator saw "backend unreachable; retry" instead of the actionable
        // "set spec.create.enabled: true". It must classify as NotFound.
        assert_eq!(
            KopiaErrorClass::classify(
                "ERROR error connecting to repository: repository not initialized in the \
                 provided storage"
            ),
            KopiaErrorClass::NotFound
        );
        // Bare form (no connect prefix) classifies the same way.
        assert_eq!(
            KopiaErrorClass::classify("repository not initialized in the provided storage"),
            KopiaErrorClass::NotFound
        );
        // NotFound is non-retryable: the fix is a spec change, not a blind retry.
        assert!(!KopiaErrorClass::NotFound.is_retryable());
    }

    #[test]
    fn notfound_distinguishes_uninitialized_from_missing_path() {
        // A genuinely empty backend (format blob absent) → uninitialized: the health
        // probe may treat this as a candidate "vanished" repository.
        assert!(notfound_is_uninitialized(
            "ERROR error connecting to repository: repository not initialized in the \
             provided storage"
        ));
        // A missing path / unbound mount also classifies NotFound, but is a backend/
        // mount fault — NOT an empty repository. Must NOT read as uninitialized, so the
        // probe never misreads a mis-mounted volume as a wipe (and never nudges a recreate).
        assert!(!notfound_is_uninitialized(
            "open /repo/kopia.repository: no such file or directory"
        ));
        assert!(!notfound_is_uninitialized("stat /mnt/nas: does not exist"));
        // Both still classify as NotFound (so first-bootstrap `create` fires for either).
        assert_eq!(
            KopiaErrorClass::classify("open /repo/kopia.repository: no such file or directory"),
            KopiaErrorClass::NotFound
        );
    }

    #[test]
    fn classify_access_denied_and_permission_denied() {
        // The exact RustFS/S3 message we observed live (bucket missing, masked as
        // Access Denied) must classify as AccessDenied, not Unknown.
        assert_eq!(
            KopiaErrorClass::classify(
                "can't connect to storage: error retrieving storage config from bucket \
                 \"kopiur\": Access Denied"
            ),
            KopiaErrorClass::AccessDenied
        );
        assert_eq!(
            KopiaErrorClass::classify("403 Forbidden"),
            KopiaErrorClass::AccessDenied
        );
        // Filesystem repo path not writable by our UID → PermissionDenied, NOT
        // the old SourceError (which marked it retryable).
        assert_eq!(
            KopiaErrorClass::classify("unable to create directory /repo: permission denied"),
            KopiaErrorClass::PermissionDenied
        );
        assert_eq!(
            KopiaErrorClass::classify("open /repo/kopia.repository: operation not permitted"),
            KopiaErrorClass::PermissionDenied
        );
    }

    #[test]
    fn from_label_roundtrips_every_variant() {
        for c in [
            KopiaErrorClass::RepositoryUnavailable,
            KopiaErrorClass::AuthFailure,
            KopiaErrorClass::AccessDenied,
            KopiaErrorClass::PermissionDenied,
            KopiaErrorClass::NotFound,
            KopiaErrorClass::Locked,
            KopiaErrorClass::SourceError,
            KopiaErrorClass::Unknown,
        ] {
            assert_eq!(KopiaErrorClass::from_label(c.as_str()), c);
        }
        assert_eq!(
            KopiaErrorClass::from_label("not-a-real-class"),
            KopiaErrorClass::Unknown
        );
    }

    #[test]
    fn summary_is_stable_and_volatile_free() {
        // Every class yields a non-empty, stable summary with no per-attempt
        // volatile content (the temp-filename suffix kopia emits in stderr must
        // never leak into the condition message — that volatility is what caused
        // the reconcile hot-loop).
        for c in [
            KopiaErrorClass::RepositoryUnavailable,
            KopiaErrorClass::AuthFailure,
            KopiaErrorClass::AccessDenied,
            KopiaErrorClass::PermissionDenied,
            KopiaErrorClass::NotFound,
            KopiaErrorClass::Locked,
            KopiaErrorClass::SourceError,
            KopiaErrorClass::Unknown,
        ] {
            let s = c.summary();
            assert!(!s.is_empty());
            assert!(
                !s.contains(".shards"),
                "summary leaks a volatile temp path: {s}"
            );
            assert!(
                !s.contains(".tmp"),
                "summary leaks a volatile temp path: {s}"
            );
            // Stable across calls (it returns a 'static str, but assert intent).
            assert_eq!(s, c.summary());
        }
        // The PermissionDenied summary is the actionable one for the reported bug.
        assert!(
            KopiaErrorClass::PermissionDenied
                .summary()
                .contains("not writable")
        );
    }

    #[test]
    fn retryable_classification() {
        assert!(KopiaErrorClass::RepositoryUnavailable.is_retryable());
        assert!(KopiaErrorClass::Locked.is_retryable());
        assert!(!KopiaErrorClass::AuthFailure.is_retryable());
        assert!(!KopiaErrorClass::AccessDenied.is_retryable());
        assert!(!KopiaErrorClass::PermissionDenied.is_retryable());
        assert!(!KopiaErrorClass::NotFound.is_retryable());
        assert!(!KopiaErrorClass::Unknown.is_retryable());
    }

    #[test]
    fn tail_keeps_last_lines() {
        let blob: String = (0..50)
            .map(|i| format!("line {i}\n"))
            .collect::<Vec<_>>()
            .join("");
        let tail = tail_lines(&blob);
        let kept: Vec<&str> = tail.lines().collect();
        assert_eq!(kept.len(), STDERR_TAIL_LINES);
        assert_eq!(*kept.last().unwrap(), "line 49");
    }

    #[test]
    fn error_class_propagation() {
        let e = KopiaError::NonZeroExit {
            args: "snapshot create".into(),
            code: Some(1),
            class: KopiaErrorClass::Locked,
            stderr_tail: "repository is locked".into(),
        };
        assert_eq!(e.class(), KopiaErrorClass::Locked);
        assert_eq!(e.stderr_tail(), Some("repository is locked"));
    }
}