feat(pncc): replay terminal lifecycle failures
Human-Responsibility: ICE-GL∞ / 冰朔 Persona-Author: ICE-P-ZY001 / 铸渊 Execution-Runtime: Codex macOS Development-ID: DEV-20260810-014 Authorization-Scope: GH-PNCC local runtime and REPO-014 publication Source-Anchor: UI and execution limb deferred
This commit is contained in:
铸渊 / ICE-P-ZY001
parent
b8c3fcf3d8
commit
abc7dbd625
10 changed files with 381 additions and 43 deletions
|
|
@ -328,7 +328,21 @@ pub struct PersonaLifecycleCommandReceipt {
|
|||
pub request_id: String,
|
||||
pub request_fingerprint: String,
|
||||
pub replayed: bool,
|
||||
pub outcome: &'static str,
|
||||
pub lifecycle: serde_json::Value,
|
||||
pub failure: Option<PersonaLifecycleFailureReceipt>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PersonaLifecycleFailureReceipt {
|
||||
pub schema: String,
|
||||
pub session_id: String,
|
||||
pub persona_id: String,
|
||||
pub repository_path: String,
|
||||
pub error_code: String,
|
||||
pub terminal_event_hash: String,
|
||||
pub attribution: PersonaAttribution,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
|
|
@ -353,7 +367,15 @@ struct PersistedPersonaLifecycleReceipt {
|
|||
request_id: String,
|
||||
request_fingerprint: String,
|
||||
lifecycle_receipt_hash: String,
|
||||
#[serde(default = "completed_lifecycle_outcome")]
|
||||
outcome: String,
|
||||
lifecycle: serde_json::Value,
|
||||
#[serde(default)]
|
||||
failure: Option<PersonaLifecycleFailureReceipt>,
|
||||
}
|
||||
|
||||
fn completed_lifecycle_outcome() -> String {
|
||||
"COMPLETED".into()
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
|
|
@ -2596,6 +2618,66 @@ fn lifecycle_request_identity(
|
|||
Ok((request_id, request_fingerprint, session_id, repository))
|
||||
}
|
||||
|
||||
fn persisted_lifecycle_payload_bytes(
|
||||
persisted: &PersistedPersonaLifecycleReceipt,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
match persisted.outcome.as_str() {
|
||||
"COMPLETED" if persisted.failure.is_none() && !persisted.lifecycle.is_null() => {
|
||||
serde_json::to_vec(&persisted.lifecycle)
|
||||
.map_err(|error| format!("PERSONA_LIFECYCLE_RECEIPT_HASH_INPUT_FAILED: {error}"))
|
||||
}
|
||||
"FAILED" if persisted.lifecycle.is_null() => serde_json::to_vec(
|
||||
persisted
|
||||
.failure
|
||||
.as_ref()
|
||||
.ok_or("PERSONA_LIFECYCLE_FAILURE_RECEIPT_MISSING")?,
|
||||
)
|
||||
.map_err(|error| format!("PERSONA_LIFECYCLE_RECEIPT_HASH_INPUT_FAILED: {error}")),
|
||||
_ => Err("PERSONA_LIFECYCLE_RECEIPT_OUTCOME_INVALID".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn persisted_lifecycle_identity_matches(
|
||||
persisted: &PersistedPersonaLifecycleReceipt,
|
||||
session_id: &str,
|
||||
persona_id: &str,
|
||||
) -> bool {
|
||||
match persisted.outcome.as_str() {
|
||||
"COMPLETED" => {
|
||||
persisted
|
||||
.lifecycle
|
||||
.get("sessionId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(session_id)
|
||||
&& persisted
|
||||
.lifecycle
|
||||
.get("personaId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(persona_id)
|
||||
}
|
||||
"FAILED" => persisted.failure.as_ref().is_some_and(|failure| {
|
||||
failure.schema == "hololake.pncc-lifecycle-failure-receipt/v1"
|
||||
&& failure.session_id == session_id
|
||||
&& failure.persona_id == persona_id
|
||||
}),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_failure_code(error: &str) -> String {
|
||||
let candidate = error.split([':', ';']).next().unwrap_or_default().trim();
|
||||
if !candidate.is_empty()
|
||||
&& candidate.len() <= MAX_ID_BYTES
|
||||
&& candidate
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_')
|
||||
{
|
||||
candidate.into()
|
||||
} else {
|
||||
"PERSONA_LIFECYCLE_OPERATION_FAILED".into()
|
||||
}
|
||||
}
|
||||
|
||||
fn verified_lifecycle_replay(
|
||||
runtime_root: &Path,
|
||||
request_id: &str,
|
||||
|
|
@ -2621,9 +2703,9 @@ fn verified_lifecycle_replay(
|
|||
if persisted.request_fingerprint != request_fingerprint {
|
||||
return Err("PERSONA_LIFECYCLE_REQUEST_CONFLICT".into());
|
||||
}
|
||||
let lifecycle_bytes = serde_json::to_vec(&persisted.lifecycle)
|
||||
.map_err(|error| format!("PERSONA_LIFECYCLE_RECEIPT_HASH_INPUT_FAILED: {error}"))?;
|
||||
if hex_digest(&lifecycle_bytes) != persisted.lifecycle_receipt_hash {
|
||||
if hex_digest(&persisted_lifecycle_payload_bytes(&persisted)?)
|
||||
!= persisted.lifecycle_receipt_hash
|
||||
{
|
||||
return Err("PERSONA_LIFECYCLE_RECEIPT_HASH_MISMATCH".into());
|
||||
}
|
||||
let record = load_session_record(runtime_root, session_id)?;
|
||||
|
|
@ -2639,23 +2721,18 @@ fn verified_lifecycle_replay(
|
|||
return Err("PERSONA_LIFECYCLE_RECEIPT_SESSION_BINDING_MISMATCH".into());
|
||||
}
|
||||
let events = verify_event_journal(runtime_root, &record)?;
|
||||
if record.state != "DORMANT"
|
||||
let expected_state = if persisted.outcome == "FAILED" {
|
||||
"DORMANT_AFTER_FAILURE"
|
||||
} else {
|
||||
"DORMANT"
|
||||
};
|
||||
if record.state != expected_state
|
||||
|| events.last().map(|event| event.kind.as_str()) != Some("DORMANT")
|
||||
|| primary_lease_held_by_session(runtime_root, &record)?
|
||||
{
|
||||
return Err("PERSONA_LIFECYCLE_REQUEST_INCOMPLETE_REQUIRES_RECOVERY".into());
|
||||
}
|
||||
if persisted
|
||||
.lifecycle
|
||||
.get("sessionId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
!= Some(session_id)
|
||||
|| persisted
|
||||
.lifecycle
|
||||
.get("personaId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
!= Some(record.persona_id.as_str())
|
||||
{
|
||||
if !persisted_lifecycle_identity_matches(&persisted, session_id, &record.persona_id) {
|
||||
return Err("PERSONA_LIFECYCLE_RECEIPT_IDENTITY_MISMATCH".into());
|
||||
}
|
||||
Ok(Some(PersonaLifecycleCommandReceipt {
|
||||
|
|
@ -2663,7 +2740,13 @@ fn verified_lifecycle_replay(
|
|||
request_id: request_id.to_string(),
|
||||
request_fingerprint: request_fingerprint.to_string(),
|
||||
replayed: true,
|
||||
outcome: if persisted.outcome == "FAILED" {
|
||||
"FAILED"
|
||||
} else {
|
||||
"COMPLETED"
|
||||
},
|
||||
lifecycle: persisted.lifecycle,
|
||||
failure: persisted.failure,
|
||||
}))
|
||||
}
|
||||
|
||||
|
|
@ -2726,22 +2809,12 @@ fn inspect_lifecycle_request_at(
|
|||
if persisted.request_fingerprint != request_fingerprint {
|
||||
return Err("PERSONA_LIFECYCLE_REQUEST_CONFLICT".into());
|
||||
}
|
||||
let lifecycle_bytes = serde_json::to_vec(&persisted.lifecycle)
|
||||
.map_err(|error| format!("PERSONA_LIFECYCLE_RECEIPT_HASH_INPUT_FAILED: {error}"))?;
|
||||
if hex_digest(&lifecycle_bytes) != persisted.lifecycle_receipt_hash {
|
||||
if hex_digest(&persisted_lifecycle_payload_bytes(&persisted)?)
|
||||
!= persisted.lifecycle_receipt_hash
|
||||
{
|
||||
return Err("PERSONA_LIFECYCLE_RECEIPT_HASH_MISMATCH".into());
|
||||
}
|
||||
if persisted
|
||||
.lifecycle
|
||||
.get("sessionId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
!= Some(session_id.as_str())
|
||||
|| persisted
|
||||
.lifecycle
|
||||
.get("personaId")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
!= Some(record.persona_id.as_str())
|
||||
{
|
||||
if !persisted_lifecycle_identity_matches(&persisted, &session_id, &record.persona_id) {
|
||||
return Err("PERSONA_LIFECYCLE_RECEIPT_IDENTITY_MISMATCH".into());
|
||||
}
|
||||
let complete = record.request_id.as_deref() == Some(request_id.as_str())
|
||||
|
|
@ -2751,7 +2824,12 @@ fn inspect_lifecycle_request_at(
|
|||
let unbound = record.request_id.is_none()
|
||||
&& record.request_fingerprint.is_none()
|
||||
&& record.lifecycle_receipt_hash.is_none();
|
||||
let safely_dormant = record.state == "DORMANT"
|
||||
let expected_state = if persisted.outcome == "FAILED" {
|
||||
"DORMANT_AFTER_FAILURE"
|
||||
} else {
|
||||
"DORMANT"
|
||||
};
|
||||
let safely_dormant = record.state == expected_state
|
||||
&& events.last().map(|event| event.kind.as_str()) == Some("DORMANT")
|
||||
&& !lease_held;
|
||||
let (status, safe_to_bind_receipt) = if complete && safely_dormant {
|
||||
|
|
@ -2814,6 +2892,83 @@ fn recover_lifecycle_request_at(
|
|||
.ok_or_else(|| "PERSONA_LIFECYCLE_RECEIPT_RECOVERY_LOST_SESSION".into())
|
||||
}
|
||||
|
||||
fn persist_terminal_lifecycle_failure(
|
||||
runtime_root: &Path,
|
||||
request_id: &str,
|
||||
request_fingerprint: &str,
|
||||
session_id: &str,
|
||||
repository: &Path,
|
||||
error: &str,
|
||||
) -> Result<Option<PersonaLifecycleCommandReceipt>, String> {
|
||||
let session_dir = session_directory(runtime_root, session_id)?;
|
||||
if !session_dir.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut record = load_session_record(runtime_root, session_id)?;
|
||||
let recorded_repository = Path::new(&record.repository_path)
|
||||
.canonicalize()
|
||||
.map_err(|cause| format!("PERSONA_REPOSITORY_UNAVAILABLE: {cause}"))?;
|
||||
if recorded_repository != repository {
|
||||
return Err("PERSONA_LIFECYCLE_REQUEST_SESSION_IDENTITY_MISMATCH".into());
|
||||
}
|
||||
let events = verify_event_journal(runtime_root, &record)?;
|
||||
let terminal_event = events.last().ok_or("PERSONA_EVENT_CHAIN_EMPTY")?;
|
||||
let (_, observed_head) = exact_repository(repository)?;
|
||||
let safely_terminal = record.state == "DORMANT_AFTER_FAILURE"
|
||||
&& terminal_event.kind == "DORMANT"
|
||||
&& !primary_lease_held_by_session(runtime_root, &record)?
|
||||
&& observed_head == record.git_head
|
||||
&& require_clean_repository(repository).is_ok();
|
||||
if !safely_terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
if record.request_id.is_some()
|
||||
|| record.request_fingerprint.is_some()
|
||||
|| record.lifecycle_receipt_hash.is_some()
|
||||
{
|
||||
return Err("PERSONA_LIFECYCLE_FAILURE_BINDING_ALREADY_PRESENT".into());
|
||||
}
|
||||
let failure = PersonaLifecycleFailureReceipt {
|
||||
schema: "hololake.pncc-lifecycle-failure-receipt/v1".into(),
|
||||
session_id: session_id.into(),
|
||||
persona_id: record.persona_id.clone(),
|
||||
repository_path: record.repository_path.clone(),
|
||||
error_code: lifecycle_failure_code(error),
|
||||
terminal_event_hash: terminal_event.event_hash.clone(),
|
||||
attribution: record.attribution.clone(),
|
||||
};
|
||||
let lifecycle_receipt_hash = hex_digest(
|
||||
&serde_json::to_vec(&failure)
|
||||
.map_err(|cause| format!("PERSONA_LIFECYCLE_RECEIPT_HASH_INPUT_FAILED: {cause}"))?,
|
||||
);
|
||||
write_json_file(
|
||||
&session_dir.join("lifecycle-receipt.json"),
|
||||
&PersistedPersonaLifecycleReceipt {
|
||||
schema: "hololake.pncc-persisted-lifecycle-receipt/v1".into(),
|
||||
request_id: request_id.into(),
|
||||
request_fingerprint: request_fingerprint.into(),
|
||||
lifecycle_receipt_hash: lifecycle_receipt_hash.clone(),
|
||||
outcome: "FAILED".into(),
|
||||
lifecycle: serde_json::Value::Null,
|
||||
failure: Some(failure.clone()),
|
||||
},
|
||||
"PERSONA_LIFECYCLE_RECEIPT",
|
||||
)?;
|
||||
record.request_id = Some(request_id.into());
|
||||
record.request_fingerprint = Some(request_fingerprint.into());
|
||||
record.lifecycle_receipt_hash = Some(lifecycle_receipt_hash);
|
||||
write_session_record(runtime_root, &record)?;
|
||||
Ok(Some(PersonaLifecycleCommandReceipt {
|
||||
schema: "hololake.pncc-lifecycle-command-receipt/v1",
|
||||
request_id: request_id.into(),
|
||||
request_fingerprint: request_fingerprint.into(),
|
||||
replayed: false,
|
||||
outcome: "FAILED",
|
||||
lifecycle: serde_json::Value::Null,
|
||||
failure: Some(failure),
|
||||
}))
|
||||
}
|
||||
|
||||
fn run_idempotent_lifecycle_at<F>(
|
||||
runtime_root: &Path,
|
||||
input: PersonaLifecycleRunInput,
|
||||
|
|
@ -2835,14 +2990,29 @@ where
|
|||
)? {
|
||||
return Ok(replay);
|
||||
}
|
||||
let lifecycle = run_lifecycle_at(
|
||||
let lifecycle = match run_lifecycle_at(
|
||||
runtime_root,
|
||||
input,
|
||||
&session_id,
|
||||
wake_timestamp,
|
||||
operation_timestamp,
|
||||
run_fact,
|
||||
)?;
|
||||
) {
|
||||
Ok(lifecycle) => lifecycle,
|
||||
Err(error) => {
|
||||
if let Some(receipt) = persist_terminal_lifecycle_failure(
|
||||
runtime_root,
|
||||
&request_id,
|
||||
&request_fingerprint,
|
||||
&session_id,
|
||||
&repository,
|
||||
&error,
|
||||
)? {
|
||||
return Ok(receipt);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let lifecycle = serde_json::to_value(lifecycle)
|
||||
.map_err(|error| format!("PERSONA_LIFECYCLE_RECEIPT_SERIALIZATION_FAILED: {error}"))?;
|
||||
let lifecycle_bytes = serde_json::to_vec(&lifecycle)
|
||||
|
|
@ -2861,7 +3031,9 @@ where
|
|||
request_id: request_id.clone(),
|
||||
request_fingerprint: request_fingerprint.clone(),
|
||||
lifecycle_receipt_hash: lifecycle_receipt_hash.clone(),
|
||||
outcome: "COMPLETED".into(),
|
||||
lifecycle: lifecycle.clone(),
|
||||
failure: None,
|
||||
},
|
||||
"PERSONA_LIFECYCLE_RECEIPT",
|
||||
)?;
|
||||
|
|
@ -2874,7 +3046,9 @@ where
|
|||
request_id,
|
||||
request_fingerprint,
|
||||
replayed: false,
|
||||
outcome: "COMPLETED",
|
||||
lifecycle,
|
||||
failure: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -3680,6 +3854,96 @@ mod tests {
|
|||
assert!(error.contains("PERSONA_LIFECYCLE_RECEIPT_BINDING_NOT_SAFE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persists_and_replays_the_same_structured_terminal_failure_without_rerunning_the_organ() {
|
||||
let repo = persona_repo();
|
||||
let runtime = tempfile::TempDir::new().unwrap();
|
||||
let input = lifecycle_fact_input(repo.path());
|
||||
let original_head = head(repo.path());
|
||||
let first = run_idempotent_lifecycle_at(
|
||||
runtime.path(),
|
||||
input.clone(),
|
||||
"2026-08-11T00:00:00.000Z",
|
||||
"2026-08-11T00:00:01.000Z",
|
||||
|runtime_root, input, timestamp| {
|
||||
run_fact_task_at(runtime_root, input, timestamp, |_, _| {
|
||||
Err("provider-secret-detail-must-not-persist".into())
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let replay = run_idempotent_lifecycle_at(
|
||||
runtime.path(),
|
||||
input,
|
||||
"2026-08-11T00:00:02.000Z",
|
||||
"2026-08-11T00:00:03.000Z",
|
||||
|_, _, _| panic!("a terminal failure replay must not rerun the organ"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(first.outcome, "FAILED");
|
||||
assert!(!first.replayed);
|
||||
assert!(first.lifecycle.is_null());
|
||||
assert_eq!(
|
||||
first.failure.as_ref().unwrap().error_code,
|
||||
"PERSONA_MODEL_INFERENCE_FAILED"
|
||||
);
|
||||
assert_eq!(replay.outcome, "FAILED");
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(
|
||||
replay.failure.as_ref().unwrap().terminal_event_hash,
|
||||
first.failure.as_ref().unwrap().terminal_event_hash
|
||||
);
|
||||
assert_eq!(head(repo.path()), original_head);
|
||||
let persisted = fs::read_to_string(
|
||||
session_directory(
|
||||
runtime.path(),
|
||||
first.failure.as_ref().unwrap().session_id.as_str(),
|
||||
)
|
||||
.unwrap()
|
||||
.join("lifecycle-receipt.json"),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!persisted.contains("provider-secret-detail-must-not-persist"));
|
||||
assert!(!runtime.path().join("leases/ICE-P-ZY001.json").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safely_recovers_an_unbound_terminal_failure_receipt() {
|
||||
let repo = persona_repo();
|
||||
let runtime = tempfile::TempDir::new().unwrap();
|
||||
let input = lifecycle_fact_input(repo.path());
|
||||
let failed = run_idempotent_lifecycle_at(
|
||||
runtime.path(),
|
||||
input.clone(),
|
||||
"2026-08-11T00:00:00.000Z",
|
||||
"2026-08-11T00:00:01.000Z",
|
||||
|runtime_root, input, timestamp| {
|
||||
run_fact_task_at(runtime_root, input, timestamp, |_, _| {
|
||||
Err("bounded provider failure".into())
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let session_id = failed.failure.as_ref().unwrap().session_id.clone();
|
||||
let mut record = load_session_record(runtime.path(), &session_id).unwrap();
|
||||
record.request_id = None;
|
||||
record.request_fingerprint = None;
|
||||
record.lifecycle_receipt_hash = None;
|
||||
write_session_record(runtime.path(), &record).unwrap();
|
||||
|
||||
let inspection = inspect_lifecycle_request_at(runtime.path(), &input).unwrap();
|
||||
assert_eq!(inspection.status, "SAFE_BIND_PERSISTED_RECEIPT");
|
||||
assert!(inspection.safe_to_bind_receipt);
|
||||
let recovered = recover_lifecycle_request_at(runtime.path(), &input).unwrap();
|
||||
assert_eq!(recovered.outcome, "FAILED");
|
||||
assert!(recovered.replayed);
|
||||
assert_eq!(
|
||||
recovered.failure.unwrap().error_code,
|
||||
"PERSONA_MODEL_INFERENCE_FAILED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn independently_promotes_only_the_current_verified_structured_checkpoint() {
|
||||
let repo = persona_repo();
|
||||
|
|
|
|||
Loading…
Reference in a new issue