feat(os): bind bounded restart grants and rollback
This commit is contained in:
parent
20d6c1bd7b
commit
e94c4f608f
3 changed files with 164 additions and 17 deletions
|
|
@ -45,9 +45,11 @@
|
||||||
2. 每个适配器在代码中定义动作类型和参数形状。
|
2. 每个适配器在代码中定义动作类型和参数形状。
|
||||||
3. 节点策略声明允许操作的精确资源。
|
3. 节点策略声明允许操作的精确资源。
|
||||||
4. 变更动作必须匹配当前授权和回滚检查点。
|
4. 变更动作必须匹配当前授权和回滚检查点。
|
||||||
5. 命令退出码只表示执行器观察,目标侧读回才决定 `PASS_100`。
|
5. 变更授权必须由节点策略精确绑定授权编号、动作、资源和回滚检查点,不能只凭请求自行声明。
|
||||||
6. Linux 管理入口与光湖正常入口分离;紧急入口启用必须留下维护回执。
|
6. 重启动作前必须先读回服务为 active;动作失败后执行固定的 `reset-failed → start → active` 恢复链,并在回执中分别记录动作失败和回滚结果。
|
||||||
7. 密钥、令牌和模型 API 凭据保留在节点受保护边界,不进入语言记录或执行请求。
|
7. 命令退出码只表示执行器观察,目标侧读回才决定 `PASS_100`。
|
||||||
|
8. Linux 管理入口与光湖正常入口分离;紧急入口启用必须留下维护回执。
|
||||||
|
9. 密钥、令牌和模型 API 凭据保留在节点受保护边界,不进入语言记录或执行请求。
|
||||||
|
|
||||||
## 京东节点落地顺序
|
## 京东节点落地顺序
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -67,6 +67,16 @@ pub struct ExecutionPolicy {
|
||||||
pub allowed_services: Vec<String>,
|
pub allowed_services: Vec<String>,
|
||||||
pub allow_status: bool,
|
pub allow_status: bool,
|
||||||
pub allow_restart: bool,
|
pub allow_restart: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub mutation_grants: Vec<MutationGrant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct MutationGrant {
|
||||||
|
pub authorization_id: String,
|
||||||
|
pub allowed_action: ActionKind,
|
||||||
|
pub resource: String,
|
||||||
|
pub rollback_checkpoint_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||||
|
|
@ -101,6 +111,8 @@ pub struct ExecutionReceipt {
|
||||||
pub stdout: String,
|
pub stdout: String,
|
||||||
pub stderr: String,
|
pub stderr: String,
|
||||||
pub rollback_checkpoint_id: Option<String>,
|
pub rollback_checkpoint_id: Option<String>,
|
||||||
|
pub rollback_attempted: bool,
|
||||||
|
pub rollback_succeeded: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
|
@ -207,6 +219,7 @@ pub fn compile_request(
|
||||||
"service restart is disabled by policy",
|
"service restart is disabled by policy",
|
||||||
)?;
|
)?;
|
||||||
validate_mutation_references(request)?;
|
validate_mutation_references(request)?;
|
||||||
|
validate_mutation_grant(request, policy)?;
|
||||||
(
|
(
|
||||||
vec!["restart".to_owned(), request.action.resource.clone()],
|
vec!["restart".to_owned(), request.action.resource.clone()],
|
||||||
true,
|
true,
|
||||||
|
|
@ -268,6 +281,29 @@ fn validate_mutation_references(request: &ExecutionRequest) -> Result<(), Bridge
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_mutation_grant(
|
||||||
|
request: &ExecutionRequest,
|
||||||
|
policy: &ExecutionPolicy,
|
||||||
|
) -> Result<(), BridgeError> {
|
||||||
|
let authorization = request
|
||||||
|
.authorization
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| BridgeError::new("mutating action requires authorization"))?;
|
||||||
|
let rollback = request
|
||||||
|
.rollback
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| BridgeError::new("mutating action requires rollback reference"))?;
|
||||||
|
require(
|
||||||
|
policy.mutation_grants.iter().any(|grant| {
|
||||||
|
grant.authorization_id == authorization.authorization_id
|
||||||
|
&& grant.allowed_action == request.action.kind
|
||||||
|
&& grant.resource == request.action.resource
|
||||||
|
&& grant.rollback_checkpoint_id == rollback.checkpoint_id
|
||||||
|
}),
|
||||||
|
"mutation authorization is not granted by the execution policy",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn execute_plan(
|
pub fn execute_plan(
|
||||||
plan: &ExecutionPlan,
|
plan: &ExecutionPlan,
|
||||||
executor: &dyn CommandExecutor,
|
executor: &dyn CommandExecutor,
|
||||||
|
|
@ -288,27 +324,80 @@ pub fn execute_plan(
|
||||||
"compiled plan is not an allowlisted Linux systemd adapter",
|
"compiled plan is not an allowlisted Linux systemd adapter",
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
if plan.action.kind == ActionKind::ServiceRestart {
|
||||||
|
let precheck_arguments = vec!["is-active".to_owned(), plan.action.resource.clone()];
|
||||||
|
let precheck = executor
|
||||||
|
.execute(&plan.program, &precheck_arguments)
|
||||||
|
.map_err(BridgeError::new)?;
|
||||||
|
require(
|
||||||
|
precheck.exit_code == Some(0) && precheck.stdout.trim() == "active",
|
||||||
|
"restart precondition failed: service was not active",
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
let result = executor
|
let result = executor
|
||||||
.execute(&plan.program, &plan.arguments)
|
.execute(&plan.program, &plan.arguments)
|
||||||
.map_err(BridgeError::new)?;
|
.map_err(BridgeError::new)?;
|
||||||
let command_succeeded = result.exit_code == Some(0);
|
let command_succeeded = result.exit_code == Some(0);
|
||||||
|
|
||||||
let (target_state_verified, final_state, stdout, stderr, exit_code) = if plan.action.kind
|
let (
|
||||||
== ActionKind::ServiceRestart
|
target_state_verified,
|
||||||
&& command_succeeded
|
final_state,
|
||||||
{
|
stdout,
|
||||||
|
stderr,
|
||||||
|
exit_code,
|
||||||
|
rollback_attempted,
|
||||||
|
rollback_succeeded,
|
||||||
|
) = if plan.action.kind == ActionKind::ServiceRestart {
|
||||||
let verification_arguments = vec!["is-active".to_owned(), plan.action.resource.clone()];
|
let verification_arguments = vec!["is-active".to_owned(), plan.action.resource.clone()];
|
||||||
let verification = executor
|
let verification = executor
|
||||||
.execute(&plan.program, &verification_arguments)
|
.execute(&plan.program, &verification_arguments)
|
||||||
.map_err(BridgeError::new)?;
|
.map_err(BridgeError::new)?;
|
||||||
let verified = verification.exit_code == Some(0) && verification.stdout.trim() == "active";
|
let verified = command_succeeded
|
||||||
(
|
&& verification.exit_code == Some(0)
|
||||||
verified,
|
&& verification.stdout.trim() == "active";
|
||||||
if verified { "PASS_100" } else { "FAIL_0" }.to_owned(),
|
if verified {
|
||||||
join_observations(&result.stdout, &verification.stdout),
|
(
|
||||||
join_observations(&result.stderr, &verification.stderr),
|
true,
|
||||||
verification.exit_code,
|
"PASS_100".to_owned(),
|
||||||
)
|
join_observations(&result.stdout, &verification.stdout),
|
||||||
|
join_observations(&result.stderr, &verification.stderr),
|
||||||
|
verification.exit_code,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let reset_arguments = vec!["reset-failed".to_owned(), plan.action.resource.clone()];
|
||||||
|
let reset = executor
|
||||||
|
.execute(&plan.program, &reset_arguments)
|
||||||
|
.map_err(BridgeError::new)?;
|
||||||
|
let start_arguments = vec!["start".to_owned(), plan.action.resource.clone()];
|
||||||
|
let start = executor
|
||||||
|
.execute(&plan.program, &start_arguments)
|
||||||
|
.map_err(BridgeError::new)?;
|
||||||
|
let recovery_verification = executor
|
||||||
|
.execute(&plan.program, &verification_arguments)
|
||||||
|
.map_err(BridgeError::new)?;
|
||||||
|
let recovered = reset.exit_code == Some(0)
|
||||||
|
&& start.exit_code == Some(0)
|
||||||
|
&& recovery_verification.exit_code == Some(0)
|
||||||
|
&& recovery_verification.stdout.trim() == "active";
|
||||||
|
(
|
||||||
|
false,
|
||||||
|
"FAIL_0".to_owned(),
|
||||||
|
join_observations(
|
||||||
|
&join_observations(&result.stdout, &verification.stdout),
|
||||||
|
&join_observations(&start.stdout, &recovery_verification.stdout),
|
||||||
|
),
|
||||||
|
join_observations(
|
||||||
|
&join_observations(&result.stderr, &verification.stderr),
|
||||||
|
&join_observations(&reset.stderr, &start.stderr),
|
||||||
|
),
|
||||||
|
verification.exit_code,
|
||||||
|
true,
|
||||||
|
Some(recovered),
|
||||||
|
)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
let verified = command_succeeded
|
let verified = command_succeeded
|
||||||
&& (plan.action.kind != ActionKind::ServiceStatus || result.stdout.trim() == "active");
|
&& (plan.action.kind != ActionKind::ServiceStatus || result.stdout.trim() == "active");
|
||||||
|
|
@ -318,6 +407,8 @@ pub fn execute_plan(
|
||||||
result.stdout,
|
result.stdout,
|
||||||
result.stderr,
|
result.stderr,
|
||||||
result.exit_code,
|
result.exit_code,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -336,6 +427,8 @@ pub fn execute_plan(
|
||||||
stdout,
|
stdout,
|
||||||
stderr,
|
stderr,
|
||||||
rollback_checkpoint_id: plan.rollback_checkpoint_id.clone(),
|
rollback_checkpoint_id: plan.rollback_checkpoint_id.clone(),
|
||||||
|
rollback_attempted,
|
||||||
|
rollback_succeeded,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,12 @@ fn policy() -> ExecutionPolicy {
|
||||||
allowed_services: vec!["guanghu-broadcast-tower.service".to_owned()],
|
allowed_services: vec!["guanghu-broadcast-tower.service".to_owned()],
|
||||||
allow_status: true,
|
allow_status: true,
|
||||||
allow_restart: true,
|
allow_restart: true,
|
||||||
|
mutation_grants: vec![guanghu_execution_bridge::MutationGrant {
|
||||||
|
authorization_id: "AUTH-001".to_owned(),
|
||||||
|
allowed_action: ActionKind::ServiceRestart,
|
||||||
|
resource: "guanghu-broadcast-tower.service".to_owned(),
|
||||||
|
rollback_checkpoint_id: "CHECKPOINT-001".to_owned(),
|
||||||
|
}],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -95,6 +101,23 @@ fn restart_requires_matching_authorization_and_rollback() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restart_rejects_an_authorization_not_bound_by_policy() {
|
||||||
|
let mut request = request(ActionKind::ServiceRestart);
|
||||||
|
request.authorization = Some(AuthorizationReference {
|
||||||
|
authorization_id: "AUTH-INVENTED".to_owned(),
|
||||||
|
allowed_action: ActionKind::ServiceRestart,
|
||||||
|
target_node_id: "JD-FD-PRIMARY".to_owned(),
|
||||||
|
});
|
||||||
|
request.rollback = Some(RollbackReference {
|
||||||
|
checkpoint_id: "CHECKPOINT-001".to_owned(),
|
||||||
|
recovery_action: "restore previous active service state".to_owned(),
|
||||||
|
});
|
||||||
|
|
||||||
|
let error = compile_request(&request, &policy()).expect_err("invented grant must fail");
|
||||||
|
assert!(error.to_string().contains("not granted"));
|
||||||
|
}
|
||||||
|
|
||||||
struct FakeExecutor {
|
struct FakeExecutor {
|
||||||
results: RefCell<VecDeque<CommandResult>>,
|
results: RefCell<VecDeque<CommandResult>>,
|
||||||
calls: RefCell<Vec<Vec<String>>>,
|
calls: RefCell<Vec<Vec<String>>>,
|
||||||
|
|
@ -135,6 +158,11 @@ fn restart_receipt_requires_target_side_active_readback() {
|
||||||
});
|
});
|
||||||
let plan = compile_request(&request, &policy()).expect("compile restart");
|
let plan = compile_request(&request, &policy()).expect("compile restart");
|
||||||
let executor = FakeExecutor::new(vec![
|
let executor = FakeExecutor::new(vec![
|
||||||
|
CommandResult {
|
||||||
|
exit_code: Some(0),
|
||||||
|
stdout: "active".to_owned(),
|
||||||
|
stderr: String::new(),
|
||||||
|
},
|
||||||
CommandResult {
|
CommandResult {
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout: String::new(),
|
stdout: String::new(),
|
||||||
|
|
@ -150,8 +178,10 @@ fn restart_receipt_requires_target_side_active_readback() {
|
||||||
let receipt = execute_plan(&plan, &executor).expect("execute plan");
|
let receipt = execute_plan(&plan, &executor).expect("execute plan");
|
||||||
assert_eq!(receipt.final_state, "PASS_100");
|
assert_eq!(receipt.final_state, "PASS_100");
|
||||||
assert!(receipt.target_state_verified);
|
assert!(receipt.target_state_verified);
|
||||||
assert_eq!(executor.calls.borrow().len(), 2);
|
assert!(!receipt.rollback_attempted);
|
||||||
assert_eq!(executor.calls.borrow()[1][1], "is-active");
|
assert_eq!(executor.calls.borrow().len(), 3);
|
||||||
|
assert_eq!(executor.calls.borrow()[0][1], "is-active");
|
||||||
|
assert_eq!(executor.calls.borrow()[2][1], "is-active");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -168,6 +198,11 @@ fn successful_restart_command_without_active_readback_is_fail_zero() {
|
||||||
});
|
});
|
||||||
let plan = compile_request(&request, &policy()).expect("compile restart");
|
let plan = compile_request(&request, &policy()).expect("compile restart");
|
||||||
let executor = FakeExecutor::new(vec![
|
let executor = FakeExecutor::new(vec![
|
||||||
|
CommandResult {
|
||||||
|
exit_code: Some(0),
|
||||||
|
stdout: "active".to_owned(),
|
||||||
|
stderr: String::new(),
|
||||||
|
},
|
||||||
CommandResult {
|
CommandResult {
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout: String::new(),
|
stdout: String::new(),
|
||||||
|
|
@ -178,11 +213,28 @@ fn successful_restart_command_without_active_readback_is_fail_zero() {
|
||||||
stdout: "inactive".to_owned(),
|
stdout: "inactive".to_owned(),
|
||||||
stderr: String::new(),
|
stderr: String::new(),
|
||||||
},
|
},
|
||||||
|
CommandResult {
|
||||||
|
exit_code: Some(0),
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: String::new(),
|
||||||
|
},
|
||||||
|
CommandResult {
|
||||||
|
exit_code: Some(0),
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: String::new(),
|
||||||
|
},
|
||||||
|
CommandResult {
|
||||||
|
exit_code: Some(0),
|
||||||
|
stdout: "active".to_owned(),
|
||||||
|
stderr: String::new(),
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
let receipt = execute_plan(&plan, &executor).expect("execute plan");
|
let receipt = execute_plan(&plan, &executor).expect("execute plan");
|
||||||
assert_eq!(receipt.final_state, "FAIL_0");
|
assert_eq!(receipt.final_state, "FAIL_0");
|
||||||
assert!(!receipt.target_state_verified);
|
assert!(!receipt.target_state_verified);
|
||||||
|
assert!(receipt.rollback_attempted);
|
||||||
|
assert_eq!(receipt.rollback_succeeded, Some(true));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue