fix(guanghu-os): supervise JD master child services

This commit is contained in:
冰朔 2026-09-03 11:45:13 +08:00
commit 7836f3775a
3 changed files with 113 additions and 17 deletions

View file

@ -0,0 +1,51 @@
# JD-FD-PRIMARY 光湖主控子进程监督修复 · 2026-09-03
状态:`SOURCE_IMPLEMENTED · PHYSICAL_REBOOT_PENDING`
## trigger
冰朔在当前 Codex 任务明确要求修复京东第五域服务器并继续完成剩余工程。实时读回发现 GRUB 默认仍为
`guanghu-jd-master-20260816`,但当前 boot id `755bb84d-1a79-42f3-ac8b-bedec9382725` 的 PID 1 是
Ubuntu `/sbin/init`。
## emergence
服务器保存的 `FAILED-BOOT-STATE.json` 证明 2026-08-28 光湖主控曾以自身 PID 1 启动,随后因
`runtime_process_exited:app-hub` 按设计进入 Linux 救援。原主控把任何一个长期子进程的单次退出直接视为
整机主控失败;而同一 app-hub 在 Ubuntu systemd 中采用 `Restart=always`,当前服务和隔离端口复测均可运行。
本次修复让主控 PID 1 自己承担有界子进程监督:每个子进程在 60 秒窗口内允许最多 5 次自动重启;超限后
监督包装器退出,原有 `runtime_watch → fatal → Linux rescue` 仍然生效。服务端口检查增加 5 秒恢复宽限,
运行日志从易失 `/run` 移到服务器回执根 `runtime-logs/`,使下一次失败可追踪。
## lock
```text
source_commit=PENDING
source_sha256=cf92c313a81c90bfe208b499f588b72dbded8b07a6c170d4122d719f3c98456f
target=JD-FD-PRIMARY
installed_before_sha256=a58f80535e0fb305b5b956cbeca66e4026fe63309424673b66bb0871c9feeca9
rollback=/usr/local/libexec/guanghu/guanghu-master-init.before-20260903-child-supervision
linux_rescue=gnulinux-simple-9e4550a0-452b-4f28-b5a5-d5364aa450f6
physical_result=PENDING
```
## why
语言系统主控不能因为可恢复的单个服务瞬时退出就把整个主控权交还完整 Ubuntu也不能无限重启掩盖持续故障。
有界自动恢复与保留救援回路同时满足主控韧性和失败关闭。
## rejected
- 未修复原因就直接重复重启;
- 删除 Linux 或取消救援入口;
- 无限重启子进程并掩盖持续失败;
- 只改服务器、不保留可追溯源码与回滚;
- 把源码测试通过冒充物理重启成功。
## sources
- JD `/guanghu/recovery/JD-FD-PRIMARY-master-20260816/FAILED-BOOT-STATE.json`
- JD `/usr/local/libexec/guanghu/guanghu-master-init` SHA-256 `a58f8053…`
- REPO-014 `product-source/hololake-platform/guanghu-os/scripts/guanghu-master-init.sh`
- 当前任务自然语言瞄点 `000101` 及本轮冰朔直接修复要求

View file

@ -11,6 +11,7 @@ readonly LINUX_RESCUE_ENTRY=gnulinux-simple-9e4550a0-452b-4f28-b5a5-d5364aa450f6
readonly RELEASE_ID=guanghu-master-20260816.2 readonly RELEASE_ID=guanghu-master-20260816.2
readonly STATE_ROOT=/run/guanghu/master readonly STATE_ROOT=/run/guanghu/master
readonly RECEIPT_ROOT=/guanghu/recovery/JD-FD-PRIMARY-master-20260816 readonly RECEIPT_ROOT=/guanghu/recovery/JD-FD-PRIMARY-master-20260816
readonly LOG_ROOT=${RECEIPT_ROOT}/runtime-logs
readonly HLCC=/opt/guanghu/architecture-releases/3d11ac75bea8cf08b5f223fed86ab3cd999ad2fd/server-tools/hololake-code-channel/jd-candidate/hlcc-bootstrap.py readonly HLCC=/opt/guanghu/architecture-releases/3d11ac75bea8cf08b5f223fed86ab3cd999ad2fd/server-tools/hololake-code-channel/jd-candidate/hlcc-bootstrap.py
readonly APP_HUB=/opt/guanghu/architecture-releases/333cd222c53d7d162218543cd167bdda4f8efb22/server-tools/jd-app-hub/server.js readonly APP_HUB=/opt/guanghu/architecture-releases/333cd222c53d7d162218543cd167bdda4f8efb22/server-tools/jd-app-hub/server.js
readonly AI_DISCOVERY=/opt/guanghu/ai-discovery/server.js readonly AI_DISCOVERY=/opt/guanghu/ai-discovery/server.js
@ -72,23 +73,62 @@ wait_http() {
fatal "service_not_ready:${name}:last_http_${code:-none}" fatal "service_not_ready:${name}:last_http_${code:-none}"
} }
supervise_root() {
local name=$1; shift
local failures=0 window_start now code
window_start=$(date +%s)
while :; do
log "CHILD_START ${name} failures=${failures}"
if "$@" >>"${LOG_ROOT}/${name}.log" 2>&1; then code=0; else code=$?; fi
now=$(date +%s)
if (( now - window_start > 60 )); then failures=0; window_start=$now; fi
((failures+=1))
log "CHILD_EXIT ${name} code=${code} failures=${failures}"
(( failures <= 5 )) || return 1
sleep 1
done
}
supervise_guanghu() {
local name=$1; shift
local failures=0 window_start now code
window_start=$(date +%s)
while :; do
log "CHILD_START ${name} owner=guanghu failures=${failures}"
if /usr/bin/setpriv --reuid=998 --regid=998 --init-groups --inh-caps=-all --reset-env \
/usr/bin/env HOME=/var/lib/guanghu USER=guanghu LOGNAME=guanghu \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin "$@" \
>>"${LOG_ROOT}/${name}.log" 2>&1; then code=0; else code=$?; fi
now=$(date +%s)
if (( now - window_start > 60 )); then failures=0; window_start=$now; fi
((failures+=1))
log "CHILD_EXIT ${name} code=${code} failures=${failures}"
(( failures <= 5 )) || return 1
sleep 1
done
}
start_root() { start_root() {
local name=$1; shift local name=$1; shift
log "START ${name}" supervise_root "$name" "$@" &
"$@" >>"${STATE_ROOT}/${name}.log" 2>&1 &
CHILDREN+=("$!:$name") CHILDREN+=("$!:$name")
} }
start_guanghu() { start_guanghu() {
local name=$1; shift local name=$1; shift
log "START ${name} owner=guanghu" supervise_guanghu "$name" "$@" &
/usr/bin/setpriv --reuid=998 --regid=998 --init-groups --inh-caps=-all --reset-env \
/usr/bin/env HOME=/var/lib/guanghu USER=guanghu LOGNAME=guanghu \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin "$@" \
>>"${STATE_ROOT}/${name}.log" 2>&1 &
CHILDREN+=("$!:$name") CHILDREN+=("$!:$name")
} }
require_listen() {
local name=$1 port=$2
for _ in 1 2 3 4 5; do
listen_ready "$port" && return 0
sleep 1
done
fatal "runtime_${name}_lost"
}
verify_identity() { verify_identity() {
local observed cmdline root_source local observed cmdline root_source
observed=$(tr A-F a-f </sys/class/dmi/id/product_uuid | tr -d '\r\n') observed=$(tr A-F a-f </sys/class/dmi/id/product_uuid | tr -d '\r\n')
@ -104,9 +144,9 @@ verify_identity() {
prepare_runtime() { prepare_runtime() {
mount -o remount,rw / mount -o remount,rw /
mkdir -p "$STATE_ROOT" "$RECEIPT_ROOT" /run/sshd /run/systemd/resolve mkdir -p "$STATE_ROOT" "$RECEIPT_ROOT" "$LOG_ROOT" /run/sshd /run/systemd/resolve
install -d -o guanghu -g guanghu -m 0700 "$PNCC_STATE_ROOT" install -d -o guanghu -g guanghu -m 0700 "$PNCC_STATE_ROOT"
chmod 0700 "$STATE_ROOT" "$RECEIPT_ROOT" chmod 0700 "$STATE_ROOT" "$RECEIPT_ROOT" "$LOG_ROOT"
chmod 1777 /tmp chmod 1777 /tmp
mountpoint -q /proc || mount -t proc proc /proc mountpoint -q /proc || mount -t proc proc /proc
mountpoint -q /sys || mount -t sysfs sysfs /sys mountpoint -q /sys || mount -t sysfs sysfs /sys
@ -183,12 +223,12 @@ runtime_watch() {
pid=${item%%:*}; name=${item#*:} pid=${item%%:*}; name=${item#*:}
kill -0 "$pid" 2>/dev/null || fatal "runtime_process_exited:${name}" kill -0 "$pid" 2>/dev/null || fatal "runtime_process_exited:${name}"
done done
listen_ready 22 || fatal "runtime_sshd_lost" require_listen sshd 22
listen_ready 3340 || fatal "runtime_repository_lost" require_listen repository 3340
listen_ready 8088 || fatal "runtime_projection_lost" require_listen projection 8088
listen_ready 3922 || fatal "runtime_navigation_lost" require_listen navigation 3922
listen_ready 3923 || fatal "runtime_pncc_lost" require_listen pncc 3923
listen_ready 3940 || fatal "runtime_hololake_release_lost" require_listen hololake_release 3940
done done
} }

View file

@ -26,10 +26,15 @@ grep -Fq 'wait_http code-projection http://127.0.0.1:8088/code/ 200,303' "$subje
grep -Fq 'wait_http navigation-bridge http://127.0.0.1:3922/v1/anchor 200' "$subject" grep -Fq 'wait_http navigation-bridge http://127.0.0.1:3922/v1/anchor 200' "$subject"
grep -Fq 'wait_http pncc-runtime http://127.0.0.1:3923/health 200' "$subject" grep -Fq 'wait_http pncc-runtime http://127.0.0.1:3923/health 200' "$subject"
grep -Fq '\"pncc\":\"RESIDENT_BOUND_CARRIER_SEPARATE\"' "$subject" grep -Fq '\"pncc\":\"RESIDENT_BOUND_CARRIER_SEPARATE\"' "$subject"
grep -Fq 'listen_ready 3923 || fatal "runtime_pncc_lost"' "$subject" grep -Fq 'require_listen pncc 3923' "$subject"
grep -Fq 'start_root hololake-release-bridge "$HOLOLAKE_RELEASE_BRIDGE"' "$subject" grep -Fq 'start_root hololake-release-bridge "$HOLOLAKE_RELEASE_BRIDGE"' "$subject"
grep -Fq 'wait_http hololake-release-broadcast http://127.0.0.1:3940/health 200 30' "$subject" grep -Fq 'wait_http hololake-release-broadcast http://127.0.0.1:3940/health 200 30' "$subject"
grep -Fq 'listen_ready 3940 || fatal "runtime_hololake_release_lost"' "$subject" grep -Fq 'require_listen hololake_release 3940' "$subject"
grep -Fq 'readonly LOG_ROOT=${RECEIPT_ROOT}/runtime-logs' "$subject"
grep -Fq 'supervise_root "$name" "$@" &' "$subject"
grep -Fq 'supervise_guanghu "$name" "$@" &' "$subject"
grep -Fq '(( failures <= 5 )) || return 1' "$subject"
grep -Fq 'require_listen projection 8088' "$subject"
if grep -Eq '(^|[[:space:]])(systemd|/sbin/init)([[:space:]]|$)' "$subject"; then if grep -Eq '(^|[[:space:]])(systemd|/sbin/init)([[:space:]]|$)' "$subject"; then
echo 'full Linux init must remain dormant' >&2 echo 'full Linux init must remain dormant' >&2
exit 1 exit 1