fix(guanghu-os): supervise JD master child services

2026-09-03 11:45:13 +08:00 · 2026-09-03 11:45:13 +08:00 · 7836f3775a
commit 7836f3775a
parent 60435c0ac4
3 changed files with 113 additions and 17 deletions
--- a/product-source/hololake-platform/guanghu-os/deployments/JD-FD-PRIMARY/JD-MASTER-CHILD-SUPERVISION-REPAIR-20260903.hldp
+++ b/product-source/hololake-platform/guanghu-os/deployments/JD-FD-PRIMARY/JD-MASTER-CHILD-SUPERVISION-REPAIR-20260903.hldp
@ -0,0 +1,51 @@
+# JD-FD-PRIMARY 光湖主控子进程监督修复 · 2026-09-03
+
+状态：`SOURCE_IMPLEMENTED · PHYSICAL_REBOOT_PENDING`
+
+## trigger
+
+冰朔在当前 Codex 任务明确要求修复京东第五域服务器并继续完成剩余工程。实时读回发现 GRUB 默认仍为
+`guanghu-jd-master-20260816`，但当前 boot id `755bb84d-1a79-42f3-ac8b-bedec9382725` 的 PID 1 是
+Ubuntu `/sbin/init`。
+
+## emergence
+
+服务器保存的 `FAILED-BOOT-STATE.json` 证明 2026-08-28 光湖主控曾以自身 PID 1 启动，随后因
+`runtime_process_exited:app-hub` 按设计进入 Linux 救援。原主控把任何一个长期子进程的单次退出直接视为
+整机主控失败；而同一 app-hub 在 Ubuntu systemd 中采用 `Restart=always`，当前服务和隔离端口复测均可运行。
+
+本次修复让主控 PID 1 自己承担有界子进程监督：每个子进程在 60 秒窗口内允许最多 5 次自动重启；超限后
+监督包装器退出，原有 `runtime_watch → fatal → Linux rescue` 仍然生效。服务端口检查增加 5 秒恢复宽限，
+运行日志从易失 `/run` 移到服务器回执根 `runtime-logs/`，使下一次失败可追踪。
+
+## lock
+
+```text
+source_commit=PENDING
+source_sha256=cf92c313a81c90bfe208b499f588b72dbded8b07a6c170d4122d719f3c98456f
+target=JD-FD-PRIMARY
+installed_before_sha256=a58f80535e0fb305b5b956cbeca66e4026fe63309424673b66bb0871c9feeca9
+rollback=/usr/local/libexec/guanghu/guanghu-master-init.before-20260903-child-supervision
+linux_rescue=gnulinux-simple-9e4550a0-452b-4f28-b5a5-d5364aa450f6
+physical_result=PENDING
+```
+
+## why
+
+语言系统主控不能因为可恢复的单个服务瞬时退出就把整个主控权交还完整 Ubuntu；也不能无限重启掩盖持续故障。
+有界自动恢复与保留救援回路同时满足主控韧性和失败关闭。
+
+## rejected
+
+- 未修复原因就直接重复重启；
+- 删除 Linux 或取消救援入口；
+- 无限重启子进程并掩盖持续失败；
+- 只改服务器、不保留可追溯源码与回滚；
+- 把源码测试通过冒充物理重启成功。
+
+## sources
+
+- JD `/guanghu/recovery/JD-FD-PRIMARY-master-20260816/FAILED-BOOT-STATE.json`
+- JD `/usr/local/libexec/guanghu/guanghu-master-init` SHA-256 `a58f8053…`
+- REPO-014 `product-source/hololake-platform/guanghu-os/scripts/guanghu-master-init.sh`
+- 当前任务自然语言瞄点 `000101` 及本轮冰朔直接修复要求
--- a/product-source/hololake-platform/guanghu-os/scripts/guanghu-master-init.sh
+++ b/product-source/hololake-platform/guanghu-os/scripts/guanghu-master-init.sh
@ -11,6 +11,7 @@ readonly LINUX_RESCUE_ENTRY=gnulinux-simple-9e4550a0-452b-4f28-b5a5-d5364aa450f6
 readonly RELEASE_ID=guanghu-master-20260816.2
 readonly STATE_ROOT=/run/guanghu/master
 readonly RECEIPT_ROOT=/guanghu/recovery/JD-FD-PRIMARY-master-20260816
+readonly LOG_ROOT=${RECEIPT_ROOT}/runtime-logs
 readonly HLCC=/opt/guanghu/architecture-releases/3d11ac75bea8cf08b5f223fed86ab3cd999ad2fd/server-tools/hololake-code-channel/jd-candidate/hlcc-bootstrap.py
 readonly APP_HUB=/opt/guanghu/architecture-releases/333cd222c53d7d162218543cd167bdda4f8efb22/server-tools/jd-app-hub/server.js
 readonly AI_DISCOVERY=/opt/guanghu/ai-discovery/server.js
@ -72,23 +73,62 @@ wait_http() {
  fatal "service_not_ready:${name}:last_http_${code:-none}"
 }

+supervise_root() {
+  local name=$1; shift
+  local failures=0 window_start now code
+  window_start=$(date +%s)
+  while :; do
+    log "CHILD_START ${name} failures=${failures}"
+    if "$@" >>"${LOG_ROOT}/${name}.log" 2>&1; then code=0; else code=$?; fi
+    now=$(date +%s)
+    if (( now - window_start > 60 )); then failures=0; window_start=$now; fi
+    ((failures+=1))
+    log "CHILD_EXIT ${name} code=${code} failures=${failures}"
+    (( failures <= 5 )) || return 1
+    sleep 1
+  done
+}
+
+supervise_guanghu() {
+  local name=$1; shift
+  local failures=0 window_start now code
+  window_start=$(date +%s)
+  while :; do
+    log "CHILD_START ${name} owner=guanghu failures=${failures}"
+    if /usr/bin/setpriv --reuid=998 --regid=998 --init-groups --inh-caps=-all --reset-env \
+      /usr/bin/env HOME=/var/lib/guanghu USER=guanghu LOGNAME=guanghu \
+      PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin "$@" \
+      >>"${LOG_ROOT}/${name}.log" 2>&1; then code=0; else code=$?; fi
+    now=$(date +%s)
+    if (( now - window_start > 60 )); then failures=0; window_start=$now; fi
+    ((failures+=1))
+    log "CHILD_EXIT ${name} code=${code} failures=${failures}"
+    (( failures <= 5 )) || return 1
+    sleep 1
+  done
+}
+
 start_root() {
  local name=$1; shift
-  log "START ${name}"
-  "$@" >>"${STATE_ROOT}/${name}.log" 2>&1 &
+  supervise_root "$name" "$@" &
  CHILDREN+=("$!:$name")
 }

 start_guanghu() {
  local name=$1; shift
-  log "START ${name} owner=guanghu"
-  /usr/bin/setpriv --reuid=998 --regid=998 --init-groups --inh-caps=-all --reset-env \
-    /usr/bin/env HOME=/var/lib/guanghu USER=guanghu LOGNAME=guanghu \
-    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin "$@" \
-    >>"${STATE_ROOT}/${name}.log" 2>&1 &
+  supervise_guanghu "$name" "$@" &
  CHILDREN+=("$!:$name")
 }

+require_listen() {
+  local name=$1 port=$2
+  for _ in 1 2 3 4 5; do
+    listen_ready "$port" && return 0
+    sleep 1
+  done
+  fatal "runtime_${name}_lost"
+}
+
 verify_identity() {
  local observed cmdline root_source
  observed=$(tr A-F a-f </sys/class/dmi/id/product_uuid | tr -d '\r\n')
@ -104,9 +144,9 @@ verify_identity() {

 prepare_runtime() {
  mount -o remount,rw /
-  mkdir -p "$STATE_ROOT" "$RECEIPT_ROOT" /run/sshd /run/systemd/resolve
+  mkdir -p "$STATE_ROOT" "$RECEIPT_ROOT" "$LOG_ROOT" /run/sshd /run/systemd/resolve
  install -d -o guanghu -g guanghu -m 0700 "$PNCC_STATE_ROOT"
-  chmod 0700 "$STATE_ROOT" "$RECEIPT_ROOT"
+  chmod 0700 "$STATE_ROOT" "$RECEIPT_ROOT" "$LOG_ROOT"
  chmod 1777 /tmp
  mountpoint -q /proc || mount -t proc proc /proc
  mountpoint -q /sys || mount -t sysfs sysfs /sys
@ -183,12 +223,12 @@ runtime_watch() {
      pid=${item%%:*}; name=${item#*:}
      kill -0 "$pid" 2>/dev/null || fatal "runtime_process_exited:${name}"
    done
-    listen_ready 22 || fatal "runtime_sshd_lost"
-    listen_ready 3340 || fatal "runtime_repository_lost"
-    listen_ready 8088 || fatal "runtime_projection_lost"
-    listen_ready 3922 || fatal "runtime_navigation_lost"
-    listen_ready 3923 || fatal "runtime_pncc_lost"
-    listen_ready 3940 || fatal "runtime_hololake_release_lost"
+    require_listen sshd 22
+    require_listen repository 3340
+    require_listen projection 8088
+    require_listen navigation 3922
+    require_listen pncc 3923
+    require_listen hololake_release 3940
  done
 }

--- a/product-source/hololake-platform/guanghu-os/scripts/test-guanghu-master-init.sh
+++ b/product-source/hololake-platform/guanghu-os/scripts/test-guanghu-master-init.sh
@ -26,10 +26,15 @@ grep -Fq 'wait_http code-projection http://127.0.0.1:8088/code/ 200,303' "$subje
 grep -Fq 'wait_http navigation-bridge http://127.0.0.1:3922/v1/anchor 200' "$subject"
 grep -Fq 'wait_http pncc-runtime http://127.0.0.1:3923/health 200' "$subject"
 grep -Fq '\"pncc\":\"RESIDENT_BOUND_CARRIER_SEPARATE\"' "$subject"
-grep -Fq 'listen_ready 3923 || fatal "runtime_pncc_lost"' "$subject"
+grep -Fq 'require_listen pncc 3923' "$subject"
 grep -Fq 'start_root hololake-release-bridge "$HOLOLAKE_RELEASE_BRIDGE"' "$subject"
 grep -Fq 'wait_http hololake-release-broadcast http://127.0.0.1:3940/health 200 30' "$subject"
-grep -Fq 'listen_ready 3940 || fatal "runtime_hololake_release_lost"' "$subject"
+grep -Fq 'require_listen hololake_release 3940' "$subject"
+grep -Fq 'readonly LOG_ROOT=${RECEIPT_ROOT}/runtime-logs' "$subject"
+grep -Fq 'supervise_root "$name" "$@" &' "$subject"
+grep -Fq 'supervise_guanghu "$name" "$@" &' "$subject"
+grep -Fq '(( failures <= 5 )) || return 1' "$subject"
+grep -Fq 'require_listen projection 8088' "$subject"
 if grep -Eq '(^|[[:space:]])(systemd|/sbin/init)([[:space:]]|$)' "$subject"; then
  echo 'full Linux init must remain dormant' >&2
  exit 1