feat: machine mode, PEM-only certs, ECH support, multi-instance config, batch ops, xbctl
This commit is contained in:
xboard
@@ -0,0 +1,398 @@
|
||||
// Package machine implements the machine-mode orchestrator that dynamically
|
||||
// discovers nodes from the panel's machine API and manages their lifecycles.
|
||||
package machine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cedar2025/xboard-node/internal/config"
|
||||
"github.com/cedar2025/xboard-node/internal/controlplane"
|
||||
"github.com/cedar2025/xboard-node/internal/monitor"
|
||||
"github.com/cedar2025/xboard-node/internal/nlog"
|
||||
"github.com/cedar2025/xboard-node/internal/panel"
|
||||
"github.com/cedar2025/xboard-node/internal/service"
|
||||
)
|
||||
|
||||
// nodeHandle tracks a running node service.
|
||||
type nodeHandle struct {
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
mailbox *controlplane.NodeMailbox
|
||||
}
|
||||
|
||||
// Orchestrator manages all nodes bound to a panel machine. It:
|
||||
// - discovers nodes via GET /machine/nodes
|
||||
// - starts / stops Service instances as nodes are added / removed
|
||||
// - maintains a shared WS connection that demuxes events by node_id
|
||||
// - reports machine-level load via POST /machine/status
|
||||
type Orchestrator struct {
|
||||
cfg *config.Config
|
||||
client *panel.Client // machine-level client (no node_id)
|
||||
|
||||
mu sync.Mutex
|
||||
nodes map[int]*nodeHandle // node_id → handle
|
||||
|
||||
// Per-node mailbox keyed by node_id. Shared WS events are aggregated here
|
||||
// and each node service drains the latest state when ready.
|
||||
eventsMu sync.RWMutex
|
||||
mailboxes map[int]*controlplane.NodeMailbox
|
||||
statuses map[int]chan<- controlplane.StatusChange
|
||||
|
||||
// Shared WS client (nil when WS is disabled).
|
||||
ws *panel.WSClient
|
||||
wsCancel context.CancelFunc
|
||||
|
||||
// runCtx is stored from Run() so that onWSEvent can trigger rediscover
|
||||
// for sync.nodes events without blocking the main loop.
|
||||
runCtx context.Context
|
||||
|
||||
pullInterval time.Duration
|
||||
pushInterval time.Duration
|
||||
}
|
||||
|
||||
// New creates a machine orchestrator from the given config.
|
||||
func New(cfg *config.Config) *Orchestrator {
|
||||
panelCfg := config.PanelConfig{
|
||||
URL: cfg.Panel.URL,
|
||||
Token: cfg.Machine.Token,
|
||||
MachineID: cfg.Machine.MachineID,
|
||||
}
|
||||
return &Orchestrator{
|
||||
cfg: cfg,
|
||||
client: panel.NewClient(panelCfg),
|
||||
nodes: make(map[int]*nodeHandle),
|
||||
mailboxes: make(map[int]*controlplane.NodeMailbox),
|
||||
statuses: make(map[int]chan<- controlplane.StatusChange),
|
||||
}
|
||||
}
|
||||
|
||||
// Run is the main loop. It blocks until ctx is cancelled.
|
||||
func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
o.runCtx = ctx
|
||||
nodesResp, err := o.client.GetMachineNodes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("initial node discovery: %w", err)
|
||||
}
|
||||
|
||||
o.applyIntervals(nodesResp.BaseConfig)
|
||||
nlog.Core().Info(fmt.Sprintf("machine %d: discovered %d nodes",
|
||||
o.cfg.Machine.MachineID, len(nodesResp.Nodes)))
|
||||
|
||||
// Start machine-level WS as early as possible so sync.nodes can reach an
|
||||
// empty machine before the first node is attached.
|
||||
o.tryStartWS(ctx)
|
||||
|
||||
// Start initial nodes.
|
||||
for _, n := range nodesResp.Nodes {
|
||||
o.startNode(ctx, n)
|
||||
}
|
||||
|
||||
discoveryTicker := time.NewTicker(o.pullInterval)
|
||||
statusTicker := time.NewTicker(o.pushInterval)
|
||||
defer discoveryTicker.Stop()
|
||||
defer statusTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
o.stopAll()
|
||||
return nil
|
||||
|
||||
case <-discoveryTicker.C:
|
||||
o.rediscover(ctx)
|
||||
|
||||
case <-statusTicker.C:
|
||||
o.reportMachineStatus()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Node lifecycle ──────────────────────────────────────────────────────
|
||||
|
||||
func (o *Orchestrator) startNode(ctx context.Context, mn panel.MachineNode) {
|
||||
o.mu.Lock()
|
||||
if _, exists := o.nodes[mn.ID]; exists {
|
||||
o.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
nodeCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
mb := controlplane.NewNodeMailbox()
|
||||
o.nodes[mn.ID] = &nodeHandle{cancel: cancel, done: done, mailbox: mb}
|
||||
o.mu.Unlock()
|
||||
|
||||
o.eventsMu.Lock()
|
||||
o.mailboxes[mn.ID] = mb
|
||||
o.eventsMu.Unlock()
|
||||
|
||||
nodeCfg := o.cfg.ExpandMachineNode(mn.ID, mn.Type)
|
||||
|
||||
perNodeClient := o.client.ForNode(mn.ID)
|
||||
|
||||
var push controlplane.PushClient
|
||||
if o.ws != nil {
|
||||
push = &machineNodePush{
|
||||
nodeID: mn.ID,
|
||||
ws: o.ws,
|
||||
}
|
||||
}
|
||||
|
||||
// The registerFn is called by MachinePanelControlPlane.Initial() to expose
|
||||
// the node mailbox + status channel to the Service.
|
||||
nodeID := mn.ID
|
||||
registerFn := func(st chan<- controlplane.StatusChange) *controlplane.NodeMailbox {
|
||||
o.registerNode(nodeID, st)
|
||||
return mb
|
||||
}
|
||||
|
||||
cp := controlplane.NewMachinePanelControlPlane(perNodeClient, nodeCfg.Kernel, push, registerFn)
|
||||
svc := service.NewWithControlPlane(nodeCfg, cp)
|
||||
|
||||
nlog.Core().Info(fmt.Sprintf("machine: starting node %d (%s/%s)",
|
||||
mn.ID, mn.Type, mn.Name))
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer o.unregisterNode(mn.ID)
|
||||
if err := svc.Run(nodeCtx); err != nil {
|
||||
nlog.Core().Error("machine node exited with error",
|
||||
"node_id", mn.ID, "error", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (o *Orchestrator) stopNode(nodeID int) {
|
||||
o.mu.Lock()
|
||||
h, ok := o.nodes[nodeID]
|
||||
if !ok {
|
||||
o.mu.Unlock()
|
||||
return
|
||||
}
|
||||
delete(o.nodes, nodeID)
|
||||
o.mu.Unlock()
|
||||
|
||||
o.eventsMu.Lock()
|
||||
delete(o.mailboxes, nodeID)
|
||||
o.eventsMu.Unlock()
|
||||
|
||||
nlog.Core().Info(fmt.Sprintf("machine: stopping node %d", nodeID))
|
||||
h.cancel()
|
||||
<-h.done
|
||||
}
|
||||
|
||||
func (o *Orchestrator) stopAll() {
|
||||
o.mu.Lock()
|
||||
handles := make(map[int]*nodeHandle, len(o.nodes))
|
||||
for id, h := range o.nodes {
|
||||
handles[id] = h
|
||||
}
|
||||
o.mu.Unlock()
|
||||
|
||||
for id, h := range handles {
|
||||
nlog.Core().Info(fmt.Sprintf("machine: stopping node %d", id))
|
||||
h.cancel()
|
||||
}
|
||||
for _, h := range handles {
|
||||
<-h.done
|
||||
}
|
||||
|
||||
if o.wsCancel != nil {
|
||||
o.wsCancel()
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Node discovery ──────────────────────────────────────────────────────
|
||||
|
||||
func (o *Orchestrator) rediscover(ctx context.Context) {
|
||||
nodesResp, err := o.client.GetMachineNodes()
|
||||
if err != nil {
|
||||
nlog.Core().Warn("machine node discovery failed", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
wanted := make(map[int]panel.MachineNode, len(nodesResp.Nodes))
|
||||
for _, n := range nodesResp.Nodes {
|
||||
wanted[n.ID] = n
|
||||
}
|
||||
|
||||
o.mu.Lock()
|
||||
var toRemove []int
|
||||
for id := range o.nodes {
|
||||
if _, ok := wanted[id]; !ok {
|
||||
toRemove = append(toRemove, id)
|
||||
}
|
||||
}
|
||||
o.mu.Unlock()
|
||||
|
||||
for _, id := range toRemove {
|
||||
o.stopNode(id)
|
||||
}
|
||||
|
||||
for _, n := range nodesResp.Nodes {
|
||||
o.startNode(ctx, n) // no-op if already running
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Machine status reporting ────────────────────────────────────────────
|
||||
|
||||
func (o *Orchestrator) reportMachineStatus() {
|
||||
s := monitor.Collect()
|
||||
if err := o.client.ReportMachineStatus(
|
||||
s.CPU,
|
||||
[2]uint64{s.MemTotal, s.MemUsed},
|
||||
[2]uint64{s.SwapTotal, s.SwapUsed},
|
||||
[2]uint64{s.DiskTotal, s.DiskUsed},
|
||||
); err != nil {
|
||||
nlog.Core().Warn("machine status report failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── WS mux ─────────────────────────────────────────────────────────────
|
||||
|
||||
func (o *Orchestrator) tryStartWS(ctx context.Context) {
|
||||
hs, err := o.client.Handshake()
|
||||
if err != nil {
|
||||
nlog.Core().Warn("machine ws handshake failed, REST only", "error", err)
|
||||
return
|
||||
}
|
||||
if !hs.WebSocket.Enabled || hs.WebSocket.WSURL == "" {
|
||||
nlog.Core().Info("machine: ws disabled by panel, REST only")
|
||||
return
|
||||
}
|
||||
|
||||
wsCfg := panel.WSClientConfig{
|
||||
StatusInterval: time.Duration(o.cfg.WS.StatusInterval) * time.Second,
|
||||
HandshakeTimeout: time.Duration(o.cfg.WS.HandshakeTimeout) * time.Second,
|
||||
BackoffInitial: time.Duration(o.cfg.WS.BackoffInitial) * time.Second,
|
||||
BackoffMax: time.Duration(o.cfg.WS.BackoffMax) * time.Second,
|
||||
MachineID: o.cfg.Machine.MachineID,
|
||||
}
|
||||
|
||||
o.ws = panel.NewWSClient(
|
||||
hs.WebSocket.WSURL,
|
||||
o.cfg.Machine.Token,
|
||||
0, // no single node_id
|
||||
wsCfg,
|
||||
o.onWSEvent,
|
||||
o.onWSStatus,
|
||||
nil, // per-node status is sent via machineNodePush
|
||||
)
|
||||
|
||||
wsCtx, wsCancel := context.WithCancel(ctx)
|
||||
o.wsCancel = wsCancel
|
||||
go o.ws.Run(wsCtx)
|
||||
|
||||
nlog.Core().Info("machine: ws mux started")
|
||||
}
|
||||
|
||||
// onWSEvent routes a WS event to the correct node's channel.
|
||||
// sync.nodes is a machine-level event that triggers immediate rediscovery.
|
||||
func (o *Orchestrator) onWSEvent(event panel.WSEvent) {
|
||||
// sync.nodes is a machine-level event, not per-node
|
||||
if event.Type == panel.WSEventSyncNodes {
|
||||
nlog.Core().Info("machine received sync.nodes, triggering immediate rediscovery")
|
||||
go o.rediscover(o.runCtx)
|
||||
return
|
||||
}
|
||||
|
||||
nodeID := event.NodeID
|
||||
if nodeID == 0 {
|
||||
nlog.Core().Debug("machine ws event missing node_id, dropping", "type", event.Type)
|
||||
return
|
||||
}
|
||||
|
||||
translated, err := controlplane.TranslateWSEvent(event, o.cfg.Kernel)
|
||||
if err != nil {
|
||||
nlog.Core().Warn("machine ws event translation failed",
|
||||
"type", event.Type, "node_id", nodeID, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
o.eventsMu.RLock()
|
||||
mailbox, ok := o.mailboxes[nodeID]
|
||||
o.eventsMu.RUnlock()
|
||||
if !ok {
|
||||
nlog.Core().Debug("machine ws event for unknown node", "node_id", nodeID, "type", event.Type)
|
||||
return
|
||||
}
|
||||
mailbox.Apply(translated)
|
||||
}
|
||||
|
||||
// onWSStatus broadcasts WS connectivity changes to all registered nodes.
|
||||
func (o *Orchestrator) onWSStatus(status panel.WSStatusChange) {
|
||||
change := controlplane.StatusChange{Connected: status.Connected}
|
||||
o.eventsMu.RLock()
|
||||
defer o.eventsMu.RUnlock()
|
||||
for _, ch := range o.statuses {
|
||||
select {
|
||||
case ch <- change:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Orchestrator) registerNode(nodeID int, st chan<- controlplane.StatusChange) {
|
||||
o.eventsMu.Lock()
|
||||
o.statuses[nodeID] = st
|
||||
o.eventsMu.Unlock()
|
||||
}
|
||||
|
||||
func (o *Orchestrator) unregisterNode(nodeID int) {
|
||||
o.eventsMu.Lock()
|
||||
delete(o.mailboxes, nodeID)
|
||||
delete(o.statuses, nodeID)
|
||||
o.eventsMu.Unlock()
|
||||
}
|
||||
|
||||
func (o *Orchestrator) applyIntervals(bc panel.MachineBaseConfig) {
|
||||
o.pullInterval = time.Duration(bc.PullInterval) * time.Second
|
||||
if o.pullInterval < 30*time.Second {
|
||||
o.pullInterval = 60 * time.Second
|
||||
}
|
||||
o.pushInterval = time.Duration(bc.PushInterval) * time.Second
|
||||
if o.pushInterval < 10*time.Second {
|
||||
o.pushInterval = 60 * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Virtual PushClient ─────────────────────────────────────────────────
|
||||
|
||||
// machineNodePush implements controlplane.PushClient for a single node
|
||||
// backed by the shared machine WS connection. Events are routed by the
|
||||
// WS mux directly to the Service's channels; this adapter only provides
|
||||
// connectivity status and send capabilities.
|
||||
type machineNodePush struct {
|
||||
nodeID int
|
||||
ws *panel.WSClient
|
||||
}
|
||||
|
||||
func (p *machineNodePush) Run(ctx context.Context) {
|
||||
// The shared WS mux pushes events into our channels; we just wait.
|
||||
<-ctx.Done()
|
||||
}
|
||||
|
||||
func (p *machineNodePush) IsConnected() bool {
|
||||
return p.ws != nil && p.ws.IsConnected()
|
||||
}
|
||||
|
||||
func (p *machineNodePush) SendDeviceReport(devices map[int][]string) {
|
||||
if p.ws == nil {
|
||||
return
|
||||
}
|
||||
payload := map[string]interface{}{
|
||||
"node_id": p.nodeID,
|
||||
}
|
||||
// Flatten into the standard format with node_id wrapper.
|
||||
strDevices := make(map[string][]string, len(devices))
|
||||
for uid, ips := range devices {
|
||||
strDevices[fmt.Sprintf("%d", uid)] = ips
|
||||
}
|
||||
payload["devices"] = strDevices
|
||||
data, _ := json.Marshal(payload)
|
||||
p.ws.SendRaw(panel.WSEventReportDevices, data)
|
||||
}
|
||||
Reference in New Issue
Block a user