91 lines
2.4 KiB
Bash
91 lines
2.4 KiB
Bash
#!/usr/bin/env bash
|
|
# Tmux-based monitoring dashboard
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../../lib/common.sh"
|
|
|
|
SESSION="strix-monitor"
|
|
SIMPLE=false
|
|
WITH_LOG=false
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--simple|-s) SIMPLE=true; shift ;;
|
|
--with-logging|-l) WITH_LOG=true; shift ;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
|
|
# Simple mode: just launch amdgpu_top
|
|
if $SIMPLE; then
|
|
if is_cmd amdgpu_top; then
|
|
exec amdgpu_top
|
|
elif is_cmd nvtop; then
|
|
log_warn "amdgpu_top not found, falling back to nvtop"
|
|
exec nvtop
|
|
else
|
|
log_error "No GPU monitor installed. Run: make monitor-install"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Full dashboard requires tmux
|
|
if ! is_cmd tmux; then
|
|
log_error "tmux is required for dashboard mode. Run: make monitor-install"
|
|
exit 1
|
|
fi
|
|
|
|
# Pick GPU monitor
|
|
GPU_MON="nvtop"
|
|
if is_cmd amdgpu_top; then
|
|
GPU_MON="amdgpu_top"
|
|
fi
|
|
|
|
# Pick system monitor
|
|
SYS_MON="htop"
|
|
if is_cmd btop; then
|
|
SYS_MON="btop"
|
|
elif ! is_cmd htop; then
|
|
SYS_MON="top"
|
|
fi
|
|
|
|
# Kill existing session if running
|
|
tmux kill-session -t "$SESSION" 2>/dev/null || true
|
|
|
|
# Start background logging if requested
|
|
LOG_CMD="echo Metric logging not active. Use --with-logging to enable.; read -r"
|
|
LOG_PID=""
|
|
if $WITH_LOG; then
|
|
LOG_FILE="$(data_dir logs)/metrics-$(timestamp).csv"
|
|
bash "$SCRIPT_DIR/log-metrics.sh" --output "$LOG_FILE" &
|
|
LOG_PID=$!
|
|
LOG_CMD="tail -f \"$LOG_FILE\""
|
|
log_info "Metric logger started (PID: $LOG_PID) → $LOG_FILE"
|
|
fi
|
|
|
|
# Cleanup logger on exit
|
|
cleanup() {
|
|
if [[ -n "$LOG_PID" ]]; then
|
|
kill "$LOG_PID" 2>/dev/null || true
|
|
wait "$LOG_PID" 2>/dev/null || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# Create tmux layout
|
|
# +--------------------+--------------------+
|
|
# | GPU monitor | System monitor |
|
|
# | | |
|
|
# +--------------------------------------------+
|
|
# | Metrics log tail / status |
|
|
# +--------------------------------------------+
|
|
tmux new-session -d -s "$SESSION" -x "$(tput cols 2>/dev/null || echo 120)" -y "$(tput lines 2>/dev/null || echo 40)" "$GPU_MON"
|
|
tmux split-window -t "$SESSION" -h "$SYS_MON"
|
|
tmux split-window -t "$SESSION" -v -p 20 "$LOG_CMD"
|
|
tmux select-pane -t "$SESSION:0.0"
|
|
|
|
log_info "Dashboard started. Attach with: tmux attach -t $SESSION"
|
|
log_info "Detach with Ctrl+B then D. Kill with: tmux kill-session -t $SESSION"
|
|
tmux attach -t "$SESSION"
|