diff --git a/Makefile b/Makefile index 13ac948..629119a 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ optimize-kernel: ## Configure kernel boot parameters optimize-tuned: ## Switch to accelerator-performance profile @bash scripts/optimize/tuned-profile.sh +optimize-power: ## Apply Phase 2 tuning (ryzenadj, sysctl, THP, RADV) + @bash scripts/optimize/power-profile.sh + optimize-vram: ## BIOS VRAM guidance + GTT verification @bash scripts/optimize/vram-gtt.sh diff --git a/bin/optimize b/bin/optimize index 406400b..3443e36 100755 --- a/bin/optimize +++ b/bin/optimize @@ -6,24 +6,31 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" case "${1:---all}" in --all|-a) - echo "Running optimization walkthrough..." + echo "Running full optimization walkthrough..." + echo "" + echo "=== Phase 1: Core System ===" bash "$SCRIPT_DIR/scripts/optimize/tuned-profile.sh" bash "$SCRIPT_DIR/scripts/optimize/kernel-params.sh" bash "$SCRIPT_DIR/scripts/optimize/vram-gtt.sh" echo "" + echo "=== Phase 2: System Tuning ===" + bash "$SCRIPT_DIR/scripts/optimize/power-profile.sh" + echo "" bash "$SCRIPT_DIR/scripts/optimize/verify.sh" ;; --kernel|-k) exec bash "$SCRIPT_DIR/scripts/optimize/kernel-params.sh" ;; --tuned|-t) exec bash "$SCRIPT_DIR/scripts/optimize/tuned-profile.sh" ;; --vram|-v) exec bash "$SCRIPT_DIR/scripts/optimize/vram-gtt.sh" ;; + --power|-p) exec bash "$SCRIPT_DIR/scripts/optimize/power-profile.sh" ;; --verify) exec bash "$SCRIPT_DIR/scripts/optimize/verify.sh" ;; --rollback) exec bash "$SCRIPT_DIR/scripts/optimize/rollback.sh" ;; *) - echo "Usage: optimize [--all|--kernel|--tuned|--vram|--verify|--rollback]" - echo " --all Full optimization walkthrough (default)" + echo "Usage: optimize [--all|--kernel|--tuned|--vram|--power|--verify|--rollback]" + echo " --all Full optimization walkthrough (Phase 1 + 2)" echo " --kernel Configure kernel boot parameters" echo " --tuned Switch tuned profile" echo " --vram BIOS VRAM + GTT guidance" + echo " --power Phase 2: ryzenadj, sysctl, THP, RADV" echo " --verify Post-optimization checklist" echo " --rollback Revert changes" exit 1 diff --git a/configs/models.conf b/configs/models.conf index 1edfba4..ea0b05e 100644 --- a/configs/models.conf +++ b/configs/models.conf @@ -21,3 +21,10 @@ qwen3.5-27b-q4|unsloth/Qwen3.5-27B-GGUF|Qwen3.5-27B-Q4_K_M.gguf|17|dense|Dense 2 qwen3.5-35b-a3b-q4|unsloth/Qwen3.5-35B-A3B-GGUF|Qwen3.5-35B-A3B-UD-Q4_K_L.gguf|19|moe|MoE 35B, 3B active, Unsloth dynamic qwen3.5-35b-a3b-q8|unsloth/Qwen3.5-35B-A3B-GGUF|Qwen3.5-35B-A3B-Q8_0.gguf|37|moe|MoE 35B Q8, near-full precision nemotron-30b-a3b-q4|lmstudio-community/NVIDIA-Nemotron-3-Nano-30B-A3B-GGUF|NVIDIA-Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf|23|moe|Nemotron MoE 30B, 3B active +nemotron-cascade2-q8|bartowski/nvidia_Nemotron-Cascade-2-30B-A3B-GGUF|nvidia_Nemotron-Cascade-2-30B-A3B-Q8_0.gguf|31|moe|Nemotron Cascade 2, Mamba-2 hybrid + +# ── Coding models ───────────────────────────────────────── +qwen3-coder-30b-a3b-q6|unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF|Qwen3-Coder-30B-A3B-Instruct-UD-Q6_K_XL.gguf|26|moe|Agentic coding MoE, pure Transformer + +# ── Draft models (speculative decoding) ─────────────────── +qwen3.5-0.8b-q8-draft|unsloth/Qwen3.5-0.8B-GGUF|Qwen3.5-0.8B-Q8_0.gguf|0.8|draft|Draft for Qwen3.5 speculative decoding diff --git a/configs/ryzenadj-llm.service b/configs/ryzenadj-llm.service new file mode 100644 index 0000000..1112ab8 --- /dev/null +++ b/configs/ryzenadj-llm.service @@ -0,0 +1,14 @@ +[Unit] +Description=Apply RyzenAdj power limits for LLM inference +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ryzenadj --stapm-limit=85000 --fast-limit=85000 --slow-limit=85000 --apu-slow-limit=85000 +RemainAfterExit=yes + +# Re-apply after resume from sleep/hibernate (HP firmware resets limits) +ExecStartPost=/bin/sleep 2 + +[Install] +WantedBy=multi-user.target diff --git a/configs/ryzenadj-resume.service b/configs/ryzenadj-resume.service new file mode 100644 index 0000000..37d8ad7 --- /dev/null +++ b/configs/ryzenadj-resume.service @@ -0,0 +1,10 @@ +[Unit] +Description=Re-apply RyzenAdj power limits after resume +After=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ryzenadj --stapm-limit=85000 --fast-limit=85000 --slow-limit=85000 --apu-slow-limit=85000 + +[Install] +WantedBy=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target diff --git a/docs/optimization-log.md b/docs/optimization-log.md index 1da4d57..7a33e73 100644 --- a/docs/optimization-log.md +++ b/docs/optimization-log.md @@ -37,38 +37,42 @@ Living document tracking what was applied, tested, and the actual results. Each ## Phase 2: System Tuning -### 2.1 RyzenAdj 85W PPT +### 2.1 RyzenAdj PPT Increase -- **Date**: PENDING -- **Change**: `sudo ryzenadj --stapm-limit=85000 --fast-limit=85000 --slow-limit=85000` -- **Expected**: +12-19% CPU/GPU throughput (community data from Strix Halo Wiki) -- **Benchmark**: Not yet run -- **Notes**: HP ZBook ships at 60W. 85W is the community-recommended sweet spot. -- **Verdict**: PENDING +- **Date**: 2026-03-30 +- **Change**: `sudo ryzenadj --stapm-limit=85000 --fast-limit=85000 --slow-limit=85000 --apu-slow-limit=85000` +- **Result**: STAPM raised from 59W→81W. PPT Fast raised to 81W. **However, PPT SLOW and APU SLOW stuck at 70W** — HP ZBook BIOS EC overrides these limits. Effective sustained power: ~70W (was ~59W). +- **Benchmark**: `data/benchmarks/qwen35-shootout-v2-*` (Vulkan, q4_0 KV, pp2048/tg1024) + - UD-Q4_K_L: **57.0 t/s** (was ~39 t/s before RyzenAdj = **+46%**) + - UD-Q4_K_XL: **56.4 t/s** + - Q8_0: **51.4 t/s** (was ~39-41 t/s before = **+25%**) +- **Thermals**: 70-73C under load, 30C headroom. Cooling handles it easily. +- **Notes**: Settings are volatile (reset on reboot/sleep). Use `sudo make optimize-power` or install systemd service for persistence. HP firmware hard-caps slow PPT at 70W regardless. +- **Verdict**: KEEP — significant real-world improvement despite HP firmware limit ### 2.2 VM Sysctl Tuning -- **Date**: PENDING -- **Change**: `vm.swappiness=1, vm.dirty_ratio=40, vm.max_map_count=500000` -- **Expected**: Prevent model weight eviction, reduce I/O disruption -- **Benchmark**: Not yet run -- **Verdict**: PENDING +- **Date**: 2026-03-30 +- **Change**: `vm.swappiness=1, vm.dirty_ratio=40, vm.dirty_background_ratio=10, vm.max_map_count=500000, vm.zone_reclaim_mode=0` +- **Applied via**: `sudo make optimize-power` (persists to `/etc/sysctl.d/99-llm-inference.conf`) +- **Notes**: Hard to isolate impact — applied together with other Phase 2 changes. Prevents model weight eviction and I/O disruption. +- **Verdict**: KEEP — low risk, persists across reboots ### 2.3 Transparent Huge Pages -- **Date**: PENDING -- **Change**: `transparent_hugepage=always` -- **Expected**: Faster model load time, possible 1-5% tg improvement from reduced TLB misses -- **Benchmark**: Not yet run -- **Verdict**: PENDING +- **Date**: 2026-03-30 +- **Change**: `echo always > /sys/kernel/mm/transparent_hugepage/enabled` +- **Applied via**: `sudo make optimize-power` (volatile — add `transparent_hugepage=always` to kernel cmdline for persistence) +- **Notes**: Reduces TLB misses for mmap'd model files. Hard to isolate impact. +- **Verdict**: KEEP — low risk ### 2.4 RADV_PERFTEST=nogttspill -- **Date**: PENDING -- **Change**: `export RADV_PERFTEST=nogttspill` -- **Expected**: Fix pp degradation on Vulkan RADV (community-reported fix for Strix Halo) -- **Benchmark**: Not yet run — needs Vulkan-specific benchmark comparison -- **Verdict**: PENDING +- **Date**: 2026-03-30 +- **Change**: `RADV_PERFTEST=nogttspill` persisted to `/etc/environment.d/radv-llm.conf` +- **Applied via**: `sudo make optimize-power` +- **Notes**: Prevents GTT spill management overhead on unified memory Vulkan. Takes effect on next login. For current session: `export RADV_PERFTEST=nogttspill` +- **Verdict**: KEEP — persists across reboots ### 2.5 amdgpu.noretry=0 @@ -84,11 +88,19 @@ Living document tracking what was applied, tested, and the actual results. Each ### 3.1 KV Cache Quantization -- **Date**: PENDING (sweep running) -- **Change**: `-ctk q8_0 -ctv q8_0` / `-ctk q4_0 -ctv q4_0` -- **Benchmark**: `data/benchmarks/kv-sweep-128k-*` (in progress) -- **Expected**: Q8_0: ~50% less KV memory, negligible quality loss. Q4_0: ~75% less, noticeable quality impact. -- **Verdict**: PENDING +- **Date**: 2026-03-27 +- **Change**: `--kv-types f16,q8_0,q4_0` sweep +- **Benchmark**: `data/benchmarks/kv-sweep-256k-*` +- **Result** (Vulkan RADV, Qwen3.5-35B-A3B Q8, pp2048/tg1024): + - f16: 456 pp, 39.8 tg + - q8_0: 418 pp, 38.5 tg (slight Vulkan regression — unexpected) + - **q4_0: 460 pp, 41.1 tg** (fastest overall, +3% tg over f16) +- **Result** (ROCm, same model): + - f16: 445 pp, 21.5 tg + - q8_0: 495 pp, 21.7 tg (+11% pp, same tg) + - q4_0: 494 pp, 21.8 tg (+11% pp, same tg) +- **Conclusion**: q4_0 is the sweet spot on Vulkan (fastest tg + 75% less KV memory). On ROCm, KV quant helps pp but not tg. +- **Verdict**: KEEP — use q4_0 KV as default for serving ### 3.2 MoE Batch Size `-b 256` @@ -161,6 +173,24 @@ Living document tracking what was applied, tested, and the actual results. Each --- +## Model Quant Shootout + +### Qwen3.5-35B-A3B — Q4_K_L vs Q4_K_XL vs Q8 (2026-03-30) + +- **Benchmark**: `data/benchmarks/qwen35-shootout-v2-*` +- **Config**: Vulkan RADV, q4_0 KV cache, pp2048/tg1024, 2 reps +- **RyzenAdj**: STAPM=81W (sustained ~70W due to HP firmware cap) + +| Quant | File Size | pp2048 (t/s) | tg1024 (t/s) | Recommendation | +|-------|-----------|-------------|-------------|----------------| +| UD-Q4_K_L | 18.8 GB | 825 | **57.0** | Fastest. Good quality. | +| **UD-Q4_K_XL** | 20.7 GB | 835 | **56.4** | **Daily driver** — best quality/speed. | +| Q8_0 | 34.4 GB | 850 | 51.4 | Best quality, 10% slower tg. | + +**Decision**: Keep UD-Q4_K_XL (daily driver) and Q8_0 (quality fallback). Q4_K_L can be deleted — Q4_K_XL is strictly better at only +2 GB. + +--- + ## How to Add Entries When testing a new optimization: diff --git a/scripts/optimize/power-profile.sh b/scripts/optimize/power-profile.sh new file mode 100644 index 0000000..2f5af4a --- /dev/null +++ b/scripts/optimize/power-profile.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Apply power profile and system tuning for LLM inference workloads +# Requires root. Settings are volatile — use the systemd service for persistence. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../lib/common.sh" +source "$SCRIPT_DIR/../../lib/format.sh" + +require_root + +# ── Power limits via ryzenadj ───────────────────────────── +STAPM=85000 +FAST=85000 +SLOW=85000 +APU_SLOW=85000 + +if is_cmd ryzenadj; then + log_header "Power Profile (ryzenadj)" + log_info "Setting STAPM=${STAPM}mW, Fast=${FAST}mW, Slow=${SLOW}mW, APU=${APU_SLOW}mW" + ryzenadj \ + --stapm-limit=$STAPM \ + --fast-limit=$FAST \ + --slow-limit=$SLOW \ + --apu-slow-limit=$APU_SLOW 2>&1 | grep -E 'Successfully|Error|not supported' || true + + # Verify what actually took effect + log_info "Verifying limits..." + ryzenadj -i 2>&1 | grep -E 'LIMIT|VALUE' | head -8 + echo "" + log_warn "Note: HP firmware may cap PPT SLOW/APU at 70W regardless of setting" +else + log_error "ryzenadj not found. Install: cd /tmp && git clone https://github.com/FlyGoat/RyzenAdj.git && cd RyzenAdj && mkdir build && cd build && cmake .. && make && sudo cp ryzenadj /usr/local/bin/" + exit 1 +fi + +# ── VM sysctl tuning ────────────────────────────────────── +log_header "VM Sysctl Tuning" + +declare -A SYSCTLS=( + [vm.swappiness]=1 + [vm.dirty_ratio]=40 + [vm.dirty_background_ratio]=10 + [vm.max_map_count]=500000 + [vm.zone_reclaim_mode]=0 +) + +for KEY in "${!SYSCTLS[@]}"; do + VAL="${SYSCTLS[$KEY]}" + CURRENT=$(sysctl -n "$KEY" 2>/dev/null || echo "?") + if [[ "$CURRENT" == "$VAL" ]]; then + log_success "$KEY = $VAL (already set)" + else + sysctl -w "$KEY=$VAL" > /dev/null 2>&1 + log_success "$KEY = $VAL (was $CURRENT)" + fi +done + +# Persist sysctl settings +SYSCTL_CONF="/etc/sysctl.d/99-llm-inference.conf" +if [[ ! -f "$SYSCTL_CONF" ]]; then + log_info "Persisting to $SYSCTL_CONF" + cat > "$SYSCTL_CONF" << 'EOF' +# LLM inference optimizations +vm.swappiness = 1 +vm.dirty_ratio = 40 +vm.dirty_background_ratio = 10 +vm.max_map_count = 500000 +vm.zone_reclaim_mode = 0 +EOF + log_success "Sysctl config saved (persists across reboots)" +else + log_info "Sysctl config already exists at $SYSCTL_CONF" +fi + +# ── Transparent Huge Pages ──────────────────────────────── +log_header "Transparent Huge Pages" + +THP_ENABLED=$(cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || echo "unknown") +if [[ "$THP_ENABLED" == *"[always]"* ]]; then + log_success "THP = always (already set)" +else + echo always > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true + echo defer+madvise > /sys/kernel/mm/transparent_hugepage/defrag 2>/dev/null || true + log_success "THP = always, defrag = defer+madvise" +fi +log_info "For persistence, add to kernel cmdline: transparent_hugepage=always" + +# ── RADV nogttspill ─────────────────────────────────────── +log_header "Vulkan RADV Environment" + +RADV_CONF="/etc/environment.d/radv-llm.conf" +if [[ ! -f "$RADV_CONF" ]]; then + mkdir -p /etc/environment.d + echo 'RADV_PERFTEST=nogttspill' > "$RADV_CONF" + log_success "RADV_PERFTEST=nogttspill persisted to $RADV_CONF" + log_info "Takes effect on next login. For this session: export RADV_PERFTEST=nogttspill" +else + log_success "RADV config already exists at $RADV_CONF" +fi + +# ── Summary ─────────────────────────────────────────────── +log_header "Phase 2 Optimization Summary" +log_success "Power profile: ryzenadj limits applied (volatile — resets on reboot)" +log_success "VM tuning: sysctl applied and persisted" +log_success "THP: enabled (volatile — add to kernel cmdline for persistence)" +log_success "RADV: nogttspill persisted" +echo "" +log_info "To persist ryzenadj across reboots:" +log_info " sudo cp $SCRIPT_DIR/../../configs/ryzenadj-llm.service /etc/systemd/system/" +log_info " sudo systemctl enable --now ryzenadj-llm.service"