test: add BATS test suite (79 tests)

- tests/common.bats: PROJECT_ROOT detection, is_cmd, timestamp, data_dir,
  logging functions, color handling, require_root
- tests/detect.bats: GPU sysfs reads with mock sysfs tree, kernel param
  parsing (word boundary, dot escaping, edge positions), recommended
  GTT/pages computation (64GB, 128GB, tiny, zero), firmware bad detection,
  stack detection
- tests/format.bats: human_bytes (0, KiB, MiB, GiB boundaries, 64GiB),
  human_mib (sub-GiB, exact-GiB, recommended values, empty input)
- tests/benchmark_compare.bats: improvement/regression display, empty
  results, missing files, usage output, config change detection
- tests/log_metrics.bats: CSV header, data format, field count, input
  validation, unknown argument handling
- tests/test_helper.sh: mock sysfs tree builder, bats-assert/support setup

Makefile: add 'make test' target

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Felipe Cardoso
2026-03-25 22:15:34 +01:00
parent da2c4c6b8a
commit a403dd9ce0
7 changed files with 679 additions and 1 deletions

View File

@@ -0,0 +1,84 @@
#!/usr/bin/env bats
# Tests for scripts/benchmark/compare.sh — result comparison logic
load test_helper.sh
setup() {
source_lib common.sh
source_lib format.sh
BEFORE_DIR="$(mktemp -d)"
AFTER_DIR="$(mktemp -d)"
}
teardown() {
rm -rf "$BEFORE_DIR" "$AFTER_DIR"
}
write_summary() {
local dir="$1" json="$2"
echo "$json" > "$dir/summary.json"
}
write_system_state() {
local dir="$1" json="$2"
echo "$json" > "$dir/system-state.json"
}
# ── Basic comparison ─────────────────────────────────────
@test "compare: shows improvement when after > before" {
write_summary "$BEFORE_DIR" '{"results":[{"model":"qwen3","backend":"Vulkan","test":"pp512","tokens_per_sec":500.0,"file":"test.log","size":"4GB","raw":"500.0"}]}'
write_summary "$AFTER_DIR" '{"results":[{"model":"qwen3","backend":"Vulkan","test":"pp512","tokens_per_sec":600.0,"file":"test.log","size":"4GB","raw":"600.0"}]}'
write_system_state "$BEFORE_DIR" '{"memory":{"vram_total_bytes":0},"kernel":{},"tuned_profile":"throughput-performance"}'
write_system_state "$AFTER_DIR" '{"memory":{"vram_total_bytes":0},"kernel":{},"tuned_profile":"throughput-performance"}'
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh" "$BEFORE_DIR" "$AFTER_DIR"
assert_success
assert_output --partial "500.0"
assert_output --partial "600.0"
}
@test "compare: shows regression when after < before" {
write_summary "$BEFORE_DIR" '{"results":[{"model":"qwen3","backend":"Vulkan","test":"tg128","tokens_per_sec":15.0,"file":"test.log","size":"4GB","raw":"15.0"}]}'
write_summary "$AFTER_DIR" '{"results":[{"model":"qwen3","backend":"Vulkan","test":"tg128","tokens_per_sec":12.0,"file":"test.log","size":"4GB","raw":"12.0"}]}'
write_system_state "$BEFORE_DIR" '{"memory":{},"kernel":{},"tuned_profile":""}'
write_system_state "$AFTER_DIR" '{"memory":{},"kernel":{},"tuned_profile":""}'
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh" "$BEFORE_DIR" "$AFTER_DIR"
assert_success
assert_output --partial "12.0"
}
@test "compare: handles empty results gracefully" {
write_summary "$BEFORE_DIR" '{"results":[]}'
write_summary "$AFTER_DIR" '{"results":[]}'
write_system_state "$BEFORE_DIR" '{"memory":{},"kernel":{},"tuned_profile":""}'
write_system_state "$AFTER_DIR" '{"memory":{},"kernel":{},"tuned_profile":""}'
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh" "$BEFORE_DIR" "$AFTER_DIR"
assert_success
assert_output --partial "No comparable results"
}
@test "compare: fails without summary.json" {
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh" "$BEFORE_DIR" "$AFTER_DIR"
assert_failure
assert_output --partial "No summary.json"
}
@test "compare: shows usage when called without args" {
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh"
assert_failure
assert_output --partial "Usage"
}
@test "compare: detects config changes between runs" {
write_summary "$BEFORE_DIR" '{"results":[{"model":"m","backend":"b","test":"t","tokens_per_sec":1.0,"file":"f","size":"s","raw":"1.0"}]}'
write_summary "$AFTER_DIR" '{"results":[{"model":"m","backend":"b","test":"t","tokens_per_sec":2.0,"file":"f","size":"s","raw":"2.0"}]}'
write_system_state "$BEFORE_DIR" '{"memory":{"vram_total_bytes":34359738368},"kernel":{"param_iommu":"","param_gttsize":"","param_pages_limit":""},"tuned_profile":"throughput-performance"}'
write_system_state "$AFTER_DIR" '{"memory":{"vram_total_bytes":536870912},"kernel":{"param_iommu":"pt","param_gttsize":"61440","param_pages_limit":"15728640"},"tuned_profile":"accelerator-performance"}'
run bash "$PROJECT_ROOT/scripts/benchmark/compare.sh" "$BEFORE_DIR" "$AFTER_DIR"
assert_success
assert_output --partial "Configuration changes"
}