From 4ceb8ad98cb8cfb5c5731aed5d36901d31175058 Mon Sep 17 00:00:00 2001 From: Felipe Cardoso Date: Sun, 1 Mar 2026 16:16:18 +0100 Subject: [PATCH] feat(backend): add performance benchmarks and API security tests - Introduced `benchmark`, `benchmark-save`, and `benchmark-check` Makefile targets for performance testing. - Added API security fuzzing through the `test-api-security` Makefile target, leveraging Schemathesis. - Updated Dockerfiles to use Alpine for security and CVE mitigation. - Enhanced security with `scan-image` and `scan-images` targets for Docker image vulnerability scanning via Trivy. - Integrated `pytest-benchmark` for performance regression detection, with tests for key API endpoints. - Extended `uv.lock` and `pyproject.toml` to include performance benchmarking dependencies. --- Makefile | 25 ++- backend/Dockerfile | 20 +-- backend/Makefile | 47 +++++- backend/entrypoint.sh | 2 +- backend/pyproject.toml | 6 + backend/tests/benchmarks/__init__.py | 0 .../benchmarks/test_endpoint_performance.py | 150 ++++++++++++++++++ backend/uv.lock | 24 +++ 8 files changed, 259 insertions(+), 15 deletions(-) create mode 100644 backend/tests/benchmarks/__init__.py create mode 100644 backend/tests/benchmarks/test_endpoint_performance.py diff --git a/Makefile b/Makefile index 29e3f68..d8c99cf 100755 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help dev dev-full prod down logs logs-dev clean clean-slate drop-db reset-db push-images deploy +.PHONY: help dev dev-full prod down logs logs-dev clean clean-slate drop-db reset-db push-images deploy scan-images VERSION ?= latest REGISTRY ?= ghcr.io/cardosofelipe/pragma-stack @@ -21,6 +21,7 @@ help: @echo " make prod - Start production stack" @echo " make deploy - Pull and deploy latest images" @echo " make push-images - Build and push images to registry" + @echo " make scan-images - Scan production images for CVEs (requires trivy)" @echo " make logs - Follow production container logs" @echo "" @echo "Cleanup:" @@ -89,6 +90,28 @@ push-images: docker push $(REGISTRY)/backend:$(VERSION) docker push $(REGISTRY)/frontend:$(VERSION) +scan-images: + @docker info > /dev/null 2>&1 || (echo "โŒ Docker is not running!"; exit 1) + @echo "๐Ÿณ Building and scanning production images for CVEs..." + docker build -t $(REGISTRY)/backend:scan --target production ./backend + docker build -t $(REGISTRY)/frontend:scan --target runner ./frontend + @echo "" + @echo "=== Backend Image Scan ===" + @if command -v trivy > /dev/null 2>&1; then \ + trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/backend:scan; \ + else \ + echo "โ„น๏ธ Trivy not found locally, using Docker to run Trivy..."; \ + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/backend:scan; \ + fi + @echo "" + @echo "=== Frontend Image Scan ===" + @if command -v trivy > /dev/null 2>&1; then \ + trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/frontend:scan; \ + else \ + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/frontend:scan; \ + fi + @echo "โœ… No HIGH/CRITICAL CVEs found in production images!" + # ============================================================================ # Cleanup # ============================================================================ diff --git a/backend/Dockerfile b/backend/Dockerfile index 554865b..4c35a7c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -33,11 +33,11 @@ RUN chmod +x /usr/local/bin/entrypoint.sh ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] -# Production stage -FROM python:3.12-slim AS production +# Production stage โ€” Alpine eliminates glibc CVEs (e.g. CVE-2026-0861) +FROM python:3.12-alpine AS production # Create non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser +RUN addgroup -S appuser && adduser -S -G appuser appuser WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 \ @@ -48,18 +48,18 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ UV_NO_CACHE=1 # Install system dependencies and uv -RUN apt-get update && \ - apt-get install -y --no-install-recommends postgresql-client curl ca-certificates && \ +RUN apk add --no-cache postgresql-client curl ca-certificates && \ curl -LsSf https://astral.sh/uv/install.sh | sh && \ - mv /root/.local/bin/uv* /usr/local/bin/ && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + mv /root/.local/bin/uv* /usr/local/bin/ # Copy dependency files COPY pyproject.toml uv.lock ./ -# Install only production dependencies using uv (no dev dependencies) -RUN uv sync --frozen --no-dev +# Install build dependencies, compile Python packages, then remove build deps +RUN apk add --no-cache --virtual .build-deps \ + gcc g++ musl-dev python3-dev linux-headers libffi-dev openssl-dev && \ + uv sync --frozen --no-dev && \ + apk del .build-deps # Copy application code COPY . . diff --git a/backend/Makefile b/backend/Makefile index ffe0fea..f42d010 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,4 +1,4 @@ -.PHONY: help lint lint-fix format format-check type-check test test-cov validate clean install-dev sync check-docker install-e2e test-e2e test-e2e-schema test-all dep-audit license-check audit validate-all check +.PHONY: help lint lint-fix format format-check type-check test test-cov validate clean install-dev sync check-docker install-e2e test-e2e test-e2e-schema test-all dep-audit license-check audit validate-all check benchmark benchmark-check benchmark-save scan-image test-api-security # Prevent a stale VIRTUAL_ENV in the caller's shell from confusing uv unexport VIRTUAL_ENV @@ -18,12 +18,18 @@ help: @echo " make format - Format code with Ruff" @echo " make format-check - Check if code is formatted" @echo " make type-check - Run pyright type checking" - @echo " make validate - Run all checks (lint + format + types)" + @echo " make validate - Run all checks (lint + format + types + schema fuzz)" + @echo "" + @echo "Performance:" + @echo " make benchmark - Run performance benchmarks" + @echo " make benchmark-save - Run benchmarks and save as baseline" + @echo " make benchmark-check - Run benchmarks and compare against baseline" @echo "" @echo "Security & Audit:" @echo " make dep-audit - Scan dependencies for known vulnerabilities" @echo " make license-check - Check dependency license compliance" @echo " make audit - Run all security audits (deps + licenses)" + @echo " make scan-image - Scan Docker image for CVEs (requires trivy)" @echo " make validate-all - Run all quality + security checks" @echo " make check - Full pipeline: quality + security + tests" @echo "" @@ -77,9 +83,15 @@ type-check: @echo "๐Ÿ”Ž Running pyright type checking..." @uv run pyright app/ -validate: lint format-check type-check +validate: lint format-check type-check test-api-security @echo "โœ… All quality checks passed!" +# API Security Testing (Schemathesis property-based fuzzing) +test-api-security: check-docker + @echo "๐Ÿ” Running Schemathesis API security fuzzing..." + @IS_TEST=True PYTHONPATH=. uv run pytest tests/e2e/ -v -m "schemathesis" --tb=short -n 0 + @echo "โœ… API schema security tests passed!" + # ============================================================================ # Security & Audit # ============================================================================ @@ -97,6 +109,17 @@ license-check: audit: dep-audit license-check @echo "โœ… All security audits passed!" +scan-image: check-docker + @echo "๐Ÿณ Scanning Docker image for OS-level CVEs with Trivy..." + @docker build -t pragma-backend:scan -q --target production . + @if command -v trivy > /dev/null 2>&1; then \ + trivy image --severity HIGH,CRITICAL --exit-code 1 pragma-backend:scan; \ + else \ + echo "โ„น๏ธ Trivy not found locally, using Docker to run Trivy..."; \ + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 pragma-backend:scan; \ + fi + @echo "โœ… No HIGH/CRITICAL CVEs found in Docker image!" + validate-all: validate audit @echo "โœ… All quality + security checks passed!" @@ -148,6 +171,24 @@ test-e2e-schema: check-docker @echo "๐Ÿงช Running Schemathesis API schema tests..." @IS_TEST=True PYTHONPATH=. uv run pytest tests/e2e/ -v -m "schemathesis" --tb=short -n 0 +# ============================================================================ +# Performance Benchmarks +# ============================================================================ + +benchmark: + @echo "โฑ๏ธ Running performance benchmarks..." + @IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-sort=mean -p no:xdist --override-ini='addopts=' + +benchmark-save: + @echo "โฑ๏ธ Running benchmarks and saving baseline..." + @IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-save=baseline --benchmark-sort=mean -p no:xdist --override-ini='addopts=' + @echo "โœ… Benchmark baseline saved to .benchmarks/" + +benchmark-check: + @echo "โฑ๏ธ Running benchmarks and comparing against baseline..." + @IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-compare=0001_baseline --benchmark-sort=mean --benchmark-compare-fail=mean:200% -p no:xdist --override-ini='addopts=' + @echo "โœ… No performance regressions detected!" + test-all: @echo "๐Ÿงช Running ALL tests (unit + E2E)..." @$(MAKE) test diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 78f18c8..db68a4f 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e echo "Starting Backend" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f7ce481..2682e18 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -72,6 +72,9 @@ dev = [ "pip-licenses>=4.0.0", # License compliance checking "detect-secrets>=1.5.0", # Hardcoded secrets detection + # Performance benchmarking + "pytest-benchmark>=4.0.0", # Performance regression detection + # Pre-commit hooks "pre-commit>=4.0.0", # Git pre-commit hook framework ] @@ -206,12 +209,15 @@ addopts = [ "--cov=app", "--cov-report=term-missing", "--cov-report=html", + "--ignore=tests/benchmarks", # benchmarks are incompatible with xdist; run via 'make benchmark' + "-p", "no:benchmark", # disable pytest-benchmark plugin during normal runs (conflicts with xdist) ] markers = [ "sqlite: marks tests that should run on SQLite (mocked).", "postgres: marks tests that require a real PostgreSQL database.", "e2e: marks end-to-end tests requiring Docker containers.", "schemathesis: marks Schemathesis-generated API tests.", + "benchmark: marks performance benchmark tests.", ] asyncio_default_fixture_loop_scope = "function" diff --git a/backend/tests/benchmarks/__init__.py b/backend/tests/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/benchmarks/test_endpoint_performance.py b/backend/tests/benchmarks/test_endpoint_performance.py new file mode 100644 index 0000000..82b503d --- /dev/null +++ b/backend/tests/benchmarks/test_endpoint_performance.py @@ -0,0 +1,150 @@ +""" +Performance Benchmark Tests. + +These tests establish baseline performance metrics for critical API endpoints +and detect regressions when response times degrade significantly. + +Usage: + make benchmark # Run benchmarks and save baseline + make benchmark-check # Run benchmarks and compare against saved baseline + +Baselines are stored in .benchmarks/ and should be committed to version control +so CI can detect performance regressions across commits. +""" + +import time +import uuid +from unittest.mock import patch + +import pytest +import pytest_asyncio +from fastapi.testclient import TestClient + +from app.main import app + +pytestmark = [pytest.mark.benchmark] + + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def sync_client(): + """Create a FastAPI test client with mocked database for stateless endpoints.""" + with patch("app.main.check_database_health") as mock_health_check: + mock_health_check.return_value = True + yield TestClient(app) + + +# ============================================================================= +# Stateless Endpoint Benchmarks (no DB required) +# ============================================================================= + + +def test_health_endpoint_performance(sync_client, benchmark): + """Benchmark: GET /health should respond within acceptable latency.""" + result = benchmark(sync_client.get, "/health") + assert result.status_code == 200 + + +def test_openapi_schema_performance(sync_client, benchmark): + """Benchmark: OpenAPI schema generation should not regress.""" + result = benchmark(sync_client.get, "/api/v1/openapi.json") + assert result.status_code == 200 + + +# ============================================================================= +# Database-dependent Endpoint Benchmarks (async, manual timing) +# +# pytest-benchmark does not support async functions natively. These tests +# measure latency manually and assert against a maximum threshold (in ms) +# to catch performance regressions. +# ============================================================================= + +MAX_LOGIN_MS = 500 +MAX_GET_USER_MS = 200 + + +@pytest_asyncio.fixture +async def bench_user(async_test_db): + """Create a test user for benchmark tests.""" + from app.core.auth import get_password_hash + from app.models.user import User + + _test_engine, AsyncTestingSessionLocal = async_test_db + + async with AsyncTestingSessionLocal() as session: + user = User( + id=uuid.uuid4(), + email="bench@example.com", + password_hash=get_password_hash("BenchPass123!"), + first_name="Bench", + last_name="User", + is_active=True, + is_superuser=False, + ) + session.add(user) + await session.commit() + await session.refresh(user) + return user + + +@pytest_asyncio.fixture +async def bench_token(client, bench_user): + """Get an auth token for the benchmark user.""" + response = await client.post( + "/api/v1/auth/login", + json={"email": "bench@example.com", "password": "BenchPass123!"}, + ) + assert response.status_code == 200, f"Login failed: {response.text}" + return response.json()["access_token"] + + +@pytest.mark.asyncio +async def test_login_latency(client, bench_user): + """Performance: POST /api/v1/auth/login must respond under threshold.""" + iterations = 5 + total_ms = 0.0 + + for _ in range(iterations): + start = time.perf_counter() + response = await client.post( + "/api/v1/auth/login", + json={"email": "bench@example.com", "password": "BenchPass123!"}, + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + total_ms += elapsed_ms + assert response.status_code == 200 + + mean_ms = total_ms / iterations + print(f"\n Login mean latency: {mean_ms:.1f}ms (threshold: {MAX_LOGIN_MS}ms)") + assert mean_ms < MAX_LOGIN_MS, ( + f"Login latency regression: {mean_ms:.1f}ms exceeds {MAX_LOGIN_MS}ms threshold" + ) + + +@pytest.mark.asyncio +async def test_get_current_user_latency(client, bench_token): + """Performance: GET /api/v1/users/me must respond under threshold.""" + iterations = 10 + total_ms = 0.0 + + for _ in range(iterations): + start = time.perf_counter() + response = await client.get( + "/api/v1/users/me", + headers={"Authorization": f"Bearer {bench_token}"}, + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + total_ms += elapsed_ms + assert response.status_code == 200 + + mean_ms = total_ms / iterations + print( + f"\n Get user mean latency: {mean_ms:.1f}ms (threshold: {MAX_GET_USER_MS}ms)" + ) + assert mean_ms < MAX_GET_USER_MS, ( + f"Get user latency regression: {mean_ms:.1f}ms exceeds {MAX_GET_USER_MS}ms threshold" + ) diff --git a/backend/uv.lock b/backend/uv.lock index dd42baf..d0a0008 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -615,6 +615,7 @@ dev = [ { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-xdist" }, { name = "requests" }, @@ -651,6 +652,7 @@ requires-dist = [ { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.390" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.5" }, + { name = "pytest-benchmark", marker = "extra == 'dev'", specifier = ">=4.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, @@ -1400,6 +1402,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "py-serializable" version = "2.1.0" @@ -1599,6 +1610,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/93/2fa34714b7a4ae72f2f8dad66ba17dd9a2c793220719e736dda28b7aec27/pytest_asyncio-1.2.0-py3-none-any.whl", hash = "sha256:8e17ae5e46d8e7efe51ab6494dd2010f4ca8dae51652aa3c8d55acf50bfb2e99", size = 15095, upload-time = "2025-09-12T07:33:52.639Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, +] + [[package]] name = "pytest-cov" version = "7.0.0"