cardosofelipe/pragma-stack
Template
1
0
Fork 1
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat(backend): add performance benchmarks and API security tests

Browse Source 
- Introduced `benchmark`, `benchmark-save`, and `benchmark-check` Makefile targets for performance testing.
- Added API security fuzzing through the `test-api-security` Makefile target, leveraging Schemathesis.
- Updated Dockerfiles to use Alpine for security and CVE mitigation.
- Enhanced security with `scan-image` and `scan-images` targets for Docker image vulnerability scanning via Trivy.
- Integrated `pytest-benchmark` for performance regression detection, with tests for key API endpoints.
- Extended `uv.lock` and `pyproject.toml` to include performance benchmarking dependencies.
This commit is contained in:
Felipe Cardoso
2026-03-01 16:16:18 +01:00
parent f8aafb250d
commit 4ceb8ad98c
8 changed files with 259 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
Makefile
View File

@@ -1,4 +1,4 @@
.PHONY: help dev dev-full prod down logs logs-dev clean clean-slate drop-db reset-db push-images deploy
.PHONY: help dev dev-full prod down logs logs-dev clean clean-slate drop-db reset-db push-images deploy scan-images
VERSION ?= latest
REGISTRY ?= ghcr.io/cardosofelipe/pragma-stack
@@ -21,6 +21,7 @@ help:
@echo " make prod - Start production stack"
@echo " make deploy - Pull and deploy latest images"
@echo " make push-images - Build and push images to registry"
@echo " make scan-images - Scan production images for CVEs (requires trivy)"
@echo " make logs - Follow production container logs"
@echo ""
@echo "Cleanup:"
@@ -89,6 +90,28 @@ push-images:
docker push $(REGISTRY)/backend:$(VERSION)
docker push $(REGISTRY)/frontend:$(VERSION)
scan-images:
@docker info > /dev/null 2>&1 || (echo "❌ Docker is not running!"; exit 1)
@echo "🐳 Building and scanning production images for CVEs..."
docker build -t $(REGISTRY)/backend:scan --target production ./backend
docker build -t $(REGISTRY)/frontend:scan --target runner ./frontend
@echo ""
@echo "=== Backend Image Scan ==="
@if command -v trivy > /dev/null 2>&1; then \
trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/backend:scan; \
else \
echo "ℹ️ Trivy not found locally, using Docker to run Trivy..."; \
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/backend:scan; \
fi
@echo ""
@echo "=== Frontend Image Scan ==="
@if command -v trivy > /dev/null 2>&1; then \
trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/frontend:scan; \
else \
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 $(REGISTRY)/frontend:scan; \
fi
@echo "✅ No HIGH/CRITICAL CVEs found in production images!"
# ============================================================================
# Cleanup
# ============================================================================

20
backend/Dockerfile
View File

@@ -33,11 +33,11 @@ RUN chmod +x /usr/local/bin/entrypoint.sh
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
# Production stage
FROM python:3.12-slim AS production
# Production stage — Alpine eliminates glibc CVEs (e.g. CVE-2026-0861)
FROM python:3.12-alpine AS production
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser
RUN addgroup -S appuser && adduser -S -G appuser appuser
WORKDIR /app
ENV PYTHONDONTWRITEBYTECODE=1 \
@@ -48,18 +48,18 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
UV_NO_CACHE=1
# Install system dependencies and uv
RUN apt-get update && \
apt-get install -y --no-install-recommends postgresql-client curl ca-certificates && \
RUN apk add --no-cache postgresql-client curl ca-certificates && \
curl -LsSf https://astral.sh/uv/install.sh | sh && \
mv /root/.local/bin/uv* /usr/local/bin/ && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
mv /root/.local/bin/uv* /usr/local/bin/
# Copy dependency files
COPY pyproject.toml uv.lock ./
# Install only production dependencies using uv (no dev dependencies)
RUN uv sync --frozen --no-dev
# Install build dependencies, compile Python packages, then remove build deps
RUN apk add --no-cache --virtual .build-deps \
gcc g++ musl-dev python3-dev linux-headers libffi-dev openssl-dev && \
uv sync --frozen --no-dev && \
apk del .build-deps
# Copy application code
COPY . .

47
backend/Makefile
View File

@@ -1,4 +1,4 @@
.PHONY: help lint lint-fix format format-check type-check test test-cov validate clean install-dev sync check-docker install-e2e test-e2e test-e2e-schema test-all dep-audit license-check audit validate-all check
.PHONY: help lint lint-fix format format-check type-check test test-cov validate clean install-dev sync check-docker install-e2e test-e2e test-e2e-schema test-all dep-audit license-check audit validate-all check benchmark benchmark-check benchmark-save scan-image test-api-security
# Prevent a stale VIRTUAL_ENV in the caller's shell from confusing uv
unexport VIRTUAL_ENV
@@ -18,12 +18,18 @@ help:
@echo " make format - Format code with Ruff"
@echo " make format-check - Check if code is formatted"
@echo " make type-check - Run pyright type checking"
@echo " make validate - Run all checks (lint + format + types)"
@echo " make validate - Run all checks (lint + format + types + schema fuzz)"
@echo ""
@echo "Performance:"
@echo " make benchmark - Run performance benchmarks"
@echo " make benchmark-save - Run benchmarks and save as baseline"
@echo " make benchmark-check - Run benchmarks and compare against baseline"
@echo ""
@echo "Security & Audit:"
@echo " make dep-audit - Scan dependencies for known vulnerabilities"
@echo " make license-check - Check dependency license compliance"
@echo " make audit - Run all security audits (deps + licenses)"
@echo " make scan-image - Scan Docker image for CVEs (requires trivy)"
@echo " make validate-all - Run all quality + security checks"
@echo " make check - Full pipeline: quality + security + tests"
@echo ""
@@ -77,9 +83,15 @@ type-check:
@echo "🔎 Running pyright type checking..."
@uv run pyright app/
validate: lint format-check type-check
validate: lint format-check type-check test-api-security
@echo "✅ All quality checks passed!"
# API Security Testing (Schemathesis property-based fuzzing)
test-api-security: check-docker
@echo "🔐 Running Schemathesis API security fuzzing..."
@IS_TEST=True PYTHONPATH=. uv run pytest tests/e2e/ -v -m "schemathesis" --tb=short -n 0
@echo "✅ API schema security tests passed!"
# ============================================================================
# Security & Audit
# ============================================================================
@@ -97,6 +109,17 @@ license-check:
audit: dep-audit license-check
@echo "✅ All security audits passed!"
scan-image: check-docker
@echo "🐳 Scanning Docker image for OS-level CVEs with Trivy..."
@docker build -t pragma-backend:scan -q --target production .
@if command -v trivy > /dev/null 2>&1; then \
trivy image --severity HIGH,CRITICAL --exit-code 1 pragma-backend:scan; \
else \
echo "ℹ️ Trivy not found locally, using Docker to run Trivy..."; \
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity HIGH,CRITICAL --exit-code 1 pragma-backend:scan; \
fi
@echo "✅ No HIGH/CRITICAL CVEs found in Docker image!"
validate-all: validate audit
@echo "✅ All quality + security checks passed!"
@@ -148,6 +171,24 @@ test-e2e-schema: check-docker
@echo "🧪 Running Schemathesis API schema tests..."
@IS_TEST=True PYTHONPATH=. uv run pytest tests/e2e/ -v -m "schemathesis" --tb=short -n 0
# ============================================================================
# Performance Benchmarks
# ============================================================================
benchmark:
@echo "⏱️ Running performance benchmarks..."
@IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-sort=mean -p no:xdist --override-ini='addopts='
benchmark-save:
@echo "⏱️ Running benchmarks and saving baseline..."
@IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-save=baseline --benchmark-sort=mean -p no:xdist --override-ini='addopts='
@echo "✅ Benchmark baseline saved to .benchmarks/"
benchmark-check:
@echo "⏱️ Running benchmarks and comparing against baseline..."
@IS_TEST=True PYTHONPATH=. uv run pytest tests/benchmarks/ -v --benchmark-only --benchmark-compare=0001_baseline --benchmark-sort=mean --benchmark-compare-fail=mean:200% -p no:xdist --override-ini='addopts='
@echo "✅ No performance regressions detected!"
test-all:
@echo "🧪 Running ALL tests (unit + E2E)..."
@$(MAKE) test

2
backend/entrypoint.sh
View File

@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
set -e
echo "Starting Backend"

6
backend/pyproject.toml
View File

@@ -72,6 +72,9 @@ dev = [
"pip-licenses>=4.0.0", # License compliance checking
"detect-secrets>=1.5.0", # Hardcoded secrets detection
# Performance benchmarking
"pytest-benchmark>=4.0.0", # Performance regression detection
# Pre-commit hooks
"pre-commit>=4.0.0", # Git pre-commit hook framework
]
@@ -206,12 +209,15 @@ addopts = [
"--cov=app",
"--cov-report=term-missing",
"--cov-report=html",
"--ignore=tests/benchmarks", # benchmarks are incompatible with xdist; run via 'make benchmark'
"-p", "no:benchmark", # disable pytest-benchmark plugin during normal runs (conflicts with xdist)
]
markers = [
"sqlite: marks tests that should run on SQLite (mocked).",
"postgres: marks tests that require a real PostgreSQL database.",
"e2e: marks end-to-end tests requiring Docker containers.",
"schemathesis: marks Schemathesis-generated API tests.",
"benchmark: marks performance benchmark tests.",
]
asyncio_default_fixture_loop_scope = "function"

0
backend/tests/benchmarks/__init__.py Normal file
View File

150
backend/tests/benchmarks/test_endpoint_performance.py Normal file
View File

@@ -0,0 +1,150 @@
"""
Performance Benchmark Tests.
These tests establish baseline performance metrics for critical API endpoints
and detect regressions when response times degrade significantly.
Usage:
make benchmark # Run benchmarks and save baseline
make benchmark-check # Run benchmarks and compare against saved baseline
Baselines are stored in .benchmarks/ and should be committed to version control
so CI can detect performance regressions across commits.
"""
import time
import uuid
from unittest.mock import patch
import pytest
import pytest_asyncio
from fastapi.testclient import TestClient
from app.main import app
pytestmark = [pytest.mark.benchmark]
# =============================================================================
# Fixtures
# =============================================================================
@pytest.fixture
def sync_client():
"""Create a FastAPI test client with mocked database for stateless endpoints."""
with patch("app.main.check_database_health") as mock_health_check:
mock_health_check.return_value = True
yield TestClient(app)
# =============================================================================
# Stateless Endpoint Benchmarks (no DB required)
# =============================================================================
def test_health_endpoint_performance(sync_client, benchmark):
"""Benchmark: GET /health should respond within acceptable latency."""
result = benchmark(sync_client.get, "/health")
assert result.status_code == 200
def test_openapi_schema_performance(sync_client, benchmark):
"""Benchmark: OpenAPI schema generation should not regress."""
result = benchmark(sync_client.get, "/api/v1/openapi.json")
assert result.status_code == 200
# =============================================================================
# Database-dependent Endpoint Benchmarks (async, manual timing)
#
# pytest-benchmark does not support async functions natively. These tests
# measure latency manually and assert against a maximum threshold (in ms)
# to catch performance regressions.
# =============================================================================
MAX_LOGIN_MS = 500
MAX_GET_USER_MS = 200
@pytest_asyncio.fixture
async def bench_user(async_test_db):
"""Create a test user for benchmark tests."""
from app.core.auth import get_password_hash
from app.models.user import User
_test_engine, AsyncTestingSessionLocal = async_test_db
async with AsyncTestingSessionLocal() as session:
user = User(
id=uuid.uuid4(),
email="bench@example.com",
password_hash=get_password_hash("BenchPass123!"),
first_name="Bench",
last_name="User",
is_active=True,
is_superuser=False,
)
session.add(user)
await session.commit()
await session.refresh(user)
return user
@pytest_asyncio.fixture
async def bench_token(client, bench_user):
"""Get an auth token for the benchmark user."""
response = await client.post(
"/api/v1/auth/login",
json={"email": "bench@example.com", "password": "BenchPass123!"},
)
assert response.status_code == 200, f"Login failed: {response.text}"
return response.json()["access_token"]
@pytest.mark.asyncio
async def test_login_latency(client, bench_user):
"""Performance: POST /api/v1/auth/login must respond under threshold."""
iterations = 5
total_ms = 0.0
for _ in range(iterations):
start = time.perf_counter()
response = await client.post(
"/api/v1/auth/login",
json={"email": "bench@example.com", "password": "BenchPass123!"},
)
elapsed_ms = (time.perf_counter() - start) * 1000
total_ms += elapsed_ms
assert response.status_code == 200
mean_ms = total_ms / iterations
print(f"\n Login mean latency: {mean_ms:.1f}ms (threshold: {MAX_LOGIN_MS}ms)")
assert mean_ms < MAX_LOGIN_MS, (
f"Login latency regression: {mean_ms:.1f}ms exceeds {MAX_LOGIN_MS}ms threshold"
)
@pytest.mark.asyncio
async def test_get_current_user_latency(client, bench_token):
"""Performance: GET /api/v1/users/me must respond under threshold."""
iterations = 10
total_ms = 0.0
for _ in range(iterations):
start = time.perf_counter()
response = await client.get(
"/api/v1/users/me",
headers={"Authorization": f"Bearer {bench_token}"},
)
elapsed_ms = (time.perf_counter() - start) * 1000
total_ms += elapsed_ms
assert response.status_code == 200
mean_ms = total_ms / iterations
print(
f"\n Get user mean latency: {mean_ms:.1f}ms (threshold: {MAX_GET_USER_MS}ms)"
)
assert mean_ms < MAX_GET_USER_MS, (
f"Get user latency regression: {mean_ms:.1f}ms exceeds {MAX_GET_USER_MS}ms threshold"
)

24
backend/uv.lock generated
View File

@@ -615,6 +615,7 @@ dev = [
{ name = "pyright" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-benchmark" },
{ name = "pytest-cov" },
{ name = "pytest-xdist" },
{ name = "requests" },
@@ -651,6 +652,7 @@ requires-dist = [
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.390" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.5" },
{ name = "pytest-benchmark", marker = "extra == 'dev'", specifier = ">=4.0.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" },
{ name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" },
@@ -1400,6 +1402,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" },
]
[[package]]
name = "py-cpuinfo"
version = "9.0.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" },
]
[[package]]
name = "py-serializable"
version = "2.1.0"
@@ -1599,6 +1610,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/93/2fa34714b7a4ae72f2f8dad66ba17dd9a2c793220719e736dda28b7aec27/pytest_asyncio-1.2.0-py3-none-any.whl", hash = "sha256:8e17ae5e46d8e7efe51ab6494dd2010f4ca8dae51652aa3c8d55acf50bfb2e99", size = 15095, upload-time = "2025-09-12T07:33:52.639Z" },
]
[[package]]
name = "pytest-benchmark"
version = "5.2.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "py-cpuinfo" },
{ name = "pytest" },
]
sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" },
]
[[package]]
name = "pytest-cov"
version = "7.0.0"