diff --git a/mcp-servers/knowledge-base/Makefile b/mcp-servers/knowledge-base/Makefile new file mode 100644 index 0000000..13758c7 --- /dev/null +++ b/mcp-servers/knowledge-base/Makefile @@ -0,0 +1,79 @@ +.PHONY: help install install-dev lint lint-fix format type-check test test-cov validate clean run + +# Default target +help: + @echo "Knowledge Base MCP Server - Development Commands" + @echo "" + @echo "Setup:" + @echo " make install - Install production dependencies" + @echo " make install-dev - Install development dependencies" + @echo "" + @echo "Quality Checks:" + @echo " make lint - Run Ruff linter" + @echo " make lint-fix - Run Ruff linter with auto-fix" + @echo " make format - Format code with Ruff" + @echo " make type-check - Run mypy type checker" + @echo "" + @echo "Testing:" + @echo " make test - Run pytest" + @echo " make test-cov - Run pytest with coverage" + @echo "" + @echo "All-in-one:" + @echo " make validate - Run lint, type-check, and tests" + @echo "" + @echo "Running:" + @echo " make run - Run the server locally" + @echo "" + @echo "Cleanup:" + @echo " make clean - Remove cache and build artifacts" + +# Setup +install: + @echo "Installing production dependencies..." + @uv pip install -e . + +install-dev: + @echo "Installing development dependencies..." + @uv pip install -e ".[dev]" + +# Quality checks +lint: + @echo "Running Ruff linter..." + @uv run ruff check . + +lint-fix: + @echo "Running Ruff linter with auto-fix..." + @uv run ruff check --fix . + +format: + @echo "Formatting code..." + @uv run ruff format . + +type-check: + @echo "Running mypy..." + @uv run mypy . --ignore-missing-imports + +# Testing +test: + @echo "Running tests..." + @uv run pytest tests/ -v + +test-cov: + @echo "Running tests with coverage..." + @uv run pytest tests/ -v --cov=. --cov-report=term-missing --cov-report=html + +# All-in-one validation +validate: lint type-check test + @echo "All validations passed!" + +# Running +run: + @echo "Starting Knowledge Base server..." + @uv run python server.py + +# Cleanup +clean: + @echo "Cleaning up..." + @rm -rf __pycache__ .pytest_cache .mypy_cache .ruff_cache .coverage htmlcov + @find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + @find . -type f -name "*.pyc" -delete 2>/dev/null || true diff --git a/mcp-servers/knowledge-base/collection_manager.py b/mcp-servers/knowledge-base/collection_manager.py index 25083a5..a0b774b 100644 --- a/mcp-servers/knowledge-base/collection_manager.py +++ b/mcp-servers/knowledge-base/collection_manager.py @@ -328,7 +328,7 @@ class CollectionManager: "source_path": chunk.source_path or source_path, "start_line": chunk.start_line, "end_line": chunk.end_line, - "file_type": (chunk.file_type or file_type).value if (chunk.file_type or file_type) else None, + "file_type": effective_file_type.value if (effective_file_type := chunk.file_type or file_type) else None, } embeddings_data.append(( chunk.content, diff --git a/mcp-servers/knowledge-base/database.py b/mcp-servers/knowledge-base/database.py index e28f00f..41c2da5 100644 --- a/mcp-servers/knowledge-base/database.py +++ b/mcp-servers/knowledge-base/database.py @@ -284,41 +284,40 @@ class DatabaseManager: ) try: - async with self.acquire() as conn: + async with self.acquire() as conn, conn.transaction(): # Wrap in transaction for all-or-nothing batch semantics - async with conn.transaction(): - for project_id, collection, content, embedding, chunk_type, metadata in embeddings: - content_hash = self.compute_content_hash(content) - source_path = metadata.get("source_path") - start_line = metadata.get("start_line") - end_line = metadata.get("end_line") - file_type = metadata.get("file_type") + for project_id, collection, content, embedding, chunk_type, metadata in embeddings: + content_hash = self.compute_content_hash(content) + source_path = metadata.get("source_path") + start_line = metadata.get("start_line") + end_line = metadata.get("end_line") + file_type = metadata.get("file_type") - embedding_id = await conn.fetchval( - """ - INSERT INTO knowledge_embeddings - (project_id, collection, content, embedding, chunk_type, - source_path, start_line, end_line, file_type, metadata, - content_hash, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - ON CONFLICT DO NOTHING - RETURNING id - """, - project_id, - collection, - content, - embedding, - chunk_type.value, - source_path, - start_line, - end_line, - file_type, - metadata, - content_hash, - expires_at, - ) - if embedding_id: - ids.append(str(embedding_id)) + embedding_id = await conn.fetchval( + """ + INSERT INTO knowledge_embeddings + (project_id, collection, content, embedding, chunk_type, + source_path, start_line, end_line, file_type, metadata, + content_hash, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT DO NOTHING + RETURNING id + """, + project_id, + collection, + content, + embedding, + chunk_type.value, + source_path, + start_line, + end_line, + file_type, + metadata, + content_hash, + expires_at, + ) + if embedding_id: + ids.append(str(embedding_id)) logger.info(f"Stored {len(ids)} embeddings in batch") return ids @@ -566,59 +565,58 @@ class DatabaseManager: ) try: - async with self.acquire() as conn: + async with self.acquire() as conn, conn.transaction(): # Use transaction for atomic replace - async with conn.transaction(): - # First, delete existing embeddings for this source - delete_result = await conn.execute( + # First, delete existing embeddings for this source + delete_result = await conn.execute( + """ + DELETE FROM knowledge_embeddings + WHERE project_id = $1 AND source_path = $2 AND collection = $3 + """, + project_id, + source_path, + collection, + ) + deleted_count = int(delete_result.split()[-1]) + + # Then insert new embeddings + new_ids = [] + for content, embedding, chunk_type, metadata in embeddings: + content_hash = self.compute_content_hash(content) + start_line = metadata.get("start_line") + end_line = metadata.get("end_line") + file_type = metadata.get("file_type") + + embedding_id = await conn.fetchval( """ - DELETE FROM knowledge_embeddings - WHERE project_id = $1 AND source_path = $2 AND collection = $3 + INSERT INTO knowledge_embeddings + (project_id, collection, content, embedding, chunk_type, + source_path, start_line, end_line, file_type, metadata, + content_hash, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + RETURNING id """, project_id, - source_path, collection, + content, + embedding, + chunk_type.value, + source_path, + start_line, + end_line, + file_type, + metadata, + content_hash, + expires_at, ) - deleted_count = int(delete_result.split()[-1]) + if embedding_id: + new_ids.append(str(embedding_id)) - # Then insert new embeddings - new_ids = [] - for content, embedding, chunk_type, metadata in embeddings: - content_hash = self.compute_content_hash(content) - start_line = metadata.get("start_line") - end_line = metadata.get("end_line") - file_type = metadata.get("file_type") - - embedding_id = await conn.fetchval( - """ - INSERT INTO knowledge_embeddings - (project_id, collection, content, embedding, chunk_type, - source_path, start_line, end_line, file_type, metadata, - content_hash, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - RETURNING id - """, - project_id, - collection, - content, - embedding, - chunk_type.value, - source_path, - start_line, - end_line, - file_type, - metadata, - content_hash, - expires_at, - ) - if embedding_id: - new_ids.append(str(embedding_id)) - - logger.info( - f"Replaced source {source_path}: deleted {deleted_count}, " - f"inserted {len(new_ids)} embeddings" - ) - return deleted_count, new_ids + logger.info( + f"Replaced source {source_path}: deleted {deleted_count}, " + f"inserted {len(new_ids)} embeddings" + ) + return deleted_count, new_ids except asyncpg.PostgresError as e: logger.error(f"Replace source error: {e}") diff --git a/mcp-servers/knowledge-base/server.py b/mcp-servers/knowledge-base/server.py index 7a5595a..b972901 100644 --- a/mcp-servers/knowledge-base/server.py +++ b/mcp-servers/knowledge-base/server.py @@ -193,7 +193,7 @@ async def health_check() -> dict[str, Any]: # Check Redis cache (non-critical - degraded without it) try: if _embeddings and _embeddings._redis: - await _embeddings._redis.ping() + await _embeddings._redis.ping() # type: ignore[misc] status["dependencies"]["redis"] = "connected" else: status["dependencies"]["redis"] = "not initialized" diff --git a/mcp-servers/knowledge-base/tests/test_server.py b/mcp-servers/knowledge-base/tests/test_server.py index e40f701..c04e3f9 100644 --- a/mcp-servers/knowledge-base/tests/test_server.py +++ b/mcp-servers/knowledge-base/tests/test_server.py @@ -1,8 +1,7 @@ """Tests for server module and MCP tools.""" -import json from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi.testclient import TestClient diff --git a/mcp-servers/llm-gateway/Dockerfile b/mcp-servers/llm-gateway/Dockerfile index 38c5442..244c11b 100644 --- a/mcp-servers/llm-gateway/Dockerfile +++ b/mcp-servers/llm-gateway/Dockerfile @@ -1,39 +1,25 @@ # Syndarix LLM Gateway MCP Server -# Multi-stage build for minimal image size +FROM python:3.12-slim -# Build stage -FROM python:3.12-slim AS builder +WORKDIR /app + +# Install system dependencies (needed for tiktoken regex compilation) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* # Install uv for fast package management -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv -WORKDIR /app - -# Copy dependency files +# Copy project files COPY pyproject.toml ./ +COPY *.py ./ -# Create virtual environment and install dependencies -RUN uv venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" -RUN uv pip install -e . - -# Runtime stage -FROM python:3.12-slim AS runtime +# Install dependencies to system Python +RUN uv pip install --system --no-cache . # Create non-root user for security -RUN groupadd --gid 1000 appgroup && \ - useradd --uid 1000 --gid appgroup --shell /bin/bash --create-home appuser - -WORKDIR /app - -# Copy virtual environment from builder -COPY --from=builder /app/.venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" - -# Copy application code -COPY --chown=appuser:appgroup . . - -# Switch to non-root user +RUN useradd --create-home --shell /bin/bash appuser USER appuser # Environment variables @@ -47,7 +33,7 @@ EXPOSE 8001 # Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()" || exit 1 + CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()" # Run the server CMD ["python", "server.py"] diff --git a/mcp-servers/llm-gateway/Makefile b/mcp-servers/llm-gateway/Makefile new file mode 100644 index 0000000..bf1fb73 --- /dev/null +++ b/mcp-servers/llm-gateway/Makefile @@ -0,0 +1,79 @@ +.PHONY: help install install-dev lint lint-fix format type-check test test-cov validate clean run + +# Default target +help: + @echo "LLM Gateway MCP Server - Development Commands" + @echo "" + @echo "Setup:" + @echo " make install - Install production dependencies" + @echo " make install-dev - Install development dependencies" + @echo "" + @echo "Quality Checks:" + @echo " make lint - Run Ruff linter" + @echo " make lint-fix - Run Ruff linter with auto-fix" + @echo " make format - Format code with Ruff" + @echo " make type-check - Run mypy type checker" + @echo "" + @echo "Testing:" + @echo " make test - Run pytest" + @echo " make test-cov - Run pytest with coverage" + @echo "" + @echo "All-in-one:" + @echo " make validate - Run lint, type-check, and tests" + @echo "" + @echo "Running:" + @echo " make run - Run the server locally" + @echo "" + @echo "Cleanup:" + @echo " make clean - Remove cache and build artifacts" + +# Setup +install: + @echo "Installing production dependencies..." + @uv pip install -e . + +install-dev: + @echo "Installing development dependencies..." + @uv pip install -e ".[dev]" + +# Quality checks +lint: + @echo "Running Ruff linter..." + @uv run ruff check . + +lint-fix: + @echo "Running Ruff linter with auto-fix..." + @uv run ruff check --fix . + +format: + @echo "Formatting code..." + @uv run ruff format . + +type-check: + @echo "Running mypy..." + @uv run mypy . --ignore-missing-imports + +# Testing +test: + @echo "Running tests..." + @uv run pytest tests/ -v + +test-cov: + @echo "Running tests with coverage..." + @uv run pytest tests/ -v --cov=. --cov-report=term-missing --cov-report=html + +# All-in-one validation +validate: lint type-check test + @echo "All validations passed!" + +# Running +run: + @echo "Starting LLM Gateway server..." + @uv run python server.py + +# Cleanup +clean: + @echo "Cleaning up..." + @rm -rf __pycache__ .pytest_cache .mypy_cache .ruff_cache .coverage htmlcov + @find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + @find . -type f -name "*.pyc" -delete 2>/dev/null || true diff --git a/mcp-servers/llm-gateway/failover.py b/mcp-servers/llm-gateway/failover.py index 185b6ee..45df882 100644 --- a/mcp-servers/llm-gateway/failover.py +++ b/mcp-servers/llm-gateway/failover.py @@ -110,14 +110,13 @@ class CircuitBreaker: """ if self._state == CircuitState.OPEN: time_in_open = time.time() - self._stats.state_changed_at - if time_in_open >= self.recovery_timeout: - # Only transition if still in OPEN state (double-check) - if self._state == CircuitState.OPEN: - self._transition_to(CircuitState.HALF_OPEN) - logger.info( - f"Circuit {self.name} transitioned to HALF_OPEN " - f"after {time_in_open:.1f}s" - ) + # Double-check state after time calculation (for thread safety) + if time_in_open >= self.recovery_timeout and self._state == CircuitState.OPEN: + self._transition_to(CircuitState.HALF_OPEN) + logger.info( + f"Circuit {self.name} transitioned to HALF_OPEN " + f"after {time_in_open:.1f}s" + ) def _transition_to(self, new_state: CircuitState) -> None: """Transition to a new state."""