""" Agent E2E Workflow Tests. Tests complete workflows for AI agents including: - Agent type management (admin-only) - Agent instance spawning and lifecycle - Agent status transitions (pause/resume/terminate) - Authorization and access control Usage: make test-e2e # Run all E2E tests """ from uuid import uuid4 import pytest pytestmark = [ pytest.mark.e2e, pytest.mark.postgres, pytest.mark.asyncio, ] class TestAgentTypesAdminWorkflows: """Test agent type management (admin-only operations).""" async def test_create_agent_type_requires_superuser(self, e2e_client): """Test that creating agent types requires superuser privileges.""" # Register regular user email = f"regular-{uuid4().hex[:8]}@example.com" password = "RegularPass123!" await e2e_client.post( "/api/v1/auth/register", json={ "email": email, "password": password, "first_name": "Regular", "last_name": "User", }, ) login_resp = await e2e_client.post( "/api/v1/auth/login", json={"email": email, "password": password}, ) tokens = login_resp.json() # Try to create agent type response = await e2e_client.post( "/api/v1/agent-types", headers={"Authorization": f"Bearer {tokens['access_token']}"}, json={ "name": "Test Agent", "slug": f"test-agent-{uuid4().hex[:8]}", "personality_prompt": "You are a helpful assistant.", "primary_model": "claude-3-sonnet", }, ) assert response.status_code == 403 async def test_superuser_can_create_agent_type(self, e2e_client, e2e_superuser): """Test that superuser can create and manage agent types.""" slug = f"test-type-{uuid4().hex[:8]}" # Create agent type create_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Product Owner Agent", "slug": slug, "description": "A product owner agent for requirements gathering", "expertise": ["requirements", "user_stories", "prioritization"], "personality_prompt": "You are a product owner focused on delivering value.", "primary_model": "claude-3-opus", "fallback_models": ["claude-3-sonnet"], "model_params": {"temperature": 0.7, "max_tokens": 4000}, "mcp_servers": ["knowledge-base"], "is_active": True, }, ) assert create_resp.status_code == 201, f"Failed: {create_resp.text}" agent_type = create_resp.json() assert agent_type["name"] == "Product Owner Agent" assert agent_type["slug"] == slug assert agent_type["primary_model"] == "claude-3-opus" assert agent_type["is_active"] is True assert "requirements" in agent_type["expertise"] async def test_list_agent_types_public(self, e2e_client, e2e_superuser): """Test that any authenticated user can list agent types.""" # First create an agent type as superuser slug = f"list-test-{uuid4().hex[:8]}" await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": f"List Test Agent {slug}", "slug": slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) # Register regular user email = f"lister-{uuid4().hex[:8]}@example.com" password = "ListerPass123!" await e2e_client.post( "/api/v1/auth/register", json={ "email": email, "password": password, "first_name": "List", "last_name": "User", }, ) login_resp = await e2e_client.post( "/api/v1/auth/login", json={"email": email, "password": password}, ) tokens = login_resp.json() # List agent types as regular user list_resp = await e2e_client.get( "/api/v1/agent-types", headers={"Authorization": f"Bearer {tokens['access_token']}"}, ) assert list_resp.status_code == 200 data = list_resp.json() assert "data" in data assert "pagination" in data assert data["pagination"]["total"] >= 1 async def test_get_agent_type_by_slug(self, e2e_client, e2e_superuser): """Test getting agent type by slug.""" slug = f"slug-test-{uuid4().hex[:8]}" # Create agent type await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": f"Slug Test {slug}", "slug": slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) # Get by slug (route is /slug/{slug}, not /by-slug/{slug}) get_resp = await e2e_client.get( f"/api/v1/agent-types/slug/{slug}", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert get_resp.status_code == 200 data = get_resp.json() assert data["slug"] == slug async def test_update_agent_type(self, e2e_client, e2e_superuser): """Test updating an agent type.""" slug = f"update-test-{uuid4().hex[:8]}" # Create agent type create_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Original Name", "slug": slug, "personality_prompt": "Original prompt.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = create_resp.json()["id"] # Update agent type update_resp = await e2e_client.patch( f"/api/v1/agent-types/{agent_type_id}", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Updated Name", "description": "Added description", }, ) assert update_resp.status_code == 200 updated = update_resp.json() assert updated["name"] == "Updated Name" assert updated["description"] == "Added description" assert updated["personality_prompt"] == "Original prompt." # Unchanged class TestAgentInstanceWorkflows: """Test agent instance spawning and lifecycle.""" async def test_spawn_agent_workflow(self, e2e_client, e2e_superuser): """Test complete workflow: create type -> create project -> spawn agent.""" # 1. Create agent type as superuser type_slug = f"spawn-test-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Spawn Test Agent", "slug": type_slug, "personality_prompt": "You are a helpful agent.", "primary_model": "claude-3-sonnet", }, ) assert type_resp.status_code == 201 agent_type = type_resp.json() agent_type_id = agent_type["id"] # 2. Create a project (superuser can create projects too) project_slug = f"spawn-test-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Spawn Test Project", "slug": project_slug}, ) assert project_resp.status_code == 201 project = project_resp.json() project_id = project["id"] # 3. Spawn agent instance spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "My PO Agent", }, ) assert spawn_resp.status_code == 201, f"Failed: {spawn_resp.text}" agent = spawn_resp.json() assert agent["name"] == "My PO Agent" assert agent["status"] == "idle" assert agent["project_id"] == project_id assert agent["agent_type_id"] == agent_type_id async def test_list_project_agents(self, e2e_client, e2e_superuser): """Test listing agents in a project.""" # Setup: Create agent type and project type_slug = f"list-agents-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "List Agents Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = type_resp.json()["id"] project_slug = f"list-agents-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "List Agents Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] # Spawn multiple agents for i in range(3): await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": f"Agent {i + 1}", }, ) # List agents list_resp = await e2e_client.get( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert list_resp.status_code == 200 data = list_resp.json() assert data["pagination"]["total"] == 3 assert len(data["data"]) == 3 class TestAgentLifecycle: """Test agent lifecycle operations (pause/resume/terminate).""" async def test_agent_pause_and_resume(self, e2e_client, e2e_superuser): """Test pausing and resuming an agent.""" # Setup: Create agent type, project, and agent type_slug = f"pause-test-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Pause Test Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = type_resp.json()["id"] project_slug = f"pause-test-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Pause Test Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "Pausable Agent", }, ) agent_id = spawn_resp.json()["id"] assert spawn_resp.json()["status"] == "idle" # Pause agent pause_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents/{agent_id}/pause", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert pause_resp.status_code == 200, f"Failed: {pause_resp.text}" assert pause_resp.json()["status"] == "paused" # Resume agent resume_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents/{agent_id}/resume", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert resume_resp.status_code == 200, f"Failed: {resume_resp.text}" assert resume_resp.json()["status"] == "idle" async def test_agent_terminate(self, e2e_client, e2e_superuser): """Test terminating an agent.""" # Setup type_slug = f"terminate-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Terminate Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = type_resp.json()["id"] project_slug = f"terminate-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Terminate Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "To Be Terminated", }, ) agent_id = spawn_resp.json()["id"] # Terminate agent (returns MessageResponse, not agent status) terminate_resp = await e2e_client.delete( f"/api/v1/projects/{project_id}/agents/{agent_id}", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert terminate_resp.status_code == 200 assert "message" in terminate_resp.json() # Verify terminated agent cannot be resumed (returns 400 or 422) resume_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents/{agent_id}/resume", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert resume_resp.status_code in [400, 422] # Invalid transition class TestAgentAccessControl: """Test agent access control and authorization.""" async def test_user_cannot_access_other_project_agents( self, e2e_client, e2e_superuser ): """Test that users cannot access agents in projects they don't own.""" # Superuser creates agent type type_slug = f"access-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Access Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = type_resp.json()["id"] # Superuser creates project and spawns agent project_slug = f"protected-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Protected Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "Protected Agent", }, ) agent_id = spawn_resp.json()["id"] # Create a different user email = f"other-user-{uuid4().hex[:8]}@example.com" password = "OtherPass123!" await e2e_client.post( "/api/v1/auth/register", json={ "email": email, "password": password, "first_name": "Other", "last_name": "User", }, ) login_resp = await e2e_client.post( "/api/v1/auth/login", json={"email": email, "password": password}, ) other_tokens = login_resp.json() # Other user tries to access the agent get_resp = await e2e_client.get( f"/api/v1/projects/{project_id}/agents/{agent_id}", headers={"Authorization": f"Bearer {other_tokens['access_token']}"}, ) # Should be forbidden or not found assert get_resp.status_code in [403, 404] async def test_cannot_spawn_with_inactive_agent_type( self, e2e_client, e2e_superuser ): """Test that agents cannot be spawned from inactive agent types.""" # Create agent type type_slug = f"inactive-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Inactive Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", "is_active": True, }, ) agent_type_id = type_resp.json()["id"] # Deactivate the agent type await e2e_client.patch( f"/api/v1/agent-types/{agent_type_id}", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"is_active": False}, ) # Create project project_slug = f"inactive-spawn-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Inactive Spawn Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] # Try to spawn agent with inactive type spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "Should Fail", }, ) # 422 is correct for validation errors per REST conventions assert spawn_resp.status_code == 422 class TestAgentMetrics: """Test agent metrics endpoint.""" async def test_get_agent_metrics(self, e2e_client, e2e_superuser): """Test retrieving agent metrics.""" # Setup type_slug = f"metrics-type-{uuid4().hex[:8]}" type_resp = await e2e_client.post( "/api/v1/agent-types", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "name": "Metrics Type", "slug": type_slug, "personality_prompt": "Test agent.", "primary_model": "claude-3-sonnet", }, ) agent_type_id = type_resp.json()["id"] project_slug = f"metrics-project-{uuid4().hex[:8]}" project_resp = await e2e_client.post( "/api/v1/projects", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={"name": "Metrics Project", "slug": project_slug}, ) project_id = project_resp.json()["id"] spawn_resp = await e2e_client.post( f"/api/v1/projects/{project_id}/agents", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, json={ "agent_type_id": agent_type_id, "project_id": project_id, "name": "Metrics Agent", }, ) agent_id = spawn_resp.json()["id"] # Get metrics metrics_resp = await e2e_client.get( f"/api/v1/projects/{project_id}/agents/{agent_id}/metrics", headers={ "Authorization": f"Bearer {e2e_superuser['tokens']['access_token']}" }, ) assert metrics_resp.status_code == 200 metrics = metrics_resp.json() # Verify AgentInstanceMetrics structure assert "total_instances" in metrics assert "active_instances" in metrics assert "idle_instances" in metrics assert "total_tasks_completed" in metrics assert "total_tokens_used" in metrics assert "total_cost_incurred" in metrics