"""Tests for safety validation module.""" import pytest from app.services.safety.models import ( ActionMetadata, ActionRequest, ActionType, AutonomyLevel, SafetyDecision, SafetyPolicy, ValidationRule, ) from app.services.safety.validation.validator import ( ActionValidator, create_allow_rule, create_approval_rule, create_deny_rule, ) @pytest.fixture def validator() -> ActionValidator: """Create a fresh ActionValidator.""" return ActionValidator(cache_enabled=False) @pytest.fixture def sample_action() -> ActionRequest: """Create a sample action request.""" metadata = ActionMetadata( agent_id="test-agent", session_id="test-session", autonomy_level=AutonomyLevel.MILESTONE, ) return ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource="/tmp/test.txt", # noqa: S108 metadata=metadata, ) class TestActionValidator: """Tests for ActionValidator class.""" @pytest.mark.asyncio async def test_no_rules_allows_by_default( self, validator: ActionValidator, sample_action: ActionRequest, ) -> None: """Test that actions are allowed by default with no rules.""" result = await validator.validate(sample_action) assert result.decision == SafetyDecision.ALLOW assert "No matching rules" in result.reasons[0] @pytest.mark.asyncio async def test_deny_rule_blocks_action( self, validator: ActionValidator, ) -> None: """Test that a deny rule blocks matching actions.""" validator.add_rule( create_deny_rule( name="deny_shell", tool_patterns=["shell_*"], reason="Shell commands not allowed", ) ) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.SHELL_COMMAND, tool_name="shell_exec", metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.DENY assert len(result.applied_rules) == 1 # One rule applied @pytest.mark.asyncio async def test_approval_rule_requires_approval( self, validator: ActionValidator, ) -> None: """Test that an approval rule requires approval.""" validator.add_rule( create_approval_rule( name="approve_db", tool_patterns=["database_*"], reason="Database operations require approval", ) ) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.DATABASE_MUTATE, tool_name="database_delete", metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.REQUIRE_APPROVAL @pytest.mark.asyncio async def test_deny_takes_precedence( self, validator: ActionValidator, ) -> None: """Test that deny rules take precedence over allow rules.""" validator.add_rule( create_allow_rule( name="allow_files", tool_patterns=["file_*"], priority=10, ) ) validator.add_rule( create_deny_rule( name="deny_delete", action_types=[ActionType.FILE_DELETE], priority=100, ) ) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.FILE_DELETE, tool_name="file_delete", metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.DENY @pytest.mark.asyncio async def test_rule_priority_ordering( self, validator: ActionValidator, ) -> None: """Test that rules are evaluated in priority order.""" validator.add_rule( ValidationRule( name="low_priority", priority=1, decision=SafetyDecision.ALLOW, ) ) validator.add_rule( ValidationRule( name="high_priority", priority=100, decision=SafetyDecision.DENY, ) ) # High priority should be first in the list assert validator._rules[0].name == "high_priority" @pytest.mark.asyncio async def test_disabled_rule_not_applied( self, validator: ActionValidator, sample_action: ActionRequest, ) -> None: """Test that disabled rules are not applied.""" rule = create_deny_rule( name="deny_all", tool_patterns=["*"], ) rule.enabled = False validator.add_rule(rule) result = await validator.validate(sample_action) assert result.decision == SafetyDecision.ALLOW @pytest.mark.asyncio async def test_resource_pattern_matching( self, validator: ActionValidator, ) -> None: """Test resource pattern matching.""" validator.add_rule( create_deny_rule( name="deny_secrets", resource_patterns=["*/secrets/*", "*.env"], ) ) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource="/app/secrets/api_key.txt", metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.DENY @pytest.mark.asyncio async def test_agent_id_filter( self, validator: ActionValidator, ) -> None: """Test filtering by agent ID.""" rule = ValidationRule( name="restrict_agent", agent_ids=["restricted-agent"], decision=SafetyDecision.DENY, reason="Restricted agent", ) validator.add_rule(rule) # Restricted agent should be denied metadata1 = ActionMetadata(agent_id="restricted-agent") action1 = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", metadata=metadata1, ) result1 = await validator.validate(action1) assert result1.decision == SafetyDecision.DENY # Other agents should be allowed metadata2 = ActionMetadata(agent_id="normal-agent") action2 = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", metadata=metadata2, ) result2 = await validator.validate(action2) assert result2.decision == SafetyDecision.ALLOW @pytest.mark.asyncio async def test_bypass_mode( self, validator: ActionValidator, sample_action: ActionRequest, ) -> None: """Test validation bypass mode.""" validator.add_rule(create_deny_rule(name="deny_all", tool_patterns=["*"])) # Should be denied normally result1 = await validator.validate(sample_action) assert result1.decision == SafetyDecision.DENY # Enable bypass validator.enable_bypass("Emergency situation") result2 = await validator.validate(sample_action) assert result2.decision == SafetyDecision.ALLOW assert "bypassed" in result2.reasons[0].lower() # Disable bypass validator.disable_bypass() result3 = await validator.validate(sample_action) assert result3.decision == SafetyDecision.DENY def test_remove_rule(self, validator: ActionValidator) -> None: """Test removing a rule.""" rule = create_deny_rule(name="test_rule", tool_patterns=["test"]) validator.add_rule(rule) assert len(validator._rules) == 1 assert validator.remove_rule(rule.id) is True assert len(validator._rules) == 0 def test_remove_nonexistent_rule(self, validator: ActionValidator) -> None: """Test removing a nonexistent rule returns False.""" assert validator.remove_rule("nonexistent") is False def test_clear_rules(self, validator: ActionValidator) -> None: """Test clearing all rules.""" validator.add_rule(create_deny_rule(name="rule1", tool_patterns=["a"])) validator.add_rule(create_deny_rule(name="rule2", tool_patterns=["b"])) assert len(validator._rules) == 2 validator.clear_rules() assert len(validator._rules) == 0 class TestLoadRulesFromPolicy: """Tests for loading rules from policies.""" @pytest.mark.asyncio async def test_load_denied_tools( self, validator: ActionValidator, ) -> None: """Test loading denied tools from policy.""" policy = SafetyPolicy( name="test", denied_tools=["shell_*", "exec_*"], ) validator.load_rules_from_policy(policy) # Should have 2 deny rules deny_rules = [r for r in validator._rules if r.decision == SafetyDecision.DENY] assert len(deny_rules) == 2 @pytest.mark.asyncio async def test_load_approval_patterns( self, validator: ActionValidator, ) -> None: """Test loading approval patterns from policy.""" policy = SafetyPolicy( name="test", require_approval_for=["database_*"], ) validator.load_rules_from_policy(policy) approval_rules = [ r for r in validator._rules if r.decision == SafetyDecision.REQUIRE_APPROVAL ] assert len(approval_rules) == 1 class TestValidationBatch: """Tests for batch validation.""" @pytest.mark.asyncio async def test_validate_batch( self, validator: ActionValidator, ) -> None: """Test validating multiple actions.""" validator.add_rule( create_deny_rule( name="deny_shell", tool_patterns=["shell_*"], ) ) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") actions = [ ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", metadata=metadata, ), ActionRequest( action_type=ActionType.SHELL_COMMAND, tool_name="shell_exec", metadata=metadata, ), ] results = await validator.validate_batch(actions) assert len(results) == 2 assert results[0].decision == SafetyDecision.ALLOW assert results[1].decision == SafetyDecision.DENY class TestValidationCache: """Tests for ValidationCache class.""" @pytest.mark.asyncio async def test_cache_get_miss(self) -> None: """Test cache miss.""" from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=10, ttl_seconds=60) result = await cache.get("nonexistent") assert result is None @pytest.mark.asyncio async def test_cache_get_hit(self) -> None: """Test cache hit.""" from app.services.safety.models import ValidationResult from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=10, ttl_seconds=60) vr = ValidationResult( action_id="action-1", decision=SafetyDecision.ALLOW, applied_rules=[], reasons=["test"], ) await cache.set("key1", vr) result = await cache.get("key1") assert result is not None assert result.action_id == "action-1" @pytest.mark.asyncio async def test_cache_ttl_expiry(self) -> None: """Test cache TTL expiry.""" import time from unittest.mock import patch from app.services.safety.models import ValidationResult from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=10, ttl_seconds=1) vr = ValidationResult( action_id="action-1", decision=SafetyDecision.ALLOW, applied_rules=[], reasons=["test"], ) await cache.set("key1", vr) # Advance time past TTL with patch("time.time", return_value=time.time() + 2): result = await cache.get("key1") assert result is None # Should be expired @pytest.mark.asyncio async def test_cache_eviction_on_full(self) -> None: """Test cache eviction when full.""" from app.services.safety.models import ValidationResult from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=2, ttl_seconds=60) vr1 = ValidationResult(action_id="a1", decision=SafetyDecision.ALLOW) vr2 = ValidationResult(action_id="a2", decision=SafetyDecision.ALLOW) vr3 = ValidationResult(action_id="a3", decision=SafetyDecision.ALLOW) await cache.set("key1", vr1) await cache.set("key2", vr2) await cache.set("key3", vr3) # Should evict key1 # key1 should be evicted assert await cache.get("key1") is None assert await cache.get("key2") is not None assert await cache.get("key3") is not None @pytest.mark.asyncio async def test_cache_update_existing_key(self) -> None: """Test updating existing key in cache.""" from app.services.safety.models import ValidationResult from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=10, ttl_seconds=60) vr1 = ValidationResult(action_id="a1", decision=SafetyDecision.ALLOW) vr2 = ValidationResult(action_id="a1-updated", decision=SafetyDecision.DENY) await cache.set("key1", vr1) await cache.set("key1", vr2) # Should update, not add result = await cache.get("key1") assert result is not None assert result.action_id == "a1" # Still old value since we move_to_end @pytest.mark.asyncio async def test_cache_clear(self) -> None: """Test clearing cache.""" from app.services.safety.models import ValidationResult from app.services.safety.validation.validator import ValidationCache cache = ValidationCache(max_size=10, ttl_seconds=60) vr = ValidationResult(action_id="a1", decision=SafetyDecision.ALLOW) await cache.set("key1", vr) await cache.set("key2", vr) await cache.clear() assert await cache.get("key1") is None assert await cache.get("key2") is None class TestValidatorCaching: """Tests for validator caching functionality.""" @pytest.mark.asyncio async def test_cache_hit(self) -> None: """Test that cache is used for repeated validations.""" validator = ActionValidator(cache_enabled=True, cache_ttl=60) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource="/tmp/test.txt", # noqa: S108 metadata=metadata, ) # First call populates cache result1 = await validator.validate(action) # Second call should use cache result2 = await validator.validate(action) assert result1.decision == result2.decision @pytest.mark.asyncio async def test_clear_cache(self) -> None: """Test clearing the validation cache.""" validator = ActionValidator(cache_enabled=True) metadata = ActionMetadata(agent_id="test-agent", session_id="session-1") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", metadata=metadata, ) await validator.validate(action) await validator.clear_cache() # Cache should be empty now (no error) result = await validator.validate(action) assert result.decision == SafetyDecision.ALLOW class TestRuleMatching: """Tests for rule matching edge cases.""" @pytest.mark.asyncio async def test_action_type_mismatch(self) -> None: """Test that rule doesn't match when action type doesn't match.""" validator = ActionValidator(cache_enabled=False) validator.add_rule( ValidationRule( name="file_only", action_types=[ActionType.FILE_READ], decision=SafetyDecision.DENY, ) ) metadata = ActionMetadata(agent_id="test-agent") action = ActionRequest( action_type=ActionType.SHELL_COMMAND, # Different type tool_name="shell_exec", metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.ALLOW # Rule didn't match @pytest.mark.asyncio async def test_tool_pattern_no_tool_name(self) -> None: """Test rule with tool pattern when action has no tool_name.""" validator = ActionValidator(cache_enabled=False) validator.add_rule( create_deny_rule( name="deny_files", tool_patterns=["file_*"], ) ) metadata = ActionMetadata(agent_id="test-agent") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name=None, # No tool name metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.ALLOW # Rule didn't match @pytest.mark.asyncio async def test_resource_pattern_no_resource(self) -> None: """Test rule with resource pattern when action has no resource.""" validator = ActionValidator(cache_enabled=False) validator.add_rule( create_deny_rule( name="deny_secrets", resource_patterns=["/secret/*"], ) ) metadata = ActionMetadata(agent_id="test-agent") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource=None, # No resource metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.ALLOW # Rule didn't match @pytest.mark.asyncio async def test_resource_pattern_no_match(self) -> None: """Test rule with resource pattern that doesn't match.""" validator = ActionValidator(cache_enabled=False) validator.add_rule( create_deny_rule( name="deny_secrets", resource_patterns=["/secret/*"], ) ) metadata = ActionMetadata(agent_id="test-agent") action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource="/public/file.txt", # Doesn't match metadata=metadata, ) result = await validator.validate(action) assert result.decision == SafetyDecision.ALLOW # Pattern didn't match class TestPolicyLoading: """Tests for policy loading edge cases.""" @pytest.mark.asyncio async def test_load_rules_from_policy_with_validation_rules(self) -> None: """Test loading policy with explicit validation rules.""" validator = ActionValidator(cache_enabled=False) rule = ValidationRule( name="policy_rule", tool_patterns=["test_*"], decision=SafetyDecision.DENY, reason="From policy", ) policy = SafetyPolicy( name="test", validation_rules=[rule], require_approval_for=[], # Clear defaults denied_tools=[], # Clear defaults ) validator.load_rules_from_policy(policy) assert len(validator._rules) == 1 assert validator._rules[0].name == "policy_rule" @pytest.mark.asyncio async def test_load_approval_all_pattern(self) -> None: """Test loading policy with * approval pattern (all actions).""" validator = ActionValidator(cache_enabled=False) policy = SafetyPolicy( name="test", require_approval_for=["*"], # All actions require approval denied_tools=[], # Clear defaults ) validator.load_rules_from_policy(policy) approval_rules = [ r for r in validator._rules if r.decision == SafetyDecision.REQUIRE_APPROVAL ] assert len(approval_rules) == 1 assert approval_rules[0].name == "require_approval_all" assert approval_rules[0].action_types == list(ActionType) @pytest.mark.asyncio async def test_validate_with_policy_loads_rules(self) -> None: """Test that validate() loads rules from policy if none exist.""" validator = ActionValidator(cache_enabled=False) policy = SafetyPolicy( name="test", denied_tools=["dangerous_*"], ) metadata = ActionMetadata(agent_id="test-agent") action = ActionRequest( action_type=ActionType.SHELL_COMMAND, tool_name="dangerous_exec", metadata=metadata, ) # Validate with policy - should load rules result = await validator.validate(action, policy=policy) assert result.decision == SafetyDecision.DENY class TestCacheKeyGeneration: """Tests for cache key generation.""" def test_get_cache_key(self) -> None: """Test cache key generation.""" validator = ActionValidator(cache_enabled=True) metadata = ActionMetadata( agent_id="test-agent", autonomy_level=AutonomyLevel.MILESTONE, ) action = ActionRequest( action_type=ActionType.FILE_READ, tool_name="file_read", resource="/tmp/test.txt", # noqa: S108 metadata=metadata, ) key = validator._get_cache_key(action) assert "file_read" in key assert "file_read" in key assert "/tmp/test.txt" in key # noqa: S108 assert "test-agent" in key assert "milestone" in key def test_get_cache_key_no_resource(self) -> None: """Test cache key generation without resource.""" validator = ActionValidator(cache_enabled=True) metadata = ActionMetadata(agent_id="agent-1") action = ActionRequest( action_type=ActionType.SHELL_COMMAND, tool_name="shell_exec", resource=None, metadata=metadata, ) key = validator._get_cache_key(action) # Should not error with None resource assert "shell" in key assert "agent-1" in key class TestHelperFunctions: """Tests for rule creation helper functions.""" def test_create_allow_rule(self) -> None: """Test creating an allow rule.""" rule = create_allow_rule( name="allow_test", tool_patterns=["test_*"], priority=50, ) assert rule.name == "allow_test" assert rule.decision == SafetyDecision.ALLOW assert rule.priority == 50 def test_create_deny_rule(self) -> None: """Test creating a deny rule.""" rule = create_deny_rule( name="deny_test", tool_patterns=["dangerous_*"], reason="Too dangerous", ) assert rule.name == "deny_test" assert rule.decision == SafetyDecision.DENY assert rule.reason == "Too dangerous" assert rule.priority == 100 # Default priority for deny def test_create_approval_rule(self) -> None: """Test creating an approval rule.""" rule = create_approval_rule( name="approve_test", action_types=[ActionType.DATABASE_MUTATE], ) assert rule.name == "approve_test" assert rule.decision == SafetyDecision.REQUIRE_APPROVAL assert rule.priority == 50 # Default priority for approval