initial commit after one day coding agent session

tests/execution/simple/test_backend.py

@@ -0,0 +1,395 @@
+"""Tests for the Simple Backend module."""
+
+import pytest
+from unittest.mock import Mock, MagicMock, patch
+from pathlib import Path
+
+from mcp_forge.execution.simple.backend import SimpleBackend
+from mcp_forge.execution.simple.executor import ExecutionResult
+from mcp_forge.config.schema import ForgeConfig, ExecutionConfig, ImageConfig
+from mcp_forge.podman.containers import SecureContainerManager
+from mcp_forge.security.audit import AuditLogger
+from mcp_forge.security.resource_limits import ResourceLimits
+
+
+@pytest.fixture
+def mock_config():
+    """Mock ForgeConfig with execution settings."""
+    config = Mock(spec=ForgeConfig)
+    
+    # Execution configuration
+    config.execution = Mock(spec=ExecutionConfig)
+    config.execution.default_timeout = 300
+    config.execution.max_timeout = 1800
+    config.execution.default_memory = "512m"
+    config.execution.max_memory = "2g"
+    config.execution.default_cpu_quota = 50000
+    config.execution.max_cpu_quota = 100000
+    
+    # Image configuration
+    config.images = Mock(spec=ImageConfig)
+    config.images.python_3_11 = "mcp-forge/python:3.11"
+    config.images.python_3_12 = "mcp-forge/python:3.12"
+    
+    return config
+
+
+@pytest.fixture
+def mock_container_manager():
+    """Mock SecureContainerManager."""
+    return Mock(spec=SecureContainerManager)
+
+
+@pytest.fixture
+def mock_audit_logger(tmp_path):
+    """Mock AuditLogger."""
+    log_file = tmp_path / "audit.log"
+    return Mock(spec=AuditLogger)
+
+
+@pytest.fixture
+def backend(mock_config, mock_container_manager, mock_audit_logger):
+    """SimpleBackend instance with mocked dependencies."""
+    return SimpleBackend(
+        config=mock_config,
+        container_manager=mock_container_manager,
+        audit_logger=mock_audit_logger
+    )
+
+
+def test_execute_without_custom_params_uses_defaults(backend, mock_config):
+    """Test execute uses configuration defaults when no params specified."""
+    # Mock executor to return a result
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=42,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        result = backend.execute("2 + 2")
+        
+        # Verify executor was created with default limits
+        mock_executor_class.assert_called_once()
+        args, kwargs = mock_executor_class.call_args
+        
+        # Check resource limits
+        resource_limits = kwargs.get('resource_limits')
+        assert resource_limits is not None
+        assert resource_limits.memory_bytes == 512 * 1024 * 1024  # 512m in bytes
+        assert resource_limits.cpu_quota == 50000
+        assert resource_limits.timeout == 300
+        
+        # Check image
+        assert kwargs.get('image') == "mcp-forge/python:3.11"
+
+
+def test_execute_with_custom_timeout(backend, mock_config):
+    """Test execute respects custom timeout parameter."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        backend.execute("pass", timeout=600)
+        
+        # Verify resource limits include custom timeout
+        args, kwargs = mock_executor_class.call_args
+        resource_limits = kwargs.get('resource_limits')
+        assert resource_limits.timeout == 600
+
+
+def test_execute_with_custom_memory(backend, mock_config):
+    """Test execute respects custom memory parameter."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        backend.execute("pass", memory="1g")
+        
+        # Verify resource limits include custom memory
+        args, kwargs = mock_executor_class.call_args
+        resource_limits = kwargs.get('resource_limits')
+        assert resource_limits.memory_bytes == 1024 * 1024 * 1024  # 1g in bytes
+
+
+def test_execute_with_custom_cpu_quota(backend, mock_config):
+    """Test execute respects custom CPU quota parameter."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        backend.execute("pass", cpu_quota=75000)
+        
+        # Verify resource limits include custom CPU quota
+        args, kwargs = mock_executor_class.call_args
+        resource_limits = kwargs.get('resource_limits')
+        assert resource_limits.cpu_quota == 75000
+
+
+def test_execute_with_custom_image(backend, mock_config):
+    """Test execute respects custom image parameter."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        backend.execute("pass", custom_image="mcp-forge/python:3.12")
+        
+        # Verify correct image was used
+        args, kwargs = mock_executor_class.call_args
+        assert kwargs.get('image') == "mcp-forge/python:3.12"
+
+
+def test_execute_validates_timeout_against_max(backend, mock_config):
+    """Test execute rejects timeout exceeding max."""
+    with pytest.raises(ValueError, match="timeout.*exceeds maximum"):
+        backend.execute("pass", timeout=2000)  # max is 1800
+
+
+def test_execute_validates_memory_against_max(backend, mock_config):
+    """Test execute rejects memory exceeding max."""
+    with pytest.raises(ValueError, match="memory.*exceeds maximum"):
+        backend.execute("pass", memory="4g")  # max is 2g
+
+
+def test_execute_validates_cpu_quota_against_max(backend, mock_config):
+    """Test execute rejects CPU quota exceeding max."""
+    with pytest.raises(ValueError, match="cpu_quota.*exceeds maximum"):
+        backend.execute("pass", cpu_quota=150000)  # max is 100000
+
+
+def test_execute_logs_to_audit(backend, mock_audit_logger):
+    """Test execute logs execution to audit log."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=42,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        backend.execute("x = 2 + 2")
+        
+        # Verify audit log was called
+        mock_audit_logger.log.assert_called()
+        call_args = mock_audit_logger.log.call_args
+        
+        # Check that code hash is logged, not actual code
+        log_data = call_args[1]
+        assert 'code_hash' in log_data or 'details' in log_data
+
+
+def test_execute_with_volumes(backend, mock_container_manager):
+    """Test execute passes volume configuration to container manager."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        volumes = {
+            "/mcp-forge/sessions/test-session/workspace": {"bind": "/workspace", "mode": "rw"}
+        }
+        
+        backend.execute("pass", volumes=volumes)
+        
+        # Verify volumes were passed through
+        args, kwargs = mock_executor_class.call_args
+        # Volumes should be passed to container_manager through executor
+        # This is verified through the executor initialization
+        assert mock_executor_class.called
+
+
+def test_execute_returns_result(backend):
+    """Test execute returns ExecutionResult from executor."""
+    expected_result = ExecutionResult(
+        success=True,
+        stdout="Hello\n",
+        stderr="",
+        result=42,
+        execution_time=0.5,
+        exit_code=0
+    )
+    
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = expected_result
+        
+        result = backend.execute('print("Hello"); 42')
+        
+        assert result == expected_result
+        assert result.success is True
+        assert result.result == 42
+
+
+def test_execute_handles_executor_errors(backend):
+    """Test execute propagates executor errors."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.side_effect = RuntimeError("Container failed")
+        
+        with pytest.raises(RuntimeError, match="Container failed"):
+            backend.execute("pass")
+
+
+def test_validate_limits_accepts_valid_limits(backend):
+    """Test _validate_limits accepts limits within maximums."""
+    # Should not raise
+    backend._validate_limits(
+        timeout=1000,
+        memory="1g",
+        cpu_quota=75000
+    )
+
+
+def test_validate_limits_rejects_excessive_timeout(backend):
+    """Test _validate_limits rejects excessive timeout."""
+    with pytest.raises(ValueError, match="timeout"):
+        backend._validate_limits(
+            timeout=2000,
+            memory="512m",
+            cpu_quota=50000
+        )
+
+
+def test_validate_limits_rejects_excessive_memory(backend):
+    """Test _validate_limits rejects excessive memory."""
+    with pytest.raises(ValueError, match="memory"):
+        backend._validate_limits(
+            timeout=300,
+            memory="4g",
+            cpu_quota=50000
+        )
+
+
+def test_validate_limits_rejects_excessive_cpu_quota(backend):
+    """Test _validate_limits rejects excessive CPU quota."""
+    with pytest.raises(ValueError, match="cpu_quota"):
+        backend._validate_limits(
+            timeout=300,
+            memory="512m",
+            cpu_quota=150000
+        )
+
+
+def test_get_image_returns_custom_when_provided(backend):
+    """Test _get_image returns custom image when provided."""
+    image = backend._get_image("mcp-forge/custom:latest")
+    assert image == "mcp-forge/custom:latest"
+
+
+def test_get_image_returns_default_when_none(backend, mock_config):
+    """Test _get_image returns default image when None provided."""
+    image = backend._get_image(None)
+    assert image == mock_config.images.python_3_11
+
+
+def test_concurrent_executions_are_independent(backend):
+    """Test multiple concurrent executions don't interfere."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        # Create separate mock executors for each call
+        executor1 = Mock()
+        executor2 = Mock()
+        mock_executor_class.side_effect = [executor1, executor2]
+        
+        executor1.execute.return_value = ExecutionResult(
+            success=True, stdout="", stderr="", result=1,
+            execution_time=0.1, exit_code=0
+        )
+        executor2.execute.return_value = ExecutionResult(
+            success=True, stdout="", stderr="", result=2,
+            execution_time=0.1, exit_code=0
+        )
+        
+        result1 = backend.execute("1")
+        result2 = backend.execute("2")
+        
+        assert result1.result == 1
+        assert result2.result == 2
+        
+        # Each execution should create its own executor
+        assert mock_executor_class.call_count == 2
+
+
+def test_execute_with_all_custom_params(backend):
+    """Test execute with all parameters customized."""
+    with patch('mcp_forge.execution.simple.backend.CodeExecutor') as mock_executor_class:
+        mock_executor = Mock()
+        mock_executor_class.return_value = mock_executor
+        mock_executor.execute.return_value = ExecutionResult(
+            success=True,
+            stdout="",
+            stderr="",
+            result=None,
+            execution_time=0.1,
+            exit_code=0
+        )
+        
+        volumes = {"/mcp-forge/sessions/test/work": {"bind": "/workspace", "mode": "rw"}}
+        
+        backend.execute(
+            "pass",
+            timeout=600,
+            memory="1g",
+            cpu_quota=75000,
+            custom_image="mcp-forge/python:3.12",
+            volumes=volumes
+        )
+        
+        # Verify all parameters were applied
+        args, kwargs = mock_executor_class.call_args
+        
+        resource_limits = kwargs.get('resource_limits')
+        assert resource_limits.timeout == 600
+        assert resource_limits.memory_bytes == 1024 * 1024 * 1024  # 1g in bytes
+        assert resource_limits.cpu_quota == 75000
+        
+        assert kwargs.get('image') == "mcp-forge/python:3.12"

tests/execution/simple/test_executor.py

@@ -0,0 +1,299 @@
+"""Tests for the Code Executor module."""
+
+import pytest
+from unittest.mock import Mock
+import json
+
+from mcp_forge.execution.simple.executor import CodeExecutor, ExecutionResult
+from mcp_forge.podman.containers import SecureContainerManager
+from mcp_forge.security.resource_limits import ResourceLimits
+
+
+@pytest.fixture
+def resource_limits():
+    """Standard resource limits for testing."""
+    return ResourceLimits(
+        memory="512m",
+        cpu_quota=50000,
+        storage="1g",
+        timeout=30
+    )
+
+
+@pytest.fixture
+def mock_container_manager():
+    """Mock SecureContainerManager."""
+    manager = Mock(spec=SecureContainerManager)
+    
+    # Mock container lifecycle
+    manager.create_container.return_value = "test-container-123"
+    manager.start_container.return_value = None
+    manager.stop_container.return_value = None
+    manager.remove_container.return_value = None
+    manager.wait_for_container.return_value = 0  # exit code
+    manager.get_container_logs.return_value = ("", "")  # (stdout, stderr)
+    
+    return manager
+
+
+@pytest.fixture
+def executor(mock_container_manager, resource_limits):
+    """CodeExecutor instance with mocked dependencies."""
+    return CodeExecutor(
+        container_manager=mock_container_manager,
+        image="mcp-forge/python:3.11",
+        resource_limits=resource_limits
+    )
+
+
+def test_execute_simple_python_code_returns_result(executor, mock_container_manager):
+    """Test executing simple Python code returns the result."""
+    # Mock successful execution with result
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": 42, "error": None}),
+        ""
+    )
+    
+    result = executor.execute("2 + 2")
+    
+    assert result.success is True
+    assert result.result == 42
+    assert result.exit_code == 0
+    assert result.error is None
+    
+    # Verify container lifecycle
+    mock_container_manager.create_container.assert_called_once()
+    mock_container_manager.start_container.assert_called_once_with("test-container-123")
+    mock_container_manager.wait_for_container.assert_called_once_with("test-container-123", timeout=30)
+    mock_container_manager.remove_container.assert_called_once_with("test-container-123")
+
+
+def test_execute_code_with_stdout_capture(executor, mock_container_manager):
+    """Test code execution captures stdout."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}) + "\n" + "Hello, World!",
+        ""
+    )
+    
+    result = executor.execute('print("Hello, World!")')
+    
+    assert result.success is True
+    assert "Hello, World!" in result.stdout
+    assert result.stderr == ""
+
+
+def test_execute_code_with_stderr_capture(executor, mock_container_manager):
+    """Test code execution captures stderr."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}),
+        "Warning: something happened"
+    )
+    
+    result = executor.execute('import sys; print("warning", file=sys.stderr)')
+    
+    assert result.success is True
+    assert result.stderr == "Warning: something happened"
+
+
+def test_execute_code_timeout_enforcement(executor, mock_container_manager):
+    """Test code execution enforces timeout."""
+    # Simulate timeout by having wait_for_container take too long
+    mock_container_manager.wait_for_container.side_effect = TimeoutError("Container exceeded timeout")
+    
+    result = executor.execute("import time; time.sleep(60)", timeout=1)
+    
+    assert result.success is False
+    assert result.error is not None
+    assert "timeout" in result.error.lower()
+    
+    # Verify cleanup still happens
+    mock_container_manager.remove_container.assert_called_once_with("test-container-123")
+
+
+def test_execute_code_with_exception_handling(executor, mock_container_manager):
+    """Test code execution handles exceptions gracefully."""
+    error_msg = "ZeroDivisionError: division by zero"
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": error_msg}),
+        ""
+    )
+    mock_container_manager.wait_for_container.return_value = 1  # non-zero exit
+    
+    result = executor.execute("1 / 0")
+    
+    assert result.success is False
+    assert result.error == error_msg
+    assert result.exit_code == 1
+
+
+def test_execute_code_with_syntax_error_returns_clear_error(executor, mock_container_manager):
+    """Test code with syntax error returns clear error message."""
+    error_msg = "SyntaxError: invalid syntax"
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": error_msg}),
+        ""
+    )
+    mock_container_manager.wait_for_container.return_value = 1
+    
+    result = executor.execute("def foo( :")
+    
+    assert result.success is False
+    assert "SyntaxError" in result.error
+
+
+def test_execute_code_with_runtime_error_returns_clear_error(executor, mock_container_manager):
+    """Test code with runtime error returns clear error with traceback."""
+    error_msg = "NameError: name 'undefined_var' is not defined"
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": error_msg}),
+        ""
+    )
+    mock_container_manager.wait_for_container.return_value = 1
+    
+    result = executor.execute("print(undefined_var)")
+    
+    assert result.success is False
+    assert "NameError" in result.error
+
+
+def test_result_serialization_json_compatible_types(executor, mock_container_manager):
+    """Test execution result contains only JSON-serializable data."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": [1, 2, {"key": "value"}], "error": None}),
+        ""
+    )
+    
+    result = executor.execute('[1, 2, {"key": "value"}]')
+    
+    # Verify result can be serialized to JSON
+    result_dict = result.to_dict()
+    json_str = json.dumps(result_dict)
+    assert json_str is not None
+    
+    # Verify result data
+    assert result.result == [1, 2, {"key": "value"}]
+
+
+def test_large_output_handling(executor, mock_container_manager):
+    """Test execution handles large output without issues."""
+    large_output = "x" * 10000  # 10KB of output
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}) + "\n" + large_output,
+        ""
+    )
+    
+    result = executor.execute('print("x" * 10000)')
+    
+    assert result.success is True
+    assert len(result.stdout) >= 10000
+
+
+def test_execution_result_to_dict(resource_limits):
+    """Test ExecutionResult.to_dict() returns proper dictionary."""
+    result = ExecutionResult(
+        success=True,
+        stdout="output",
+        stderr="",
+        result=42,
+        execution_time=0.5,
+        exit_code=0,
+        error=None
+    )
+    
+    result_dict = result.to_dict()
+    
+    assert isinstance(result_dict, dict)
+    assert result_dict["success"] is True
+    assert result_dict["stdout"] == "output"
+    assert result_dict["stderr"] == ""
+    assert result_dict["result"] == 42
+    assert result_dict["execution_time"] == 0.5
+    assert result_dict["exit_code"] == 0
+    assert result_dict["error"] is None
+
+
+def test_prepare_code_wraps_code_properly(executor):
+    """Test _prepare_code wraps code to capture result."""
+    code = "x = 2 + 2\nx"
+    wrapped = executor._prepare_code(code)
+    
+    # Wrapped code should be executable Python
+    assert "import" in wrapped
+    assert "json" in wrapped
+    assert code in wrapped or "2 + 2" in wrapped
+
+
+def test_parse_output_extracts_result_and_error(executor):
+    """Test _parse_output correctly extracts result and error from JSON."""
+    # Test successful result
+    stdout = json.dumps({"result": 42, "error": None})
+    result, error = executor._parse_output(stdout)
+    assert result == 42
+    assert error is None
+    
+    # Test error
+    stdout = json.dumps({"result": None, "error": "ValueError: invalid"})
+    result, error = executor._parse_output(stdout)
+    assert result is None
+    assert error == "ValueError: invalid"
+
+
+def test_cleanup_happens_even_on_create_failure(executor, mock_container_manager):
+    """Test container cleanup happens even if create fails."""
+    mock_container_manager.create_container.side_effect = Exception("Create failed")
+    
+    with pytest.raises(Exception, match="Create failed"):
+        executor.execute("print('test')")
+    
+    # No container to remove since create failed
+    mock_container_manager.remove_container.assert_not_called()
+
+
+def test_cleanup_happens_even_on_start_failure(executor, mock_container_manager):
+    """Test container cleanup happens even if start fails."""
+    mock_container_manager.start_container.side_effect = Exception("Start failed")
+    
+    with pytest.raises(Exception, match="Start failed"):
+        executor.execute("print('test')")
+    
+    # Container should still be removed
+    mock_container_manager.remove_container.assert_called_once_with("test-container-123")
+
+
+def test_execution_time_tracking(executor, mock_container_manager):
+    """Test execution time is tracked accurately."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}),
+        ""
+    )
+    
+    result = executor.execute("pass")
+    
+    assert result.execution_time >= 0
+    assert isinstance(result.execution_time, float)
+
+
+def test_execute_with_custom_timeout(executor, mock_container_manager):
+    """Test execute respects custom timeout parameter."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}),
+        ""
+    )
+    
+    executor.execute("pass", timeout=60)
+    
+    # Verify wait was called with custom timeout
+    mock_container_manager.wait_for_container.assert_called_with("test-container-123", timeout=60)
+
+
+def test_execute_uses_default_timeout_from_resource_limits(executor, mock_container_manager):
+    """Test execute uses default timeout from resource limits when not specified."""
+    mock_container_manager.get_container_logs.return_value = (
+        json.dumps({"result": None, "error": None}),
+        ""
+    )
+    
+    executor.execute("pass")  # No timeout specified
+    
+    # Should use resource_limits.timeout (30)
+    mock_container_manager.wait_for_container.assert_called_with("test-container-123", timeout=30)