forked from cardosofelipe/fast-next-template
Felipe Cardoso
11da0d57a8
- Add Celery app configuration with Redis broker/backend - Add task modules: agent, workflow, cost, git, sync - Add task stubs for: - Agent execution (spawn, heartbeat, terminate) - Workflow orchestration (start sprint, checkpoint, code review) - Cost tracking (record usage, calculate, generate report) - Git operations (clone, commit, push, sync) - External sync (import issues, export updates) - Add task tests directory structure - Configure for production-ready Celery setup Implements #18 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
111 lines
3.5 KiB
Python
111 lines
3.5 KiB
Python
# app/celery_app.py
|
|
"""
|
|
Celery application configuration for Syndarix.
|
|
|
|
This module configures the Celery app for background task processing:
|
|
- Agent execution tasks (LLM calls, tool execution)
|
|
- Git operations (clone, commit, push, PR creation)
|
|
- Issue synchronization with external trackers
|
|
- Workflow state management
|
|
- Cost tracking and budget monitoring
|
|
|
|
Architecture:
|
|
- Redis as message broker and result backend
|
|
- Queue routing for task isolation
|
|
- JSON serialization for cross-language compatibility
|
|
- Beat scheduler for periodic tasks
|
|
"""
|
|
|
|
from celery import Celery
|
|
|
|
from app.core.config import settings
|
|
|
|
# Create Celery application instance
|
|
celery_app = Celery(
|
|
"syndarix",
|
|
broker=settings.celery_broker_url,
|
|
backend=settings.celery_result_backend,
|
|
)
|
|
|
|
# Define task queues with their own exchanges and routing keys
|
|
TASK_QUEUES = {
|
|
"agent": {"exchange": "agent", "routing_key": "agent"},
|
|
"git": {"exchange": "git", "routing_key": "git"},
|
|
"sync": {"exchange": "sync", "routing_key": "sync"},
|
|
"default": {"exchange": "default", "routing_key": "default"},
|
|
}
|
|
|
|
# Configure Celery
|
|
celery_app.conf.update(
|
|
# Serialization
|
|
task_serializer="json",
|
|
accept_content=["json"],
|
|
result_serializer="json",
|
|
# Timezone
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
# Task imports for auto-discovery
|
|
imports=("app.tasks",),
|
|
# Default queue
|
|
task_default_queue="default",
|
|
# Task queues configuration
|
|
task_queues=TASK_QUEUES,
|
|
# Task routing - route tasks to appropriate queues
|
|
task_routes={
|
|
"app.tasks.agent.*": {"queue": "agent"},
|
|
"app.tasks.git.*": {"queue": "git"},
|
|
"app.tasks.sync.*": {"queue": "sync"},
|
|
"app.tasks.*": {"queue": "default"},
|
|
},
|
|
# Time limits per ADR-003
|
|
task_soft_time_limit=300, # 5 minutes soft limit
|
|
task_time_limit=600, # 10 minutes hard limit
|
|
# Result expiration - 24 hours
|
|
result_expires=86400,
|
|
# Broker connection retry
|
|
broker_connection_retry_on_startup=True,
|
|
# Beat schedule for periodic tasks
|
|
beat_schedule={
|
|
# Cost aggregation every hour per ADR-012
|
|
"aggregate-daily-costs": {
|
|
"task": "app.tasks.cost.aggregate_daily_costs",
|
|
"schedule": 3600.0, # 1 hour in seconds
|
|
},
|
|
# Reset daily budget counters at midnight UTC
|
|
"reset-daily-budget-counters": {
|
|
"task": "app.tasks.cost.reset_daily_budget_counters",
|
|
"schedule": 86400.0, # 24 hours in seconds
|
|
},
|
|
# Check for stale workflows every 5 minutes
|
|
"recover-stale-workflows": {
|
|
"task": "app.tasks.workflow.recover_stale_workflows",
|
|
"schedule": 300.0, # 5 minutes in seconds
|
|
},
|
|
# Incremental issue sync every minute per ADR-011
|
|
"sync-issues-incremental": {
|
|
"task": "app.tasks.sync.sync_issues_incremental",
|
|
"schedule": 60.0, # 1 minute in seconds
|
|
},
|
|
# Full issue reconciliation every 15 minutes per ADR-011
|
|
"sync-issues-full": {
|
|
"task": "app.tasks.sync.sync_issues_full",
|
|
"schedule": 900.0, # 15 minutes in seconds
|
|
},
|
|
},
|
|
# Task execution settings
|
|
task_acks_late=True, # Acknowledge tasks after execution
|
|
task_reject_on_worker_lost=True, # Reject tasks if worker dies
|
|
worker_prefetch_multiplier=1, # Fair task distribution
|
|
)
|
|
|
|
# Auto-discover tasks from task modules
|
|
celery_app.autodiscover_tasks(
|
|
[
|
|
"app.tasks.agent",
|
|
"app.tasks.git",
|
|
"app.tasks.sync",
|
|
"app.tasks.workflow",
|
|
"app.tasks.cost",
|
|
]
|
|
)
|