From 805ed647cdeb840dd9d22a89edd65708f6667d88 Mon Sep 17 00:00:00 2001 From: Felipe Cardoso Date: Wed, 22 Jan 2025 18:02:07 +0100 Subject: [PATCH] Add basic backend infrastructure Signed-off-by: Felipe Cardoso --- backend/Dockerfile | 0 backend/app/__init__.py | 0 backend/app/api/__init__.py | 0 backend/app/api/routes/__init__.py | 0 backend/app/api/routes/samples.py | 34 ++++++++++++++++++++++++ backend/app/api/routes/training.py | 29 ++++++++++++++++++++ backend/app/core/__init__.py | 0 backend/app/core/config.py | 21 +++++++++++++++ backend/app/main.py | 29 ++++++++++++++++++++ backend/app/models/__init__.py | 0 backend/app/models/sample.py | 11 ++++++++ backend/app/models/training.py | 14 ++++++++++ backend/app/services/__init__.py | 0 backend/app/services/sample_manager.py | 15 +++++++++++ backend/app/services/training_monitor.py | 13 +++++++++ backend/requirements.txt | 13 +++++++++ 16 files changed, 179 insertions(+) create mode 100644 backend/Dockerfile create mode 100644 backend/app/__init__.py create mode 100644 backend/app/api/__init__.py create mode 100644 backend/app/api/routes/__init__.py create mode 100644 backend/app/api/routes/samples.py create mode 100644 backend/app/api/routes/training.py create mode 100644 backend/app/core/__init__.py create mode 100644 backend/app/core/config.py create mode 100644 backend/app/main.py create mode 100644 backend/app/models/__init__.py create mode 100644 backend/app/models/sample.py create mode 100644 backend/app/models/training.py create mode 100644 backend/app/services/__init__.py create mode 100644 backend/app/services/sample_manager.py create mode 100644 backend/app/services/training_monitor.py create mode 100644 backend/requirements.txt diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/routes/__init__.py b/backend/app/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/routes/samples.py b/backend/app/api/routes/samples.py new file mode 100644 index 0000000..fa8ed92 --- /dev/null +++ b/backend/app/api/routes/samples.py @@ -0,0 +1,34 @@ +from typing import List + +from fastapi import APIRouter, HTTPException, Query + +from app.models.sample import Sample +from app.services.sample_manager import SampleManager + +router = APIRouter() +sample_manager = SampleManager() + + +@router.get("/list", response_model=List[Sample]) +async def list_samples( + limit: int = Query(20, ge=1, le=100), + offset: int = Query(0, ge=0) +): + """ + List sample images with pagination + """ + try: + return await sample_manager.list_samples(limit, offset) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/latest", response_model=List[Sample]) +async def get_latest_samples(count: int = Query(5, ge=1, le=20)): + """ + Get the most recent sample images + """ + try: + return await sample_manager.get_latest_samples(count) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/app/api/routes/training.py b/backend/app/api/routes/training.py new file mode 100644 index 0000000..bcf8b17 --- /dev/null +++ b/backend/app/api/routes/training.py @@ -0,0 +1,29 @@ +from fastapi import APIRouter, HTTPException + +from app.models.training import TrainingStatus +from app.services.training_monitor import TrainingMonitor + +router = APIRouter() +monitor = TrainingMonitor() + + +@router.get("/status", response_model=TrainingStatus) +async def get_training_status(): + """ + Get current training status including progress, loss, and learning rate + """ + try: + return await monitor.get_status() + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/log") +async def get_training_log(): + """ + Get recent training log entries + """ + try: + return await monitor.get_log() + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 0000000..e990256 --- /dev/null +++ b/backend/app/core/config.py @@ -0,0 +1,21 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + # SFTP Settings + SFTP_HOST: str + SFTP_USER: str + SFTP_PASSWORD: str + SFTP_PATH: str + SFTP_PORT: int = 22 + + # API Settings + API_VER_STR: str = "/api/v1" + PROJECT_NAME: str = "Training Monitor" + + class Config: + env_file = ".env" + case_sensitive = True + + +settings = Settings() diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..7d94259 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,29 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from app.api.routes import training, samples +from app.core.config import settings + +app = FastAPI( + title="Training Monitor API", + description="API for monitoring ML training progress and samples", + version="1.0.0", +) + +# Configure CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # In production, replace with specific origins + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Include routers with versioning +app.include_router(training.router, prefix=f"{settings.API_VER_STR}/training", tags=["training"]) +app.include_router(samples.router, prefix=f"{settings.API_VER_STR}/samples", tags=["samples"]) + + +@app.get("/health") +async def health_check(): + return {"status": "healthy"} diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/models/sample.py b/backend/app/models/sample.py new file mode 100644 index 0000000..164dd00 --- /dev/null +++ b/backend/app/models/sample.py @@ -0,0 +1,11 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + + +class Sample(BaseModel): + filename: str + url: str + created_at: datetime + step: Optional[int] = None diff --git a/backend/app/models/training.py b/backend/app/models/training.py new file mode 100644 index 0000000..41342b6 --- /dev/null +++ b/backend/app/models/training.py @@ -0,0 +1,14 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + + +class TrainingStatus(BaseModel): + current_step: int + total_steps: int + loss: float + learning_rate: float + eta_seconds: Optional[float] + steps_per_second: float + updated_at: datetime diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/sample_manager.py b/backend/app/services/sample_manager.py new file mode 100644 index 0000000..8cfb468 --- /dev/null +++ b/backend/app/services/sample_manager.py @@ -0,0 +1,15 @@ +from typing import List + +from app.models.sample import Sample + + +class SampleManager: + async def list_samples(self, limit: int, offset: int) -> List[Sample]: + # Implementation for listing samples from SFTP + # This is a placeholder - actual implementation needed + pass + + async def get_latest_samples(self, count: int) -> List[Sample]: + # Implementation for getting latest samples + # This is a placeholder - actual implementation needed + pass diff --git a/backend/app/services/training_monitor.py b/backend/app/services/training_monitor.py new file mode 100644 index 0000000..0ebf278 --- /dev/null +++ b/backend/app/services/training_monitor.py @@ -0,0 +1,13 @@ +from app.models.training import TrainingStatus + + +class TrainingMonitor: + async def get_status(self) -> TrainingStatus: + # Implementation for parsing tqdm output + # This is a placeholder - actual implementation needed + pass + + async def get_log(self): + # Implementation for getting recent log entries + # This is a placeholder - actual implementation needed + pass diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..65b88a5 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,13 @@ +fastapi>=0.104.0 +uvicorn>=0.24.0 +python-multipart>=0.0.6 +python-jose[cryptography]>=3.3.0 +passlib[bcrypt]>=1.7.4 +pydantic>=2.4.2 +pydantic-settings>=2.0.3 +paramiko>=3.3.1 +python-dotenv>=1.0.0 +aiofiles>=23.2.1 +pytest>=7.4.3 +httpx>=0.25.1 +pytest-asyncio>=0.21.1 \ No newline at end of file