diff --git a/api/main.py b/api/main.py index 75c7a8e..bee08f6 100644 --- a/api/main.py +++ b/api/main.py @@ -1,9 +1,10 @@ -from fastapi import FastAPI, Depends, HTTPException, status +from fastapi import BackgroundTasks, FastAPI, Depends, HTTPException, status from fastapi.middleware.cors import CORSMiddleware from sqlalchemy import text -from datetime import timedelta +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo from database.database import SessionLocal -from database.models import Document, User +from database.models import Document, ScraperRun, User from api.auth import ( UserLogin, UserCreate, Token, UserResponse, RoleUpdate, get_password_hash, verify_password, @@ -212,4 +213,98 @@ def admin_delete_document(doc_id: int, token: str): db.delete(doc) db.commit() db.close() - return {"message": "Document supprimé"} \ No newline at end of file + return {"message": "Document supprimé"} +# ── Scrapers ───────────────────────────────────────────────── +# Les scrapers tournent aussi via le cron hôte (scripts/regwatch-scrape.sh, +# dimanche 3h). Ces endpoints permettent de les déclencher à la demande depuis +# la page admin et de consulter l'historique des exécutions. + +SCRAPER_SOURCES = ("CIR", "SCCS") +PARIS = ZoneInfo("Europe/Paris") + +def _serialize_run(run: ScraperRun) -> dict: + started = run.started_at.astimezone(PARIS) if run.started_at else None + return { + "id": str(run.id), + "when": started.strftime("%d/%m/%Y %H:%M") if started else "—", + "source": run.source, + "status": run.status, + "documents_added": run.documents_added or 0, + "message": run.message, + } + +def _execute_scraper(run_id: int, source: str) -> None: + """Exécuté en tâche de fond : lance le scraper puis clôt la ligne de journal.""" + # Import tardif : ollama/pdfplumber/bs4 sont lourds, inutile de les charger + # au démarrage de l'API alors qu'ils ne servent qu'ici. + from scrapers.cir_scraper import scrape_cir + from scrapers.sccs_scraper import scrape_sccs + + db = SessionLocal() + run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first() + run.status = "running" + db.commit() + before = db.query(Document).filter(Document.source == source).count() + db.close() + + status, message = "success", None + try: + if source == "CIR": + scrape_cir() + else: + scrape_sccs() + except Exception as exc: + status, message = "error", str(exc)[:500] + + db = SessionLocal() + run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first() + run.status = status + run.message = message + run.documents_added = max(db.query(Document).filter(Document.source == source).count() - before, 0) + run.finished_at = datetime.now(timezone.utc) + db.commit() + db.close() + +@app.get("/admin/scrapers/logs") +def admin_scraper_logs(token: str, limit: int = 20): + require_admin(token) + db = SessionLocal() + runs = db.query(ScraperRun).order_by(ScraperRun.id.desc()).limit(limit).all() + result = [_serialize_run(r) for r in runs] + db.close() + return result + +@app.post("/admin/scrapers/run") +def admin_scraper_run(token: str, source: str, background_tasks: BackgroundTasks): + require_admin(token) + + source = source.upper() + if source not in SCRAPER_SOURCES: + raise HTTPException(status_code=400, detail=f"Source inconnue : {source}") + + db = SessionLocal() + already = ( + db.query(ScraperRun) + .filter(ScraperRun.source == source, ScraperRun.status.in_(("pending", "running"))) + .first() + ) + if already: + payload = _serialize_run(already) + db.close() + raise HTTPException(status_code=409, detail=f"Un scraper {source} est déjà en cours (#{payload['id']})") + + run = ScraperRun( + source=source, + status="pending", + started_at=datetime.now(timezone.utc), + documents_added=0, + ) + db.add(run) + db.commit() + db.refresh(run) + payload = _serialize_run(run) + run_id = run.id + db.close() + + background_tasks.add_task(_execute_scraper, run_id, source) + return payload diff --git a/database/models.py b/database/models.py index 6638365..4d7dad8 100644 --- a/database/models.py +++ b/database/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, String +from sqlalchemy import Column, DateTime, Integer, String from database.database import Base class User(Base): @@ -17,4 +17,15 @@ class Document(Base): source = Column(String) document_type = Column(String) meeting_date = Column(String, nullable=True) - pdf_url = Column(String, unique=True) \ No newline at end of file + pdf_url = Column(String, unique=True) + +class ScraperRun(Base): + """Une exécution de scraper, déclenchée depuis la page admin ou par le cron.""" + __tablename__ = "scraper_runs" + id = Column(Integer, primary_key=True, index=True) + source = Column(String, index=True) # CIR | SCCS + status = Column(String, default="pending") # pending | running | success | error + started_at = Column(DateTime(timezone=True)) + finished_at = Column(DateTime(timezone=True), nullable=True) + documents_added = Column(Integer, default=0) + message = Column(String, nullable=True) \ No newline at end of file