Files
regwatch-backend/api/main.py
T
neckfireandClaude Opus 5 f8c8e16e7e
Build & Deploy / build (push) Successful in 16s
Scrapers : endpoints de lancement à la demande et d'historique
La page admin appelait /admin/scrapers/logs et /admin/scrapers/run, absents de
l'API : 404 sur cet onglet.

- Nouvelle table scraper_runs (source, status, started_at, finished_at,
  documents_added, message), créée au boot par le create_all de entrypoint.sh.
- GET /admin/scrapers/logs : les 20 dernières exécutions, réservé aux admins.
- POST /admin/scrapers/run?source=CIR|SCCS : journalise puis lance le scraper en
  tâche de fond, et renvoie la ligne de journal attendue par le front. Refuse en
  409 si une exécution de la même source est déjà en cours.
- Le nombre de documents ajoutés est mesuré avant/après, comme le fait déjà le
  cron hôte.

Les imports de scrapers sont tardifs : ollama/pdfplumber/bs4 sont lourds et ne
servent qu'ici, inutile de les charger au démarrage de l'API.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-15 16:33:45 +02:00

311 lines
11 KiB
Python

from fastapi import BackgroundTasks, FastAPI, Depends, HTTPException, status
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy import text
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo
from database.database import SessionLocal
from database.models import Document, ScraperRun, User
from api.auth import (
UserLogin, UserCreate, Token, UserResponse, RoleUpdate,
get_password_hash, verify_password,
create_access_token, decode_token, ACCESS_TOKEN_EXPIRE_MINUTES
)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# ── Fonction vérification admin ──────────────────────────────
def require_admin(token: str):
email = decode_token(token)
if not email:
raise HTTPException(status_code=401, detail="Invalid token")
db = SessionLocal()
user = db.query(User).filter(User.email == email).first()
db.close()
if not user or user.role != "admin":
raise HTTPException(status_code=403, detail="Accès refusé")
return user
# ── Endpoints existants ──────────────────────────────────────
@app.get("/")
def root():
return {"message": "RegWatch API running"}
@app.post("/signup", response_model=UserResponse)
def signup(user: UserCreate):
db = SessionLocal()
existing_user = db.query(User).filter(User.email == user.email).first()
if existing_user:
db.close()
raise HTTPException(status_code=400, detail="Email already registered")
hashed_password = get_password_hash(user.password)
db_user = User(email=user.email, password=hashed_password, full_name=user.full_name)
db.add(db_user)
db.commit()
db.refresh(db_user)
db.close()
return UserResponse(id=db_user.id, email=db_user.email, full_name=db_user.full_name, role=db_user.role)
@app.post("/login", response_model=Token)
def login(credentials: UserLogin):
try:
db = SessionLocal()
user = db.query(User).filter(User.email == credentials.email).first()
db.close()
if not user:
raise HTTPException(status_code=401, detail="User not found")
if not verify_password(credentials.password, user.password):
raise HTTPException(status_code=401, detail="Invalid password")
access_token = create_access_token(
data={"sub": user.email},
expires_delta=timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
)
return Token(access_token=access_token, token_type="bearer")
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")
@app.get("/me", response_model=UserResponse)
def get_current_user(token: str):
email = decode_token(token)
if not email:
raise HTTPException(status_code=401, detail="Invalid token")
db = SessionLocal()
user = db.query(User).filter(User.email == email).first()
db.close()
if not user:
raise HTTPException(status_code=404, detail="User not found")
return UserResponse(id=user.id, email=user.email, full_name=user.full_name, role=user.role)
@app.get("/documents")
def get_documents():
db = SessionLocal()
documents = db.query(Document).all()
result = []
for doc in documents:
result.append({
"id": doc.id,
"title": doc.title,
"ingredient": doc.ingredient,
"source": doc.source,
"type": doc.document_type,
"date": doc.meeting_date,
"pdf_url": doc.pdf_url
})
db.close()
return result
# ── Migration colonne role ───────────────────────────────────
@app.post("/admin/migrate-add-role")
def migrate_add_role():
"""Endpoint temporaire — ajoute la colonne role si absente"""
db = SessionLocal()
try:
db.execute(text("ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR DEFAULT 'user'"))
db.commit()
return {"message": "Colonne role ajoutée avec succès"}
except Exception as e:
return {"message": str(e)}
finally:
db.close()
# ── Premier admin sans neckfire ──────────────────────────────
@app.post("/admin/make-admin")
def make_admin(token: str, target_email: str):
"""
Passe un user en admin.
Fonctionne sans auth si aucun admin n'existe encore.
Se désactive automatiquement si un admin existe déjà.
"""
db = SessionLocal()
existing_admins = db.query(User).filter(User.role == "admin").count()
if existing_admins > 0:
# Il y a déjà un admin — vérifier que l'appelant est admin
email = decode_token(token)
caller = db.query(User).filter(User.email == email).first()
if not caller or caller.role != "admin":
db.close()
raise HTTPException(status_code=403, detail="Accès refusé")
target = db.query(User).filter(User.email == target_email).first()
if not target:
db.close()
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
target.role = "admin"
db.commit()
db.close()
return {"message": f"{target_email} est maintenant admin"}
# ── Endpoints admin ──────────────────────────────────────────
@app.get("/admin/stats")
def admin_stats(token: str):
require_admin(token)
db = SessionLocal()
total_docs = db.query(Document).count()
cir_docs = db.query(Document).filter(Document.source == "CIR").count()
sccs_docs = db.query(Document).filter(Document.source == "SCCS").count()
total_users = db.query(User).count()
db.close()
return {
"total_documents": total_docs,
"cir_documents": cir_docs,
"sccs_documents": sccs_docs,
"total_users": total_users
}
@app.get("/admin/users")
def admin_get_users(token: str):
require_admin(token)
db = SessionLocal()
users = db.query(User).all()
result = [{"id": u.id, "email": u.email, "full_name": u.full_name, "role": u.role} for u in users]
db.close()
return result
@app.delete("/admin/users/{user_id}")
def admin_delete_user(user_id: int, token: str):
require_admin(token)
db = SessionLocal()
user = db.query(User).filter(User.id == user_id).first()
if not user:
db.close()
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
db.delete(user)
db.commit()
db.close()
return {"message": "Utilisateur supprimé"}
@app.patch("/admin/users/{user_id}/role")
def admin_update_role(user_id: int, role_update: RoleUpdate, token: str):
require_admin(token)
db = SessionLocal()
user = db.query(User).filter(User.id == user_id).first()
if not user:
db.close()
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
user.role = role_update.role
db.commit()
db.close()
return {"message": f"Rôle mis à jour : {role_update.role}"}
@app.delete("/admin/documents/{doc_id}")
def admin_delete_document(doc_id: int, token: str):
require_admin(token)
db = SessionLocal()
doc = db.query(Document).filter(Document.id == doc_id).first()
if not doc:
db.close()
raise HTTPException(status_code=404, detail="Document introuvable")
db.delete(doc)
db.commit()
db.close()
return {"message": "Document supprimé"}
# ── Scrapers ─────────────────────────────────────────────────
# Les scrapers tournent aussi via le cron hôte (scripts/regwatch-scrape.sh,
# dimanche 3h). Ces endpoints permettent de les déclencher à la demande depuis
# la page admin et de consulter l'historique des exécutions.
SCRAPER_SOURCES = ("CIR", "SCCS")
PARIS = ZoneInfo("Europe/Paris")
def _serialize_run(run: ScraperRun) -> dict:
started = run.started_at.astimezone(PARIS) if run.started_at else None
return {
"id": str(run.id),
"when": started.strftime("%d/%m/%Y %H:%M") if started else "—",
"source": run.source,
"status": run.status,
"documents_added": run.documents_added or 0,
"message": run.message,
}
def _execute_scraper(run_id: int, source: str) -> None:
"""Exécuté en tâche de fond : lance le scraper puis clôt la ligne de journal."""
# Import tardif : ollama/pdfplumber/bs4 sont lourds, inutile de les charger
# au démarrage de l'API alors qu'ils ne servent qu'ici.
from scrapers.cir_scraper import scrape_cir
from scrapers.sccs_scraper import scrape_sccs
db = SessionLocal()
run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first()
run.status = "running"
db.commit()
before = db.query(Document).filter(Document.source == source).count()
db.close()
status, message = "success", None
try:
if source == "CIR":
scrape_cir()
else:
scrape_sccs()
except Exception as exc:
status, message = "error", str(exc)[:500]
db = SessionLocal()
run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first()
run.status = status
run.message = message
run.documents_added = max(db.query(Document).filter(Document.source == source).count() - before, 0)
run.finished_at = datetime.now(timezone.utc)
db.commit()
db.close()
@app.get("/admin/scrapers/logs")
def admin_scraper_logs(token: str, limit: int = 20):
require_admin(token)
db = SessionLocal()
runs = db.query(ScraperRun).order_by(ScraperRun.id.desc()).limit(limit).all()
result = [_serialize_run(r) for r in runs]
db.close()
return result
@app.post("/admin/scrapers/run")
def admin_scraper_run(token: str, source: str, background_tasks: BackgroundTasks):
require_admin(token)
source = source.upper()
if source not in SCRAPER_SOURCES:
raise HTTPException(status_code=400, detail=f"Source inconnue : {source}")
db = SessionLocal()
already = (
db.query(ScraperRun)
.filter(ScraperRun.source == source, ScraperRun.status.in_(("pending", "running")))
.first()
)
if already:
payload = _serialize_run(already)
db.close()
raise HTTPException(status_code=409, detail=f"Un scraper {source} est déjà en cours (#{payload['id']})")
run = ScraperRun(
source=source,
status="pending",
started_at=datetime.now(timezone.utc),
documents_added=0,
)
db.add(run)
db.commit()
db.refresh(run)
payload = _serialize_run(run)
run_id = run.id
db.close()
background_tasks.add_task(_execute_scraper, run_id, source)
return payload