Build & Deploy / build (push) Successful in 16s
La page admin appelait /admin/scrapers/logs et /admin/scrapers/run, absents de l'API : 404 sur cet onglet. - Nouvelle table scraper_runs (source, status, started_at, finished_at, documents_added, message), créée au boot par le create_all de entrypoint.sh. - GET /admin/scrapers/logs : les 20 dernières exécutions, réservé aux admins. - POST /admin/scrapers/run?source=CIR|SCCS : journalise puis lance le scraper en tâche de fond, et renvoie la ligne de journal attendue par le front. Refuse en 409 si une exécution de la même source est déjà en cours. - Le nombre de documents ajoutés est mesuré avant/après, comme le fait déjà le cron hôte. Les imports de scrapers sont tardifs : ollama/pdfplumber/bs4 sont lourds et ne servent qu'ici, inutile de les charger au démarrage de l'API. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
311 lines
11 KiB
Python
311 lines
11 KiB
Python
from fastapi import BackgroundTasks, FastAPI, Depends, HTTPException, status
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from sqlalchemy import text
|
|
from datetime import datetime, timedelta, timezone
|
|
from zoneinfo import ZoneInfo
|
|
from database.database import SessionLocal
|
|
from database.models import Document, ScraperRun, User
|
|
from api.auth import (
|
|
UserLogin, UserCreate, Token, UserResponse, RoleUpdate,
|
|
get_password_hash, verify_password,
|
|
create_access_token, decode_token, ACCESS_TOKEN_EXPIRE_MINUTES
|
|
)
|
|
|
|
app = FastAPI()
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# ── Fonction vérification admin ──────────────────────────────
|
|
def require_admin(token: str):
|
|
email = decode_token(token)
|
|
if not email:
|
|
raise HTTPException(status_code=401, detail="Invalid token")
|
|
db = SessionLocal()
|
|
user = db.query(User).filter(User.email == email).first()
|
|
db.close()
|
|
if not user or user.role != "admin":
|
|
raise HTTPException(status_code=403, detail="Accès refusé")
|
|
return user
|
|
|
|
# ── Endpoints existants ──────────────────────────────────────
|
|
|
|
@app.get("/")
|
|
def root():
|
|
return {"message": "RegWatch API running"}
|
|
|
|
@app.post("/signup", response_model=UserResponse)
|
|
def signup(user: UserCreate):
|
|
db = SessionLocal()
|
|
existing_user = db.query(User).filter(User.email == user.email).first()
|
|
if existing_user:
|
|
db.close()
|
|
raise HTTPException(status_code=400, detail="Email already registered")
|
|
hashed_password = get_password_hash(user.password)
|
|
db_user = User(email=user.email, password=hashed_password, full_name=user.full_name)
|
|
db.add(db_user)
|
|
db.commit()
|
|
db.refresh(db_user)
|
|
db.close()
|
|
return UserResponse(id=db_user.id, email=db_user.email, full_name=db_user.full_name, role=db_user.role)
|
|
|
|
@app.post("/login", response_model=Token)
|
|
def login(credentials: UserLogin):
|
|
try:
|
|
db = SessionLocal()
|
|
user = db.query(User).filter(User.email == credentials.email).first()
|
|
db.close()
|
|
if not user:
|
|
raise HTTPException(status_code=401, detail="User not found")
|
|
if not verify_password(credentials.password, user.password):
|
|
raise HTTPException(status_code=401, detail="Invalid password")
|
|
access_token = create_access_token(
|
|
data={"sub": user.email},
|
|
expires_delta=timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
|
)
|
|
return Token(access_token=access_token, token_type="bearer")
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")
|
|
|
|
@app.get("/me", response_model=UserResponse)
|
|
def get_current_user(token: str):
|
|
email = decode_token(token)
|
|
if not email:
|
|
raise HTTPException(status_code=401, detail="Invalid token")
|
|
db = SessionLocal()
|
|
user = db.query(User).filter(User.email == email).first()
|
|
db.close()
|
|
if not user:
|
|
raise HTTPException(status_code=404, detail="User not found")
|
|
return UserResponse(id=user.id, email=user.email, full_name=user.full_name, role=user.role)
|
|
|
|
@app.get("/documents")
|
|
def get_documents():
|
|
db = SessionLocal()
|
|
documents = db.query(Document).all()
|
|
result = []
|
|
for doc in documents:
|
|
result.append({
|
|
"id": doc.id,
|
|
"title": doc.title,
|
|
"ingredient": doc.ingredient,
|
|
"source": doc.source,
|
|
"type": doc.document_type,
|
|
"date": doc.meeting_date,
|
|
"pdf_url": doc.pdf_url
|
|
})
|
|
db.close()
|
|
return result
|
|
|
|
# ── Migration colonne role ───────────────────────────────────
|
|
|
|
@app.post("/admin/migrate-add-role")
|
|
def migrate_add_role():
|
|
"""Endpoint temporaire — ajoute la colonne role si absente"""
|
|
db = SessionLocal()
|
|
try:
|
|
db.execute(text("ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR DEFAULT 'user'"))
|
|
db.commit()
|
|
return {"message": "Colonne role ajoutée avec succès"}
|
|
except Exception as e:
|
|
return {"message": str(e)}
|
|
finally:
|
|
db.close()
|
|
|
|
# ── Premier admin sans neckfire ──────────────────────────────
|
|
|
|
@app.post("/admin/make-admin")
|
|
def make_admin(token: str, target_email: str):
|
|
"""
|
|
Passe un user en admin.
|
|
Fonctionne sans auth si aucun admin n'existe encore.
|
|
Se désactive automatiquement si un admin existe déjà.
|
|
"""
|
|
db = SessionLocal()
|
|
existing_admins = db.query(User).filter(User.role == "admin").count()
|
|
|
|
if existing_admins > 0:
|
|
# Il y a déjà un admin — vérifier que l'appelant est admin
|
|
email = decode_token(token)
|
|
caller = db.query(User).filter(User.email == email).first()
|
|
if not caller or caller.role != "admin":
|
|
db.close()
|
|
raise HTTPException(status_code=403, detail="Accès refusé")
|
|
|
|
target = db.query(User).filter(User.email == target_email).first()
|
|
if not target:
|
|
db.close()
|
|
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
|
|
|
|
target.role = "admin"
|
|
db.commit()
|
|
db.close()
|
|
return {"message": f"{target_email} est maintenant admin"}
|
|
|
|
# ── Endpoints admin ──────────────────────────────────────────
|
|
|
|
@app.get("/admin/stats")
|
|
def admin_stats(token: str):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
total_docs = db.query(Document).count()
|
|
cir_docs = db.query(Document).filter(Document.source == "CIR").count()
|
|
sccs_docs = db.query(Document).filter(Document.source == "SCCS").count()
|
|
total_users = db.query(User).count()
|
|
db.close()
|
|
return {
|
|
"total_documents": total_docs,
|
|
"cir_documents": cir_docs,
|
|
"sccs_documents": sccs_docs,
|
|
"total_users": total_users
|
|
}
|
|
|
|
@app.get("/admin/users")
|
|
def admin_get_users(token: str):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
users = db.query(User).all()
|
|
result = [{"id": u.id, "email": u.email, "full_name": u.full_name, "role": u.role} for u in users]
|
|
db.close()
|
|
return result
|
|
|
|
@app.delete("/admin/users/{user_id}")
|
|
def admin_delete_user(user_id: int, token: str):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
user = db.query(User).filter(User.id == user_id).first()
|
|
if not user:
|
|
db.close()
|
|
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
|
|
db.delete(user)
|
|
db.commit()
|
|
db.close()
|
|
return {"message": "Utilisateur supprimé"}
|
|
|
|
@app.patch("/admin/users/{user_id}/role")
|
|
def admin_update_role(user_id: int, role_update: RoleUpdate, token: str):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
user = db.query(User).filter(User.id == user_id).first()
|
|
if not user:
|
|
db.close()
|
|
raise HTTPException(status_code=404, detail="Utilisateur introuvable")
|
|
user.role = role_update.role
|
|
db.commit()
|
|
db.close()
|
|
return {"message": f"Rôle mis à jour : {role_update.role}"}
|
|
|
|
@app.delete("/admin/documents/{doc_id}")
|
|
def admin_delete_document(doc_id: int, token: str):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
doc = db.query(Document).filter(Document.id == doc_id).first()
|
|
if not doc:
|
|
db.close()
|
|
raise HTTPException(status_code=404, detail="Document introuvable")
|
|
db.delete(doc)
|
|
db.commit()
|
|
db.close()
|
|
return {"message": "Document supprimé"}
|
|
# ── Scrapers ─────────────────────────────────────────────────
|
|
# Les scrapers tournent aussi via le cron hôte (scripts/regwatch-scrape.sh,
|
|
# dimanche 3h). Ces endpoints permettent de les déclencher à la demande depuis
|
|
# la page admin et de consulter l'historique des exécutions.
|
|
|
|
SCRAPER_SOURCES = ("CIR", "SCCS")
|
|
PARIS = ZoneInfo("Europe/Paris")
|
|
|
|
def _serialize_run(run: ScraperRun) -> dict:
|
|
started = run.started_at.astimezone(PARIS) if run.started_at else None
|
|
return {
|
|
"id": str(run.id),
|
|
"when": started.strftime("%d/%m/%Y %H:%M") if started else "—",
|
|
"source": run.source,
|
|
"status": run.status,
|
|
"documents_added": run.documents_added or 0,
|
|
"message": run.message,
|
|
}
|
|
|
|
def _execute_scraper(run_id: int, source: str) -> None:
|
|
"""Exécuté en tâche de fond : lance le scraper puis clôt la ligne de journal."""
|
|
# Import tardif : ollama/pdfplumber/bs4 sont lourds, inutile de les charger
|
|
# au démarrage de l'API alors qu'ils ne servent qu'ici.
|
|
from scrapers.cir_scraper import scrape_cir
|
|
from scrapers.sccs_scraper import scrape_sccs
|
|
|
|
db = SessionLocal()
|
|
run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first()
|
|
run.status = "running"
|
|
db.commit()
|
|
before = db.query(Document).filter(Document.source == source).count()
|
|
db.close()
|
|
|
|
status, message = "success", None
|
|
try:
|
|
if source == "CIR":
|
|
scrape_cir()
|
|
else:
|
|
scrape_sccs()
|
|
except Exception as exc:
|
|
status, message = "error", str(exc)[:500]
|
|
|
|
db = SessionLocal()
|
|
run = db.query(ScraperRun).filter(ScraperRun.id == run_id).first()
|
|
run.status = status
|
|
run.message = message
|
|
run.documents_added = max(db.query(Document).filter(Document.source == source).count() - before, 0)
|
|
run.finished_at = datetime.now(timezone.utc)
|
|
db.commit()
|
|
db.close()
|
|
|
|
@app.get("/admin/scrapers/logs")
|
|
def admin_scraper_logs(token: str, limit: int = 20):
|
|
require_admin(token)
|
|
db = SessionLocal()
|
|
runs = db.query(ScraperRun).order_by(ScraperRun.id.desc()).limit(limit).all()
|
|
result = [_serialize_run(r) for r in runs]
|
|
db.close()
|
|
return result
|
|
|
|
@app.post("/admin/scrapers/run")
|
|
def admin_scraper_run(token: str, source: str, background_tasks: BackgroundTasks):
|
|
require_admin(token)
|
|
|
|
source = source.upper()
|
|
if source not in SCRAPER_SOURCES:
|
|
raise HTTPException(status_code=400, detail=f"Source inconnue : {source}")
|
|
|
|
db = SessionLocal()
|
|
already = (
|
|
db.query(ScraperRun)
|
|
.filter(ScraperRun.source == source, ScraperRun.status.in_(("pending", "running")))
|
|
.first()
|
|
)
|
|
if already:
|
|
payload = _serialize_run(already)
|
|
db.close()
|
|
raise HTTPException(status_code=409, detail=f"Un scraper {source} est déjà en cours (#{payload['id']})")
|
|
|
|
run = ScraperRun(
|
|
source=source,
|
|
status="pending",
|
|
started_at=datetime.now(timezone.utc),
|
|
documents_added=0,
|
|
)
|
|
db.add(run)
|
|
db.commit()
|
|
db.refresh(run)
|
|
payload = _serialize_run(run)
|
|
run_id = run.id
|
|
db.close()
|
|
|
|
background_tasks.add_task(_execute_scraper, run_id, source)
|
|
return payload
|