RegWatch backend : FastAPI + Postgres partagé + Docker/CI homelab

- SECRET_KEY et DATABASE_URL externalisés (Vault -> env), plus de secret en dur
- SQLite -> postgres-shared (psycopg2), schéma créé au boot (entrypoint.sh)
- Dockerfile + docker-compose (Watchtower) + .gitea/workflows/build.yml

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.env
+*.db
+venv/
+__pycache__/
+*.pyc
+.gitea/
+docker-compose.yml
+.DS_Store

.gitea/workflows/build.yml

@@ -0,0 +1,31 @@
+name: Build & Deploy
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch: {}
+
+env:
+  IMAGE: git.nfteam.ovh/mouigni/regwatch-backend
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build & push image
+        run: |
+          set -e
+          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login git.nfteam.ovh -u "${{ secrets.REGISTRY_USER }}" --password-stdin
+          SHA="${GITHUB_SHA::12}"
+          docker build -t "${IMAGE}:latest" -t "${IMAGE}:${SHA}" -f Dockerfile .
+          docker push --all-tags "${IMAGE}"
+      - name: Notify ntfy
+        if: always()
+        run: |
+          if [ "${{ job.status }}" = "success" ]; then EMOJI="white_check_mark"; PRIO="default"; else EMOJI="rotating_light"; PRIO="high"; fi
+          curl -s -H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" -H "Title: ${GITHUB_REPOSITORY} — ${{ job.status }}" \
+            -H "Priority: ${PRIO}" -H "Tags: ${EMOJI}" -H "Click: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions" \
+            -d "${GITHUB_WORKFLOW} (${GITHUB_REF_NAME} #${GITHUB_RUN_NUMBER}) : ${{ job.status }}" \
+            "${{ secrets.NTFY_URL }}/${{ secrets.NTFY_TOPIC }}" || true

.gitignore

@@ -0,0 +1,6 @@
+.env
+*.db
+venv/
+__pycache__/
+*.pyc
+.DS_Store

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Dépendances Python (wheels : psycopg2-binary, argon2-cffi, cryptography -> pas besoin de toolchain).
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+RUN chmod +x entrypoint.sh
+
+EXPOSE 8000
+CMD ["./entrypoint.sh"]

api/auth.py

@@ -0,0 +1,61 @@
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+import os
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+from pydantic import BaseModel
+
+# Secret JWT injecté par l'env (Vault -> .env). Repli dev uniquement ; en prod SECRET_KEY est obligatoire.
+SECRET_KEY = os.getenv("SECRET_KEY", "dev-only-insecure-secret-change-me")
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "30"))
+
+pwd_context = CryptContext(schemes=["argon2"], deprecated="auto")
+
+class Token(BaseModel):
+    access_token: str
+    token_type: str
+
+class TokenData(BaseModel):
+    email: Optional[str] = None
+
+class UserLogin(BaseModel):
+    email: str
+    password: str
+
+class UserCreate(BaseModel):
+    email: str
+    password: str
+    full_name: Optional[str] = None
+
+class UserResponse(BaseModel):
+    id: int
+    email: str
+    full_name: Optional[str] = None
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    return pwd_context.verify(plain_password, hashed_password)
+
+def get_password_hash(password: str) -> str:
+    return pwd_context.hash(password)
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None):
+    to_encode = data.copy()
+    if expires_delta:
+        expire = datetime.now(timezone.utc) + expires_delta
+    else:
+        expire = datetime.now(timezone.utc) + timedelta(minutes=15)
+    to_encode.update({"exp": expire})
+    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+def decode_token(token: str) -> Optional[str]:
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        email: str = payload.get("sub")
+        if email is None:
+            return None
+        return email
+    except JWTError:
+        return None
+

api/main.py

@@ -0,0 +1,141 @@
+from fastapi import FastAPI, Depends, HTTPException, status
+from fastapi.middleware.cors import CORSMiddleware
+from datetime import timedelta
+from database.database import SessionLocal
+from database.models import Document, User
+from api.auth import (
+    UserLogin, UserCreate, Token, UserResponse, get_password_hash, verify_password,
+    create_access_token, decode_token, ACCESS_TOKEN_EXPIRE_MINUTES
+)
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/")
+def root():
+    return {"message": "RegWatch API running"}
+
+@app.post("/test-login")
+def test_login(credentials: UserLogin):
+    return {"received": credentials.dict()}
+
+@app.post("/signup", response_model=UserResponse)
+def signup(user: UserCreate):
+    db = SessionLocal()
+    
+    existing_user = db.query(User).filter(User.email == user.email).first()
+    if existing_user:
+        db.close()
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Email already registered"
+        )
+    
+    hashed_password = get_password_hash(user.password)
+    db_user = User(
+        email=user.email,
+        password=hashed_password,
+        full_name=user.full_name
+    )
+    db.add(db_user)
+    db.commit()
+    db.refresh(db_user)
+    db.close()
+    
+    return UserResponse(
+        id=db_user.id,
+        email=db_user.email,
+        full_name=db_user.full_name
+    )
+
+@app.post("/login", response_model=Token)
+def login(credentials: UserLogin):
+    try:
+        db = SessionLocal()
+        
+        user = db.query(User).filter(User.email == credentials.email).first()
+        db.close()
+        
+        if not user:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="User not found"
+            )
+        
+        if not verify_password(credentials.password, user.password):
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Invalid password"
+            )
+        
+        access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
+        access_token = create_access_token(
+            data={"sub": user.email}, expires_delta=access_token_expires
+        )
+        
+        return Token(access_token=access_token, token_type="bearer")
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"ERROR in login: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(
+            status_code=500,
+            detail=f"Server error: {str(e)}"
+        )
+
+@app.get("/me", response_model=UserResponse)
+def get_current_user(token: str):
+    email = decode_token(token)
+    if not email:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token"
+        )
+    
+    db = SessionLocal()
+    user = db.query(User).filter(User.email == email).first()
+    db.close()
+    
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="User not found"
+        )
+    
+    return UserResponse(
+        id=user.id,
+        email=user.email,
+        full_name=user.full_name
+    )
+
+@app.get("/documents")
+def get_documents():
+    db = SessionLocal()
+
+    documents = db.query(Document).all()
+
+    result = []
+
+    for doc in documents:
+        result.append({
+            "id": doc.id,
+            "title": doc.title,
+            "ingredient": doc.ingredient,
+            "source": doc.source,
+            "type": doc.document_type,
+            "date": doc.meeting_date,
+            "pdf_url": doc.pdf_url
+        })
+
+    db.close()
+
+    return result

check_db.py

@@ -0,0 +1,11 @@
+from database.database import SessionLocal
+from database.models import Document
+
+db = SessionLocal()
+
+documents = db.query(Document).all()
+
+print(f"Total documents: {len(documents)}")
+
+for doc in documents[:5]:
+    print(doc.title)

database/database.py

@@ -0,0 +1,25 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base
+import os
+
+# En prod (homelab) : DATABASE_URL pointe sur postgres-shared (injecté par l'env / Vault).
+# En dev local : repli sur un SQLite à la racine du backend.
+DB_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "regwatch.db")
+DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DB_PATH}")
+
+# check_same_thread n'a de sens que pour SQLite.
+connect_args = {"check_same_thread": False} if DATABASE_URL.startswith("sqlite") else {}
+
+engine = create_engine(
+    DATABASE_URL,
+    connect_args=connect_args,
+    pool_pre_ping=True,
+)
+
+SessionLocal = sessionmaker(
+    autocommit=False,
+    autoflush=False,
+    bind=engine
+)
+
+Base = declarative_base()

database/models.py

@@ -0,0 +1,21 @@
+from sqlalchemy import Column, Integer, String
+from database.database import Base
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = Column(Integer, primary_key=True, index=True)
+    email = Column(String, unique=True, index=True)
+    password = Column(String)
+    full_name = Column(String, nullable=True)
+
+class Document(Base):
+    __tablename__ = "documents"
+
+    id = Column(Integer, primary_key=True, index=True)
+    title = Column(String)
+    ingredient = Column(String)
+    source = Column(String)
+    document_type = Column(String)
+    meeting_date = Column(String, nullable=True)
+    pdf_url = Column(String, unique=True)

docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  regwatch-backend:
+    image: git.nfteam.ovh/mouigni/regwatch-backend:latest
+    container_name: regwatch-backend
+    restart: unless-stopped
+    pull_policy: always
+    env_file: .env
+    ports:
+      - "8114:8000"
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+      - "homepage.group=RegWatch"
+      - "homepage.name=RegWatch API"
+      - "homepage.icon=fastapi.png"
+      - "homepage.href=https://regwatch-api.nfteam.ovh"
+      - "homepage.description=API veille réglementaire (FastAPI, scrapers CIR/SCCS)"

entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+# Crée les tables si absentes (idempotent). PAS de user de test en prod (contrairement à init_db.py).
+python -c "from database.models import Base; from database.database import engine; Base.metadata.create_all(bind=engine); print('regwatch: schema OK')"
+exec uvicorn api.main:app --host 0.0.0.0 --port 8000

init_db.py

@@ -0,0 +1,27 @@
+from database.database import engine, SessionLocal
+from database.models import Base, User
+from api.auth import get_password_hash
+
+Base.metadata.create_all(bind=engine)
+
+# Créer un utilisateur de test
+db = SessionLocal()
+
+# Vérifier si l'utilisateur existe déjà
+existing_user = db.query(User).filter(User.email == "admin@test.com").first()
+
+if not existing_user:
+    test_user = User(
+        email="admin@test.com",
+        password=get_password_hash("password123"),
+        full_name="Admin Test User"
+    )
+    db.add(test_user)
+    db.commit()
+    print("Test user created: admin@test.com / password123")
+else:
+    print("Test user already exists")
+
+db.close()
+
+print("Database initialized successfully.")

requirements.txt

@@ -0,0 +1,11 @@
+fastapi
+uvicorn
+sqlalchemy
+requests
+beautifulsoup4
+pydantic
+ollama
+python-jose[cryptography]
+passlib[argon2]
+python-multipart
+psycopg2-binary

scrapers/cir_scraper.py

@@ -0,0 +1,276 @@
+import sys
+import os
+import requests
+import time
+import re
+import io
+import ollama
+import json
+import pdfplumber
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+
+sys.path.append(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..")
+    )
+)
+
+from database.database import SessionLocal, engine
+from database.models import Document, Base
+
+
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+}
+
+session = requests.Session()
+session.headers.update(headers)
+
+BASE_URL = "https://www.cir-safety.org"
+MEETING_NUMBERS = range(115, 175)
+PDF_LIMIT = 100
+
+url_templates = [
+    "https://www.cir-safety.org/meeting/{num}th-expert-panel-meeting",
+    "https://www.cir-safety.org/meeting/{num}st-expert-panel-meeting",
+    "https://www.cir-safety.org/meeting/{num}nd-expert-panel-meeting",
+    "https://www.cir-safety.org/meeting/{num}rd-expert-panel-meeting",
+    "https://www.cir-safety.org/meeting/{num}th-cir-expert-panel-meeting",
+    "https://www.cir-safety.org/meeting/{num}st-cir-expert-panel-meeting",
+]
+
+PDF_RE = re.compile(r"\.pdf", re.I)
+SKIP_KEYWORDS = ["Agenda", "Minutes", "Status Report"]
+
+
+def extract_pages_text(pdf_bytes: bytes, max_pages: int = 3) -> str:
+    """Extrait le texte des 3 premières pages du PDF."""
+    try:
+        with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf:
+            if not pdf.pages:
+                return ""
+            texts = []
+            for i, page in enumerate(pdf.pages[:max_pages]):
+                text = page.extract_text() or ""
+                if text.strip():
+                    texts.append(f"--- PAGE {i+1} ---\n{text}")
+            return "\n\n".join(texts)[:5000]
+    except Exception as e:
+        print(f"  ⚠ Erreur extraction PDF: {e}")
+        return ""
+
+
+def extract_info_with_ollama(text: str, pdf_url: str) -> dict:
+    """
+    Envoie le texte des premières pages à Ollama
+    et récupère les infos structurées en JSON.
+    """
+    if not text.strip():
+        filename = pdf_url.split("/")[-1].replace("_", " ").replace(".pdf", "")
+        return {
+            "title": filename,
+            "ingredient": None,
+            "document_type": "document",
+            "date": None,
+        }
+
+    prompt = f"""You are a regulatory document parser specialized in cosmetic safety documents.
+
+Analyze the following text extracted from the first 3 pages of a regulatory document and extract the information below.
+
+Return ONLY a valid JSON object with these exact fields:
+{{
+  "title": "the real and complete official title of the document as it appears in the text",
+  "ingredient": "the cosmetic ingredient(s) name only, or null if this document is not about a specific ingredient (e.g. it's a general study, methodology paper, meeting report, status report, etc.)",
+  "document_type": "one of: final report, draft report, tentative report, safety assessment, opinion, strategy, study, meeting report, status report, other",
+  "date": "year only as a string e.g. '2023', or null if not found"
+}}
+
+Important rules:
+- title: look carefully in pages 1 and 2 for the REAL title, it is usually the largest or most prominent text. Do NOT use a generic description.
+- ingredient: extract ONLY the ingredient name(s). Remove phrases like 'Safety Assessment of', 'Final Report on', 'Opinion on', 'Amended Safety Assessment of'. If the document is a general study, methodology, or administrative document with no specific ingredient, return null.
+- document_type: detect from keywords. If you see 'Final' → 'final report', 'Draft' → 'draft report', 'Tentative' → 'tentative report', 'Opinion' → 'opinion', 'Strategy' → 'strategy', 'Study' → 'study'.
+- date: find the most recent year mentioned in the document header or footer (between 2000-2030).
+- Return ONLY the JSON object, no markdown, no explanation.
+
+Document text (first 3 pages):
+{text}"""
+
+    try:
+        response = ollama.chat(
+            model="llama3.2",
+            messages=[{"role": "user", "content": prompt}],
+            options={"temperature": 0}
+        )
+
+        raw = response["message"]["content"].strip()
+        raw = re.sub(r"```json|```", "", raw).strip()
+
+        # Extrait le JSON même s'il y a du texte autour
+        match = re.search(r"\{.*\}", raw, re.DOTALL)
+        if match:
+            raw = match.group(0)
+
+        data = json.loads(raw)
+
+        ingredient = data.get("ingredient")
+        if ingredient and str(ingredient).strip().lower() in ("null", "none", "n/a", ""):
+            ingredient = None
+
+        return {
+            "title": str(data.get("title") or "").strip()[:200] or "Unknown",
+            "ingredient": str(ingredient).strip()[:200] if ingredient else None,
+            "document_type": str(data.get("document_type") or "document").strip().lower(),
+            "date": str(data.get("date")).strip() if data.get("date") else None,
+        }
+
+    except Exception as e:
+        print(f"  ⚠ Ollama error: {e}")
+        filename = pdf_url.split("/")[-1].replace("_", " ").replace(".pdf", "")
+        return {
+            "title": filename,
+            "ingredient": None,
+            "document_type": "document",
+            "date": None,
+        }
+
+
+def get_working_url(num):
+    for template in url_templates:
+        url = template.format(num=num)
+        try:
+            r = session.get(url, timeout=20)
+            if r.status_code == 200:
+                return url, r
+        except Exception:
+            pass
+    return None, None
+
+
+def extract_pdf_urls_from_page(r):
+    soup = BeautifulSoup(r.text, "html.parser")
+    all_pdf_links = soup.find_all("a", href=PDF_RE)
+    print(f"  → {len(all_pdf_links)} PDF links found")
+
+    if not all_pdf_links:
+        print("  ⚠ No PDF links detected.")
+        return []
+
+    urls = []
+    seen = set()
+
+    for link in all_pdf_links:
+        href = link.get("href", "")
+        pdf_url = urljoin(BASE_URL, href)
+
+        if pdf_url in seen:
+            continue
+        seen.add(pdf_url)
+
+        context_el = None
+        for tag in ("tr", "li", "div", "p"):
+            context_el = link.find_parent(tag)
+            if context_el:
+                break
+        text = context_el.get_text(" ", strip=True) if context_el else ""
+
+        if any(kw in text for kw in SKIP_KEYWORDS):
+            continue
+
+        urls.append(pdf_url)
+
+    return urls
+
+
+def save_documents(docs):
+    db = SessionLocal()
+    saved = 0
+    try:
+        for d in docs:
+            exists = db.query(Document).filter(Document.pdf_url == d["pdf_url"]).first()
+            if not exists:
+                db.add(Document(**d))
+                saved += 1
+        db.commit()
+        print(f"  ✓ {saved} new document(s) saved")
+    except Exception as e:
+        print(f"  DB error: {e}")
+        db.rollback()
+    finally:
+        db.close()
+
+
+def scrape_cir(limit=PDF_LIMIT):
+    Base.metadata.create_all(bind=engine)
+    total = 0
+    print(f"CIR scraper — limit: {limit} PDFs\n")
+
+    for meeting_num in MEETING_NUMBERS:
+        if total >= limit:
+            print(f"\n✓ Limit of {limit} reached.")
+            break
+
+        print(f"\n{'='*60}")
+        print(f"Meeting: {meeting_num}  ({total}/{limit})")
+
+        url, r = get_working_url(meeting_num)
+        if not url:
+            print("  ✗ No page found")
+            continue
+
+        print(f"  ✓ {url}")
+
+        try:
+            pdf_urls = extract_pdf_urls_from_page(r)
+            docs_to_save = []
+
+            for pdf_url in pdf_urls:
+                if total >= limit:
+                    break
+
+                print(f"\n  Downloading: {pdf_url.split('/')[-1]}")
+
+                try:
+                    pdf_resp = session.get(pdf_url, timeout=30)
+                    if pdf_resp.status_code != 200:
+                        print(f"  ✗ HTTP {pdf_resp.status_code}")
+                        continue
+
+                    text = extract_pages_text(pdf_resp.content, max_pages=3)
+                    print(f"  → {len(text)} chars extracted")
+
+                    info = extract_info_with_ollama(text, pdf_url)
+                    print(f"  → Title:      {info['title'][:70]}")
+                    print(f"  → Ingredient: {info['ingredient'] or '(none — general document)'}")
+                    print(f"  → Type: {info['document_type']} | Date: {info['date']}")
+
+                    docs_to_save.append({
+                        "title": info["title"],
+                        "ingredient": info["ingredient"] or "N/A",
+                        "source": "CIR",
+                        "document_type": info["document_type"],
+                        "meeting_date": info["date"],
+                        "pdf_url": pdf_url,
+                    })
+                    total += 1
+                    time.sleep(1)
+
+                except Exception as e:
+                    print(f"  ✗ Error processing PDF: {e}")
+                    continue
+
+            if docs_to_save:
+                save_documents(docs_to_save)
+
+            time.sleep(2)
+
+        except Exception as e:
+            print(f"  ✗ Failed: {e}")
+
+    print(f"\n{'='*60}")
+    print(f"CIR scraping complete: {total}/{limit} PDFs")
+
+
+if __name__ == "__main__":
+    scrape_cir()

scrapers/sccs_scraper.py

@@ -0,0 +1,271 @@
+import sys
+import os
+import requests
+import time
+import re
+import io
+import ollama
+import json
+import pdfplumber
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+
+sys.path.append(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..")
+    )
+)
+
+from database.database import SessionLocal, engine
+from database.models import Document, Base
+
+BASE_URL = "https://health.ec.europa.eu"
+START_URL = (
+    "https://health.ec.europa.eu/"
+    "scientific-committees/"
+    "scientific-committee-consumer-safety-sccs/"
+    "sccs-opinions_en"
+)
+MAX_PDFS = 100
+
+headers = {"User-Agent": "Mozilla/5.0"}
+session = requests.Session()
+session.headers.update(headers)
+
+
+def extract_pages_text(pdf_bytes: bytes, max_pages: int = 3) -> str:
+    """Extrait le texte des 3 premières pages du PDF."""
+    try:
+        with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf:
+            if not pdf.pages:
+                return ""
+            texts = []
+            for i, page in enumerate(pdf.pages[:max_pages]):
+                text = page.extract_text() or ""
+                if text.strip():
+                    texts.append(f"--- PAGE {i+1} ---\n{text}")
+            return "\n\n".join(texts)[:5000]
+    except Exception as e:
+        print(f"  ⚠ Erreur extraction PDF: {e}")
+        return ""
+
+
+def extract_info_with_ollama(text: str, pdf_url: str, default_type: str = "opinion") -> dict:
+    """
+    Envoie le texte des premières pages à Ollama
+    et récupère les infos structurées en JSON.
+    """
+    if not text.strip():
+        filename = pdf_url.split("/")[-1].replace("_", " ").replace(".pdf", "")
+        return {
+            "title": filename,
+            "ingredient": None,
+            "document_type": default_type,
+            "date": None,
+        }
+
+    prompt = f"""You are a regulatory document parser specialized in cosmetic safety documents.
+
+Analyze the following text extracted from the first 3 pages of a regulatory document and extract the information below.
+
+Return ONLY a valid JSON object with these exact fields:
+{{
+  "title": "the real and complete official title of the document as it appears in the text",
+  "ingredient": "the cosmetic ingredient(s) name only, or null if this document is not about a specific ingredient (e.g. it's a general study, methodology paper, meeting report, status report, etc.)",
+  "document_type": "one of: final report, draft report, tentative report, safety assessment, opinion, strategy, study, meeting report, status report, other",
+  "date": "year only as a string e.g. '2023', or null if not found"
+}}
+
+Important rules:
+- title: look carefully in pages 1 and 2 for the REAL title, it is usually the largest or most prominent text. Do NOT use a generic description.
+- ingredient: extract ONLY the ingredient name(s). Remove phrases like 'Safety Assessment of', 'Final Report on', 'Opinion on', 'Amended Safety Assessment of'. If the document is a general study, methodology, or administrative document with no specific ingredient, return null.
+- document_type: detect from keywords. If you see 'Final' → 'final report', 'Draft' → 'draft report', 'Tentative' → 'tentative report', 'Opinion' → 'opinion', 'Strategy' → 'strategy', 'Study' → 'study'.
+- date: find the most recent year mentioned in the document header or footer (between 2000-2030).
+- Return ONLY the JSON object, no markdown, no explanation.
+
+Document text (first 3 pages):
+{text}"""
+
+    try:
+        response = ollama.chat(
+            model="llama3.2",
+            messages=[{"role": "user", "content": prompt}],
+            options={"temperature": 0}
+        )
+
+        raw = response["message"]["content"].strip()
+        raw = re.sub(r"```json|```", "", raw).strip()
+
+        match = re.search(r"\{.*\}", raw, re.DOTALL)
+        if match:
+            raw = match.group(0)
+
+        data = json.loads(raw)
+
+        ingredient = data.get("ingredient")
+        if ingredient and str(ingredient).strip().lower() in ("null", "none", "n/a", ""):
+            ingredient = None
+
+        return {
+            "title": str(data.get("title") or "").strip()[:200] or "Unknown",
+            "ingredient": str(ingredient).strip()[:200] if ingredient else None,
+            "document_type": str(data.get("document_type") or default_type).strip().lower(),
+            "date": str(data.get("date")).strip() if data.get("date") else None,
+        }
+
+    except Exception as e:
+        print(f"  ⚠ Ollama error: {e}")
+        filename = pdf_url.split("/")[-1].replace("_", " ").replace(".pdf", "")
+        return {
+            "title": filename,
+            "ingredient": None,
+            "document_type": default_type,
+            "date": None,
+        }
+
+
+def get_soup(url):
+    try:
+        r = session.get(url, timeout=25)
+        if r.status_code == 429:
+            print("429 → waiting 60s")
+            time.sleep(60)
+            r = session.get(url, timeout=25)
+        r.raise_for_status()
+        return BeautifulSoup(r.text, "html.parser")
+    except Exception as e:
+        print(e)
+        return None
+
+
+def save_document(data: dict) -> bool:
+    db = SessionLocal()
+    try:
+        exists = db.query(Document).filter(Document.pdf_url == data["pdf_url"]).first()
+        if exists:
+            return False
+        db.add(Document(**data))
+        db.commit()
+        return True
+    except Exception as e:
+        print(f"  DB error: {e}")
+        db.rollback()
+        return False
+    finally:
+        db.close()
+
+
+def process_pdf(pdf_url: str) -> dict | None:
+    print(f"\n  Downloading: {pdf_url.split('/')[-1]}")
+    try:
+        r = session.get(pdf_url, timeout=30)
+        if r.status_code != 200:
+            print(f"  ✗ HTTP {r.status_code}")
+            return None
+
+        text = extract_pages_text(r.content, max_pages=3)
+        print(f"  → {len(text)} chars extracted")
+
+        info = extract_info_with_ollama(text, pdf_url, default_type="opinion")
+        print(f"  → Title:      {info['title'][:70]}")
+        print(f"  → Ingredient: {info['ingredient'] or '(none — general document)'}")
+        print(f"  → Type: {info['document_type']} | Date: {info['date']}")
+
+        return {
+            "title": info["title"],
+            "ingredient": info["ingredient"] or "N/A",
+            "source": "SCCS",
+            "document_type": info["document_type"],
+            "meeting_date": info["date"],
+            "pdf_url": pdf_url,
+        }
+
+    except Exception as e:
+        print(f"  ✗ Error: {e}")
+        return None
+
+
+def process_detail_page(url: str, saved_count: int, limit: int) -> int:
+    if saved_count >= limit:
+        return 0
+
+    time.sleep(2)
+    print(f"\n  Page: {url.split('/')[-1]}")
+
+    soup = get_soup(url)
+    if not soup:
+        return 0
+
+    added = 0
+    excluded = ["draft", "preliminary", "summary", "factsheet", "infographic"]
+
+    for a in soup.find_all("a", href=re.compile(r"\.pdf$", re.I)):
+        if saved_count + added >= limit:
+            break
+
+        link_text = a.get_text(strip=True).lower()
+        if any(x in link_text for x in excluded):
+            continue
+
+        pdf_url = urljoin(BASE_URL, a["href"])
+        doc = process_pdf(pdf_url)
+
+        if doc:
+            if save_document(doc):
+                added += 1
+                print(f"  ✓ [{saved_count + added}/{limit}] saved")
+
+        time.sleep(1)
+
+    return added
+
+
+def scrape_sccs(limit=MAX_PDFS):
+    Base.metadata.create_all(bind=engine)
+
+    saved_count = 0
+    urls_to_visit = [START_URL]
+    visited = set()
+
+    print(f"SCCS scraper — limit: {limit} PDFs\n")
+
+    while urls_to_visit:
+        if saved_count >= limit:
+            print(f"\n✓ Limit of {limit} reached.")
+            break
+
+        current = urls_to_visit.pop(0)
+        if current in visited:
+            continue
+        visited.add(current)
+
+        print(f"\n{'='*60}")
+        print(f"Exploring: {current.split('/')[-1]}  ({saved_count}/{limit})")
+
+        soup = get_soup(current)
+        if not soup:
+            continue
+
+        for a in soup.find_all("a", href=True):
+            if saved_count >= limit:
+                break
+
+            href = a["href"].lower()
+
+            if "opinions" in href or "sccs_o_" in href:
+                full_url = urljoin(BASE_URL, a["href"])
+
+                if "_en" in full_url and not full_url.endswith(".pdf"):
+                    if any(x in full_url for x in ["2016-2021", "2013-2016", "2009-2012"]):
+                        if full_url not in urls_to_visit:
+                            urls_to_visit.append(full_url)
+                    else:
+                        added = process_detail_page(full_url, saved_count, limit)
+                        saved_count += added
+
+    print(f"\n{'='*60}")
+    print(f"SCCS scraping complete: {saved_count}/{limit} PDFs")
+
+
+if __name__ == "__main__":
+    scrape_sccs()