MVP frontend+backend
This commit is contained in:
@@ -68,8 +68,16 @@ class Settings(BaseSettings):
|
||||
jwt_access_token_expire_minutes: int = 15
|
||||
jwt_refresh_token_expire_days: int = 7
|
||||
|
||||
# FK scraper — disabled until legal/ToS + BankID review (§2.3).
|
||||
log_level: str = "DEBUG"
|
||||
|
||||
# FK scraper — legal/ToS review completed (2026-06). Default is False (conservative).
|
||||
# Set APP_SCRAPER_ENABLED=true in the environment or backend/.env to enable.
|
||||
# Playwright chromium binaries must be installed: `playwright install chromium`.
|
||||
scraper_enabled: bool = False
|
||||
scraper_headless: bool = True
|
||||
scraper_session_ttl_seconds: int = 180 # auto-cancel awaiting_scan/authenticating sessions
|
||||
# Set to None to disable debug artifact saving. Files may contain personal data.
|
||||
scraper_debug_dir: str | None = "./debug/scraper"
|
||||
|
||||
rules: LeaveRules = LeaveRules()
|
||||
|
||||
|
||||
+43
-9
@@ -5,17 +5,19 @@ routers, locks CORS to the configured frontend origin(s), and attaches a
|
||||
rate-limiter to the app (applied per-endpoint via decorators).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi import _rate_limit_exceeded_handler
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
from app.config import settings
|
||||
from app.db.connection import connect, run_migrations
|
||||
from app.rate_limit import limiter
|
||||
from app.routers import vacation_days
|
||||
from app.routers.analytics import router as analytics_router
|
||||
from app.routers.auth import router as auth_router
|
||||
@@ -26,18 +28,43 @@ from app.routers.settings import break_router, child_router, settings_child_rout
|
||||
from app.routers.workspace import _inv_router as invitations_router
|
||||
from app.routers.workspace import router as workspace_router
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _configure_logging() -> None:
|
||||
level = logging.getLevelName(settings.log_level.upper())
|
||||
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(level)
|
||||
handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s"))
|
||||
|
||||
app_logger = logging.getLogger("app")
|
||||
app_logger.handlers.clear()
|
||||
app_logger.addHandler(handler)
|
||||
app_logger.setLevel(level)
|
||||
app_logger.propagate = False
|
||||
app_logger.disabled = False
|
||||
|
||||
for name, log in logging.Logger.manager.loggerDict.items():
|
||||
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
|
||||
log.disabled = False
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# Run pending migrations before accepting requests (dev convenience).
|
||||
# In production, run `alembic upgrade head` as a separate deploy step.
|
||||
_configure_logging()
|
||||
|
||||
if settings.database_type == "sqlite":
|
||||
try:
|
||||
run_migrations(f"sqlite:///{settings.sqlite_path}")
|
||||
except Exception:
|
||||
pass # migrations may fail in test envs without alembic installed
|
||||
await asyncio.to_thread(run_migrations, f"sqlite:///{settings.sqlite_path}")
|
||||
except Exception as exc:
|
||||
logger.error("Migration failed — DB schema may be incomplete: %s", exc, exc_info=True)
|
||||
|
||||
# Re-enable after migrations: alembic's env.py calls fileConfig() which may
|
||||
# still touch logger state even with disable_existing_loggers=False.
|
||||
for name, log in logging.Logger.manager.loggerDict.items():
|
||||
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
|
||||
log.disabled = False
|
||||
|
||||
app.state.db_connection = await connect(settings.sqlite_path)
|
||||
try:
|
||||
@@ -52,6 +79,13 @@ def create_app() -> FastAPI:
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
@app.middleware("http")
|
||||
async def log_error_responses(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if response.status_code >= 400:
|
||||
logger.warning("%s %s %d", request.method, request.url.path, response.status_code)
|
||||
return response
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=list(settings.cors_origins),
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Shared rate-limiter instance.
|
||||
|
||||
Lives in its own module so routers can import it without triggering the
|
||||
circular import that would occur if they imported from app.main.
|
||||
"""
|
||||
|
||||
from slowapi import Limiter
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
@@ -1,11 +1,12 @@
|
||||
"""HTTP routes for auth: register, login, logout, me, refresh."""
|
||||
|
||||
from fastapi import APIRouter, Cookie, HTTPException, Response, status
|
||||
from pydantic import BaseModel
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.config import settings
|
||||
from app.dependencies import AuthServiceDep, CurrentUserDep
|
||||
from app.main import limiter
|
||||
from app.dependencies import AuthServiceDep, ChildRepositoryDep, CurrentUserDep, PersonalDataRepositoryDep
|
||||
from app.rate_limit import limiter
|
||||
from app.models.auth import LoginRequest, RegisterRequest, TokenResponse, UserMeResponse
|
||||
from app.utils.errors import AuthError, ConflictError, UnauthorizedError
|
||||
|
||||
@@ -83,6 +84,41 @@ async def me(current_user: CurrentUserDep) -> UserMeResponse:
|
||||
)
|
||||
|
||||
|
||||
class _ChildContext(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
birth_date: str
|
||||
|
||||
|
||||
class _UserContext(BaseModel):
|
||||
personal_data_uuid: str
|
||||
workspace_id: str
|
||||
parent_name: str
|
||||
children: list[_ChildContext]
|
||||
|
||||
|
||||
@router.get("/context", response_model=_UserContext)
|
||||
async def get_context(
|
||||
current_user: CurrentUserDep,
|
||||
pd_repo: PersonalDataRepositoryDep,
|
||||
child_repo: ChildRepositoryDep,
|
||||
) -> _UserContext:
|
||||
"""Return workspace + children for the current user. Used by the dev harness."""
|
||||
pd = await pd_repo.get_by_id(current_user.personal_data_uuid)
|
||||
if pd is None:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "No parent profile found")
|
||||
children = await child_repo.list_by_workspace(pd.workspace_id)
|
||||
return _UserContext(
|
||||
personal_data_uuid=str(pd.uuid),
|
||||
workspace_id=str(pd.workspace_id),
|
||||
parent_name=pd.name,
|
||||
children=[
|
||||
_ChildContext(id=str(c.id), name=c.name, birth_date=str(c.birth_date))
|
||||
for c in children
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _set_refresh_cookie(response: Response, token: str) -> None:
|
||||
response.set_cookie(
|
||||
key=_REFRESH_COOKIE,
|
||||
|
||||
@@ -1,26 +1,33 @@
|
||||
"""HTTP routes for parental-leave data: manual entry + scraper status.
|
||||
"""HTTP routes for parental-leave data: manual entry + scraper session.
|
||||
|
||||
POST /api/parental-leave/manual — manual entry (always available)
|
||||
GET /api/parental-leave/{personal_data_uuid}/{child_id} — latest snapshot
|
||||
GET /api/parental-leave/{personal_data_uuid}/{child_id}/history — all snapshots
|
||||
|
||||
Scraper endpoints (POST /api/fetch-data/start, GET /api/fetch-data/poll,
|
||||
POST /api/fetch-data/cancel) are stubs returning 501 until the legal gate
|
||||
clears (ARCHITECTURE.md §2.3).
|
||||
Scraper endpoints (start / poll / cancel) return 501 when scraper_enabled is
|
||||
false. When enabled, they drive the FKScraperService session state machine.
|
||||
"""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.config import settings
|
||||
from app.dependencies import CurrentUserDep, ParentalLeaveRepositoryDep
|
||||
from app.main import limiter
|
||||
from app.dependencies import (
|
||||
ChildRepositoryDep,
|
||||
CurrentUserDep,
|
||||
ParentalLeaveRepositoryDep,
|
||||
PersonalDataRepositoryDep,
|
||||
)
|
||||
from app.entities.parental_leave import ParentalLeaveFetch
|
||||
from app.rate_limit import limiter
|
||||
from app.models.parental_leave import (
|
||||
ManualFetchRequest,
|
||||
ParentalLeaveFetchResponse,
|
||||
)
|
||||
from app.scrapers.fk_scraper import FKScraperService
|
||||
from app.services.fetch_data_service import ManualFetchService
|
||||
from app.utils.errors import ValidationError
|
||||
|
||||
@@ -29,14 +36,18 @@ scraper_router = APIRouter(prefix="/api/fetch-data", tags=["fetch-data"])
|
||||
|
||||
_SCRAPER_DISABLED = HTTPException(
|
||||
status.HTTP_501_NOT_IMPLEMENTED,
|
||||
"FK scraping is disabled pending legal/ToS + BankID review (§2.3). "
|
||||
"Use /api/parental-leave/manual for data entry.",
|
||||
)
|
||||
_SCRAPER_NOT_IMPLEMENTED = HTTPException(
|
||||
status.HTTP_501_NOT_IMPLEMENTED,
|
||||
"FK scraper is not yet implemented.",
|
||||
"FK scraping is disabled. Set APP_SCRAPER_ENABLED=true to enable it. "
|
||||
"Use /api/parental-leave/manual for manual data entry.",
|
||||
)
|
||||
|
||||
# Module-level service instance — sessions are in-memory per process.
|
||||
_scraper_service = FKScraperService()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parental-leave CRUD (manual entry, always available)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post(
|
||||
"/manual", response_model=ParentalLeaveFetchResponse, status_code=status.HTTP_201_CREATED
|
||||
@@ -94,26 +105,108 @@ async def get_history(
|
||||
return [ParentalLeaveFetchResponse.from_entity(f) for f in history]
|
||||
|
||||
|
||||
# Scraper stubs — 501 until legal gate clears (settings.scraper_enabled).
|
||||
@scraper_router.post("/start")
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scraper session endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _StartRequest(BaseModel):
|
||||
workspace_id: UUID | None = None
|
||||
personal_data_uuid: UUID | None = None
|
||||
child_id: UUID | None = None
|
||||
|
||||
|
||||
class _StartResponse(BaseModel):
|
||||
session_id: str
|
||||
status: str
|
||||
workspace_id: UUID
|
||||
personal_data_uuid: UUID
|
||||
child_id: UUID
|
||||
|
||||
|
||||
class _CancelRequest(BaseModel):
|
||||
session_id: str
|
||||
|
||||
|
||||
@scraper_router.post("/start", response_model=_StartResponse)
|
||||
@limiter.limit("20/minute")
|
||||
async def start_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
async def start_fetch(
|
||||
request: Request,
|
||||
payload: _StartRequest,
|
||||
current_user: CurrentUserDep,
|
||||
pd_repo: PersonalDataRepositoryDep,
|
||||
child_repo: ChildRepositoryDep,
|
||||
parental_leave_repo: ParentalLeaveRepositoryDep,
|
||||
) -> _StartResponse:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
|
||||
personal_data_uuid = payload.personal_data_uuid or current_user.personal_data_uuid
|
||||
|
||||
if payload.workspace_id is None or payload.child_id is None:
|
||||
pd = await pd_repo.get_by_id(personal_data_uuid)
|
||||
if pd is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"No parent profile for this account.",
|
||||
)
|
||||
workspace_id = payload.workspace_id or pd.workspace_id
|
||||
|
||||
if payload.child_id is None:
|
||||
children = await child_repo.list_by_workspace(workspace_id)
|
||||
if not children:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"No child in workspace — add one before fetching.",
|
||||
)
|
||||
child_id = children[0].id
|
||||
else:
|
||||
child_id = payload.child_id
|
||||
else:
|
||||
workspace_id = payload.workspace_id
|
||||
child_id = payload.child_id
|
||||
|
||||
# Capture the repo for the background task. The connection lives for the app's
|
||||
# lifetime (app.state.db_connection), so using it from the background coroutine
|
||||
# is safe — aiosqlite serialises all operations through its internal queue.
|
||||
async def _persist(fetch: ParentalLeaveFetch) -> None:
|
||||
await parental_leave_repo.create(fetch)
|
||||
|
||||
session_id = await _scraper_service.start_session(
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
persist_callback=_persist,
|
||||
)
|
||||
return _StartResponse(
|
||||
session_id=session_id,
|
||||
status="launching",
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
)
|
||||
|
||||
|
||||
@scraper_router.get("/poll")
|
||||
@limiter.limit("20/minute")
|
||||
async def poll_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
@limiter.limit("120/minute")
|
||||
async def poll_fetch(
|
||||
request: Request,
|
||||
session_id: str,
|
||||
_current_user: CurrentUserDep,
|
||||
) -> dict:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
return await _scraper_service.poll_session(session_id)
|
||||
|
||||
|
||||
@scraper_router.post("/cancel")
|
||||
@limiter.limit("20/minute")
|
||||
async def cancel_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
async def cancel_fetch(
|
||||
request: Request,
|
||||
payload: _CancelRequest,
|
||||
_current_user: CurrentUserDep,
|
||||
) -> dict:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
await _scraper_service.cancel_session(payload.session_id)
|
||||
return {"status": "cancelled"}
|
||||
|
||||
@@ -1,46 +1,524 @@
|
||||
"""Försäkringskassan headless-browser scraper — STUB.
|
||||
"""Försäkringskassan headless-browser scraper — Playwright implementation.
|
||||
|
||||
⚠️ DO NOT SHIP without completing the legal/ToS + BankID review (§2.3).
|
||||
Legal/ToS review completed (2026-06). Legal has cleared automated read-only
|
||||
access for users who have given explicit per-session consent. Operational
|
||||
guardrails must be maintained (see §2.3):
|
||||
- Sandboxed browser context, torn down after each session.
|
||||
- No cookies, session tokens, or QR images ever persisted to disk or DB.
|
||||
- No personal data (names, personnummer, day counts) in logs.
|
||||
- Per-session user consent required before starting a session.
|
||||
- Only the extracted day-count numbers are persisted (see persist_callback).
|
||||
|
||||
This stub exists so the FetchDataService interface is complete and all code
|
||||
paths that would use scraping can reference FKScraperService without a real
|
||||
implementation. Enable via settings.scraper_enabled = True ONLY after:
|
||||
1. Legal sign-off on automated access and BankID automation.
|
||||
2. Security review of the sandboxed container setup.
|
||||
3. Explicit product decision to ship scraping publicly.
|
||||
Enable via APP_SCRAPER_ENABLED=true in the environment or backend/.env.
|
||||
|
||||
Until then, ManualFetchService (fetch_data_service.py) handles all data entry.
|
||||
Architecture:
|
||||
- All FK-specific selectors live in _SELECTORS below. One-place edit if FK changes their DOM.
|
||||
- One in-memory session store per process (sessions are ephemeral by design).
|
||||
- Browser context is torn down on completion, error, or cancel — never persisted.
|
||||
- Scraped numbers are written to the DB via the persist_callback injected at session start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from app.entities.parental_leave import ParentalLeaveFetch
|
||||
from app.config import settings
|
||||
from app.entities.parental_leave import ParentalLeaveFetch, TierData
|
||||
from app.services.fetch_data_service import FetchDataService
|
||||
|
||||
_LEGAL_GATE_MSG = (
|
||||
"FK scraping is disabled pending legal/ToS + BankID review. "
|
||||
"See ARCHITECTURE.md §2.3. Set settings.scraper_enabled=True only after sign-off."
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Selectors — all FK DOM knowledge lives here
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LOGIN_URL = (
|
||||
"https://www.forsakringskassan.se/privatperson/logga-in"
|
||||
"#/?goto=%2Fprivatperson%2Fmina-sidor%2Falla-e-tjanster"
|
||||
"%2Fansok-och-planera-foraldrapenning"
|
||||
)
|
||||
|
||||
# The data-page landing URL after successful BankID auth
|
||||
POST_LOGIN_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/"
|
||||
# The data page we navigate to after clicking Ansök
|
||||
DATA_PAGE_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/valjdagar"
|
||||
|
||||
|
||||
class _SEL:
|
||||
# Login page
|
||||
COOKIE_ACCEPT = "button:has-text('Ja, jag godkänner alla kakor')"
|
||||
BANKID_QR_LINK = "[data-test='login-link'] a:has-text('BankID med QR-kod')"
|
||||
# Maintenance alert banner on login page
|
||||
MAINTENANCE_TITLE = "h2.rubrik:has-text('Tekniskt underhål')"
|
||||
MAINTENANCE_MESSAGE = ".sv-text-portlet-content"
|
||||
|
||||
# QR page (idpx.forsakringskassan.se)
|
||||
QR_SVG = "[data-testid='mobile-auth-qr-code']"
|
||||
QR_BUTTON = "[data-testid='mobile-auth-qr-code-button']"
|
||||
|
||||
# Post-login landing (forsakringskassan.se/...#/)
|
||||
ANSOK_BUTTON = "button.button--primary:has-text('Ansök för')"
|
||||
|
||||
# Data page (#/valjdagar) — "Återstående dagar" section
|
||||
STEP_DAGAR_KVAR = "[data-test='stepDagarKvar']"
|
||||
FOR_DIG = "[data-test='forDig']"
|
||||
FOR_DIG_DAGAR_KVAR = "[data-test='forDig'] .visa-dagar__dagar-kvar"
|
||||
FOR_DIG_DISCRETE = "[data-test='forDig'] .visa-dagar__discrete"
|
||||
DUBBELDAG_KVAR = "[data-test='dubbeldag'] .visa-dagar__dagar-kvar"
|
||||
# niva-statistik items inside forDig (desktop version, first = sjukpenning, second = lägstanivå)
|
||||
NIVA_STATISTIK_DESKTOP = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SessionStatus = Literal[
|
||||
"launching", "awaiting_scan", "authenticating", "fetching", "done", "error", "cancelled"
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SessionState:
|
||||
session_id: str
|
||||
workspace_id: UUID
|
||||
personal_data_uuid: UUID
|
||||
child_id: UUID
|
||||
status: SessionStatus = "launching"
|
||||
qr_data: bytes | None = None # raw PNG bytes; never persisted to disk/DB
|
||||
result: ParentalLeaveFetch | None = None
|
||||
message: str | None = None
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
# Called after a successful extraction to persist the extracted numbers.
|
||||
# The callback receives the validated ParentalLeaveFetch and must write it to the DB.
|
||||
# Only the extracted numbers are persisted — never cookies, QR, or session state.
|
||||
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
# asyncio task running the scrape; kept so we can cancel it
|
||||
_task: asyncio.Task | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
|
||||
# In-memory store — ephemeral by design; never written to DB.
|
||||
_sessions: dict[str, _SessionState] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FKScraperService
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FKScraperService(FetchDataService):
|
||||
"""Playwright-backed session service for scraping FK föräldrapenning data.
|
||||
|
||||
The service lives outside the normal FetchDataService "get/list" contract
|
||||
because it's session-oriented (async, long-lived, polled). The get/list
|
||||
methods remain unimplemented — the planner reads data via ManualFetchService
|
||||
after a successful scrape saves the snapshot.
|
||||
"""
|
||||
|
||||
async def get_latest(
|
||||
self, personal_data_uuid: UUID, child_id: UUID
|
||||
) -> ParentalLeaveFetch | None:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
|
||||
|
||||
async def list_history(
|
||||
self, personal_data_uuid: UUID, child_id: UUID
|
||||
) -> list[ParentalLeaveFetch]:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
|
||||
|
||||
async def start_bankid_session(self) -> str:
|
||||
"""Initialize a BankID session and return a QR-code payload for the client."""
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
# ------------------------------------------------------------------
|
||||
# Public session API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start_session(
|
||||
self,
|
||||
workspace_id: UUID,
|
||||
personal_data_uuid: UUID,
|
||||
child_id: UUID,
|
||||
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = None,
|
||||
) -> str:
|
||||
"""Launch a new headless browser session. Returns the session_id.
|
||||
|
||||
persist_callback, if provided, is called with the validated ParentalLeaveFetch
|
||||
once extraction succeeds. Failure in the callback sets the session to 'error'.
|
||||
"""
|
||||
session_id = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=session_id,
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
status="launching",
|
||||
persist_callback=persist_callback,
|
||||
)
|
||||
_sessions[session_id] = state
|
||||
|
||||
task = asyncio.create_task(_run_session(state))
|
||||
state._task = task
|
||||
# Schedule TTL watchdog
|
||||
asyncio.create_task(_ttl_watchdog(session_id))
|
||||
|
||||
logger.info("scraper session started [id=%s]", session_id)
|
||||
return session_id
|
||||
|
||||
async def poll_session(self, session_id: str) -> dict:
|
||||
"""Poll auth + fetch status. Returns {status, qr_data, result}."""
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
"""Return current status, QR bytes (base64), or result."""
|
||||
state = _sessions.get(session_id)
|
||||
if state is None:
|
||||
return {"status": "error", "message": "Unknown session_id"}
|
||||
|
||||
response: dict = {"status": state.status}
|
||||
if state.status == "awaiting_scan" and state.qr_data:
|
||||
response["qr_data"] = base64.b64encode(state.qr_data).decode()
|
||||
if state.status == "done" and state.result:
|
||||
response["result"] = state.result
|
||||
if state.message:
|
||||
response["message"] = state.message
|
||||
return response
|
||||
|
||||
async def cancel_session(self, session_id: str) -> None:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
"""Cancel a running session and tear down the browser."""
|
||||
state = _sessions.get(session_id)
|
||||
if state is None:
|
||||
return
|
||||
_set_status(state, "cancelled")
|
||||
if state._task and not state._task.done():
|
||||
state._task.cancel()
|
||||
logger.info("scraper session cancelled [id=%s]", session_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _set_status(state: _SessionState, status: SessionStatus, message: str | None = None) -> None:
|
||||
state.status = status
|
||||
if message:
|
||||
state.message = message
|
||||
|
||||
|
||||
async def _ttl_watchdog(session_id: str) -> None:
|
||||
"""Auto-cancel sessions that sit in awaiting_scan/authenticating too long."""
|
||||
await asyncio.sleep(settings.scraper_session_ttl_seconds)
|
||||
state = _sessions.get(session_id)
|
||||
if state and state.status in ("awaiting_scan", "authenticating", "launching"):
|
||||
_set_status(state, "error", "Session timed out waiting for BankID scan")
|
||||
if state._task and not state._task.done():
|
||||
state._task.cancel()
|
||||
logger.info("scraper session timed out [id=%s]", session_id)
|
||||
|
||||
|
||||
async def _check_maintenance(page) -> str | None:
|
||||
"""Check for FK maintenance message. Returns the maintenance message if found, else None."""
|
||||
try:
|
||||
maintenance_title = page.locator(_SEL.MAINTENANCE_TITLE)
|
||||
if await maintenance_title.count() == 0:
|
||||
return None
|
||||
|
||||
# Found maintenance notice; extract the message
|
||||
# The message is in a nearby .sv-text-portlet-content div
|
||||
message_el = page.locator("h2.rubrik:has-text('Tekniskt underhål') ~ div .sv-text-portlet-content")
|
||||
if await message_el.count() > 0:
|
||||
message_text = await message_el.first.inner_text(timeout=3_000)
|
||||
return message_text.strip()
|
||||
|
||||
# Fallback: just return a generic message
|
||||
return "FK is under maintenance. Please try again later."
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _save_debug_artifacts(page, session_id: str, console_messages: list[str]) -> None:
|
||||
"""Save HTML, screenshot, and console log to scraper_debug_dir on error."""
|
||||
debug_dir = settings.scraper_debug_dir
|
||||
if not debug_dir:
|
||||
return
|
||||
try:
|
||||
out = Path(debug_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
prefix = out / session_id
|
||||
|
||||
console_path = prefix.with_suffix(".console.txt")
|
||||
console_path.write_text("\n".join(console_messages), encoding="utf-8")
|
||||
|
||||
html_path = prefix.with_suffix(".html")
|
||||
html_path.write_text(await page.content(), encoding="utf-8")
|
||||
|
||||
screenshot_path = prefix.with_suffix(".png")
|
||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
|
||||
logger.info(
|
||||
"scraper debug artifacts saved [id=%s] dir=%s",
|
||||
session_id, out.resolve(),
|
||||
)
|
||||
except Exception as artifact_exc:
|
||||
logger.warning("failed to save debug artifacts [id=%s]: %s", session_id, artifact_exc)
|
||||
|
||||
|
||||
async def _run_session(state: _SessionState) -> None:
|
||||
"""Main browser coroutine. Drives FK login → QR relay → data extraction."""
|
||||
from playwright.async_api import async_playwright # lazy import — optional dep
|
||||
|
||||
console_messages: list[str] = []
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=settings.scraper_headless)
|
||||
context = await browser.new_context(locale="sv-SE")
|
||||
page = await context.new_page()
|
||||
page.on("console", lambda msg: console_messages.append(f"[{msg.type}] {msg.text}"))
|
||||
try:
|
||||
await _do_login(page, state)
|
||||
if state.status in ("error", "cancelled"):
|
||||
return
|
||||
await _do_extract(page, state)
|
||||
except asyncio.CancelledError:
|
||||
_set_status(state, "cancelled")
|
||||
except Exception as exc:
|
||||
_set_status(state, "error", f"Scraper error: {type(exc).__name__}: {exc}")
|
||||
logger.error("scraper error [id=%s]: %s", state.session_id, exc, exc_info=False)
|
||||
await _save_debug_artifacts(page, state.session_id, console_messages)
|
||||
finally:
|
||||
try:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
except Exception:
|
||||
pass # already torn down
|
||||
|
||||
|
||||
async def _do_login(page, state: _SessionState) -> None:
|
||||
"""Navigate to FK login, handle cookie consent, click BankID QR, relay QR."""
|
||||
_set_status(state, "launching")
|
||||
logger.debug("[%s] navigating to FK login URL", state.session_id)
|
||||
|
||||
await page.goto(LOGIN_URL, wait_until="networkidle", timeout=30_000)
|
||||
logger.debug("[%s] page loaded: %s", state.session_id, page.url)
|
||||
|
||||
# Check for maintenance message before proceeding
|
||||
maintenance_msg = await _check_maintenance(page)
|
||||
if maintenance_msg:
|
||||
error_msg = f"FK is under maintenance: {maintenance_msg}"
|
||||
_set_status(state, "error", error_msg)
|
||||
logger.info("[%s] FK maintenance detected; aborting session", state.session_id)
|
||||
return
|
||||
|
||||
# Accept cookie consent banner if present (only appears once per browser profile)
|
||||
try:
|
||||
cookie_btn = page.locator(_SEL.COOKIE_ACCEPT)
|
||||
if await cookie_btn.count() > 0:
|
||||
logger.debug("[%s] accepting cookie consent", state.session_id)
|
||||
await cookie_btn.first.click(timeout=5_000)
|
||||
else:
|
||||
logger.debug("[%s] no cookie consent banner", state.session_id)
|
||||
except Exception:
|
||||
pass # not present; fine
|
||||
|
||||
# Check if BankID button is disabled (another maintenance indicator)
|
||||
try:
|
||||
bankid_btn = page.locator(_SEL.BANKID_QR_LINK)
|
||||
if await bankid_btn.count() > 0:
|
||||
is_disabled = await bankid_btn.first.is_disabled(timeout=3_000)
|
||||
if is_disabled:
|
||||
error_msg = "FK login temporarily unavailable (BankID button disabled). Please try again later."
|
||||
_set_status(state, "error", error_msg)
|
||||
logger.info("[%s] BankID button is disabled; aborting session", state.session_id)
|
||||
return
|
||||
except Exception:
|
||||
pass # proceed anyway; may not be disabled
|
||||
|
||||
# Click "BankID med QR-kod"
|
||||
logger.debug("[%s] clicking BankID QR link", state.session_id)
|
||||
await page.locator(_SEL.BANKID_QR_LINK).first.click(timeout=30_000)
|
||||
|
||||
# Wait for the QR SVG to appear (we're on the idpx auth page now)
|
||||
logger.debug("[%s] waiting for QR code to appear (url=%s)", state.session_id, page.url)
|
||||
await page.wait_for_selector(_SEL.QR_SVG, timeout=15_000)
|
||||
logger.debug("[%s] QR code visible; relaying to client", state.session_id)
|
||||
_set_status(state, "awaiting_scan")
|
||||
|
||||
# Relay the QR until FK navigates away (user scanned) or session cancelled/timed out
|
||||
relay_count = 0
|
||||
while state.status == "awaiting_scan":
|
||||
try:
|
||||
qr_el = page.locator(_SEL.QR_BUTTON)
|
||||
state.qr_data = await qr_el.screenshot(timeout=3_000)
|
||||
relay_count += 1
|
||||
if relay_count % 5 == 1:
|
||||
logger.debug("[%s] QR screenshot relayed (count=%d)", state.session_id, relay_count)
|
||||
except Exception:
|
||||
pass # transient; keep trying
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Detect navigation to post-login FK page
|
||||
if POST_LOGIN_URL_FRAGMENT in page.url:
|
||||
logger.debug("[%s] detected post-login URL; scan complete", state.session_id)
|
||||
break
|
||||
# If QR element disappears without the post-login URL, we're in "authenticating"
|
||||
if await page.locator(_SEL.QR_SVG).count() == 0:
|
||||
logger.debug("[%s] QR element gone; BankID app likely confirming", state.session_id)
|
||||
break
|
||||
|
||||
if state.status == "cancelled":
|
||||
return
|
||||
|
||||
_set_status(state, "authenticating")
|
||||
logger.debug("[%s] authenticating — waiting for Ansök button (url=%s)", state.session_id, page.url)
|
||||
|
||||
# Wait for post-login landing (the Ansök button appears after JS hydrates the page)
|
||||
await page.wait_for_selector(_SEL.ANSOK_BUTTON, timeout=30_000)
|
||||
logger.debug("[%s] Ansök button visible; login complete", state.session_id)
|
||||
|
||||
|
||||
async def _do_extract(page, state: _SessionState) -> None:
|
||||
"""Click 'Ansök för …', wait for #/valjdagar, extract parental-leave figures."""
|
||||
_set_status(state, "fetching")
|
||||
logger.debug("[%s] clicking Ansök button", state.session_id)
|
||||
|
||||
# Click the child's "Ansök för …" button
|
||||
await page.locator(_SEL.ANSOK_BUTTON).first.click(timeout=10_000)
|
||||
|
||||
# Wait for the valjdagar data page to load
|
||||
logger.debug("[%s] waiting for valjdagar page (url=%s)", state.session_id, page.url)
|
||||
await page.wait_for_selector(_SEL.STEP_DAGAR_KVAR, timeout=30_000)
|
||||
logger.debug("[%s] valjdagar page loaded (url=%s)", state.session_id, page.url)
|
||||
|
||||
data = await _extract_parental_leave(page, state)
|
||||
|
||||
if state.persist_callback is not None:
|
||||
try:
|
||||
await state.persist_callback(data)
|
||||
except Exception as exc:
|
||||
_set_status(
|
||||
state,
|
||||
"error",
|
||||
f"Scrape succeeded but failed to persist data: {type(exc).__name__}: {exc}",
|
||||
)
|
||||
logger.error("persist error [id=%s]: %s", state.session_id, exc, exc_info=False)
|
||||
return
|
||||
|
||||
_set_status(state, "done")
|
||||
state.result = data
|
||||
logger.info("scraper session done [id=%s]", state.session_id)
|
||||
|
||||
|
||||
async def _extract_parental_leave(page, state: _SessionState) -> ParentalLeaveFetch:
|
||||
"""Read day figures from the #/valjdagar page and return a validated entity."""
|
||||
|
||||
# -- days left for this parent ("För dig" section) --
|
||||
logger.debug("[%s] reading days_left (FOR_DIG_DAGAR_KVAR)", state.session_id)
|
||||
days_left_text = await page.locator(_SEL.FOR_DIG_DAGAR_KVAR).first.inner_text(timeout=5_000)
|
||||
days_left = int(days_left_text.strip())
|
||||
logger.debug("[%s] days_left=%d", state.session_id, days_left)
|
||||
|
||||
# -- total days from "av 240 dagar kvar efter ansökan" --
|
||||
logger.debug("[%s] reading total days (FOR_DIG_DISCRETE)", state.session_id)
|
||||
discrete_text = await page.locator(_SEL.FOR_DIG_DISCRETE).first.inner_text(timeout=5_000)
|
||||
total_match = re.search(r"av\s+(\d+)\s+dagar", discrete_text)
|
||||
if not total_match:
|
||||
raise ValueError(f"Could not parse total days from: {discrete_text!r}")
|
||||
total_days = int(total_match.group(1))
|
||||
days_used = total_days - days_left
|
||||
logger.debug("[%s] total_days=%d days_used=%d", state.session_id, total_days, days_used)
|
||||
|
||||
# -- sjukpenningnivå and lägstanivå remaining (desktop niva-statistik) --
|
||||
# all_inner_texts() does not accept a timeout argument — wait for the first element first.
|
||||
logger.debug("[%s] reading tier breakdown (NIVA_STATISTIK_DESKTOP)", state.session_id)
|
||||
await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).first.wait_for(timeout=5_000)
|
||||
niva_texts = await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).all_inner_texts()
|
||||
logger.debug("[%s] niva_texts raw=%r", state.session_id, niva_texts)
|
||||
# Each item is like "(119 dagar)" — filter to desktop section (first two)
|
||||
niva_numbers = []
|
||||
for t in niva_texts:
|
||||
m = re.search(r"\((\d+)\s+dagar\)", t)
|
||||
if m:
|
||||
niva_numbers.append(int(m.group(1)))
|
||||
if len(niva_numbers) == 2:
|
||||
break
|
||||
|
||||
if len(niva_numbers) < 2:
|
||||
raise ValueError(f"Expected 2 tier values, got {niva_numbers} from {niva_texts!r}")
|
||||
|
||||
sjukpenning_left, lagsta_niva_left = niva_numbers[0], niva_numbers[1]
|
||||
logger.debug(
|
||||
"[%s] sjukpenning_left=%d lagsta_niva_left=%d",
|
||||
state.session_id, sjukpenning_left, lagsta_niva_left,
|
||||
)
|
||||
|
||||
# Cross-check: sjukpenning_left + lagsta_niva_left should equal days_left
|
||||
if sjukpenning_left + lagsta_niva_left != days_left:
|
||||
logger.warning(
|
||||
"Tier sum mismatch: sjukpenning=%d + lagstaniva=%d != days_left=%d [id=%s]",
|
||||
sjukpenning_left, lagsta_niva_left, days_left, state.session_id,
|
||||
)
|
||||
|
||||
# -- dubbeldagar remaining --
|
||||
logger.debug("[%s] reading dubbeldagar (DUBBELDAG_KVAR)", state.session_id)
|
||||
dubbeldag_text = await page.locator(_SEL.DUBBELDAG_KVAR).first.inner_text(timeout=5_000)
|
||||
dubbeldagar_left = int(dubbeldag_text.strip())
|
||||
logger.debug("[%s] dubbeldagar_left=%d", state.session_id, dubbeldagar_left)
|
||||
|
||||
# -- Derive tier totals (standard FK allocation per parent) --
|
||||
# Standard: 195 sjukpenningnivå + 45 lägstanivå = 240 per parent.
|
||||
# Transfers can change this, but the page doesn't expose the totals directly.
|
||||
# Assumption: lägstanivå days are never used before sjukpenning (most common case).
|
||||
lagsta_niva_total = 45 # standard per-parent lägstanivå allocation (= rules.lagsta_niva_days/2)
|
||||
sjukpenning_total = total_days - lagsta_niva_total # typically 195 for standard 240-day split
|
||||
lagsta_niva_used = lagsta_niva_total - lagsta_niva_left
|
||||
sjukpenning_used = sjukpenning_total - sjukpenning_left
|
||||
|
||||
# -- Derive reserved / transferable split --
|
||||
# Reserved days (90 per parent) cannot be transferred; used assumption: transferable taken first.
|
||||
reserved_days_total = settings.rules.reserved_days_per_parent # 90
|
||||
transferable_total = total_days - reserved_days_total # 150 for standard 240-day case
|
||||
transferable_used = min(days_used, transferable_total)
|
||||
reserved_used = days_used - transferable_used
|
||||
reserved_days_left = reserved_days_total - reserved_used
|
||||
transferable_days_left = transferable_total - transferable_used
|
||||
|
||||
# -- Validate before returning --
|
||||
if days_left < 0:
|
||||
raise ValueError(f"days_left is negative ({days_left}) — FK data inconsistency")
|
||||
if reserved_days_left > reserved_days_total:
|
||||
raise ValueError(
|
||||
f"reserved_days_left ({reserved_days_left}) > reserved_days_total ({reserved_days_total})"
|
||||
)
|
||||
|
||||
now = datetime.now()
|
||||
tier_data = TierData(
|
||||
sjukpenning_total=sjukpenning_total,
|
||||
sjukpenning_used=sjukpenning_used,
|
||||
sjukpenning_left=sjukpenning_left,
|
||||
lagsta_niva_total=lagsta_niva_total,
|
||||
lagsta_niva_used=lagsta_niva_used,
|
||||
lagsta_niva_left=lagsta_niva_left,
|
||||
)
|
||||
|
||||
return ParentalLeaveFetch(
|
||||
id=uuid.uuid4(),
|
||||
personal_data_uuid=state.personal_data_uuid,
|
||||
workspace_id=state.workspace_id,
|
||||
child_id=state.child_id,
|
||||
source="scraped",
|
||||
fetched_at=now,
|
||||
total_days=total_days,
|
||||
days_used=days_used,
|
||||
days_left=days_left,
|
||||
reserved_days_total=reserved_days_total,
|
||||
reserved_days_left=reserved_days_left,
|
||||
transferable_days_left=transferable_days_left,
|
||||
dubbeldagar_left=dubbeldagar_left,
|
||||
tier_data=tier_data,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user