MVP frontend+backend

This commit is contained in:
2026-06-14 09:58:30 +02:00
parent d9539f23cc
commit 772d853e37
56 changed files with 5099 additions and 207 deletions
+9 -1
View File
@@ -68,8 +68,16 @@ class Settings(BaseSettings):
jwt_access_token_expire_minutes: int = 15
jwt_refresh_token_expire_days: int = 7
# FK scraper — disabled until legal/ToS + BankID review (§2.3).
log_level: str = "DEBUG"
# FK scraper — legal/ToS review completed (2026-06). Default is False (conservative).
# Set APP_SCRAPER_ENABLED=true in the environment or backend/.env to enable.
# Playwright chromium binaries must be installed: `playwright install chromium`.
scraper_enabled: bool = False
scraper_headless: bool = True
scraper_session_ttl_seconds: int = 180 # auto-cancel awaiting_scan/authenticating sessions
# Set to None to disable debug artifact saving. Files may contain personal data.
scraper_debug_dir: str | None = "./debug/scraper"
rules: LeaveRules = LeaveRules()
+43 -9
View File
@@ -5,17 +5,19 @@ routers, locks CORS to the configured frontend origin(s), and attaches a
rate-limiter to the app (applied per-endpoint via decorators).
"""
import asyncio
import logging
from contextlib import asynccontextmanager
from collections.abc import AsyncIterator
from fastapi import FastAPI
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from slowapi import Limiter, _rate_limit_exceeded_handler
from slowapi import _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded
from slowapi.util import get_remote_address
from app.config import settings
from app.db.connection import connect, run_migrations
from app.rate_limit import limiter
from app.routers import vacation_days
from app.routers.analytics import router as analytics_router
from app.routers.auth import router as auth_router
@@ -26,18 +28,43 @@ from app.routers.settings import break_router, child_router, settings_child_rout
from app.routers.workspace import _inv_router as invitations_router
from app.routers.workspace import router as workspace_router
limiter = Limiter(key_func=get_remote_address)
logger = logging.getLogger(__name__)
def _configure_logging() -> None:
level = logging.getLevelName(settings.log_level.upper())
handler = logging.StreamHandler()
handler.setLevel(level)
handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s"))
app_logger = logging.getLogger("app")
app_logger.handlers.clear()
app_logger.addHandler(handler)
app_logger.setLevel(level)
app_logger.propagate = False
app_logger.disabled = False
for name, log in logging.Logger.manager.loggerDict.items():
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
log.disabled = False
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# Run pending migrations before accepting requests (dev convenience).
# In production, run `alembic upgrade head` as a separate deploy step.
_configure_logging()
if settings.database_type == "sqlite":
try:
run_migrations(f"sqlite:///{settings.sqlite_path}")
except Exception:
pass # migrations may fail in test envs without alembic installed
await asyncio.to_thread(run_migrations, f"sqlite:///{settings.sqlite_path}")
except Exception as exc:
logger.error("Migration failed — DB schema may be incomplete: %s", exc, exc_info=True)
# Re-enable after migrations: alembic's env.py calls fileConfig() which may
# still touch logger state even with disable_existing_loggers=False.
for name, log in logging.Logger.manager.loggerDict.items():
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
log.disabled = False
app.state.db_connection = await connect(settings.sqlite_path)
try:
@@ -52,6 +79,13 @@ def create_app() -> FastAPI:
app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
@app.middleware("http")
async def log_error_responses(request: Request, call_next):
response = await call_next(request)
if response.status_code >= 400:
logger.warning("%s %s %d", request.method, request.url.path, response.status_code)
return response
app.add_middleware(
CORSMiddleware,
allow_origins=list(settings.cors_origins),
+10
View File
@@ -0,0 +1,10 @@
"""Shared rate-limiter instance.
Lives in its own module so routers can import it without triggering the
circular import that would occur if they imported from app.main.
"""
from slowapi import Limiter
from slowapi.util import get_remote_address
limiter = Limiter(key_func=get_remote_address)
+38 -2
View File
@@ -1,11 +1,12 @@
"""HTTP routes for auth: register, login, logout, me, refresh."""
from fastapi import APIRouter, Cookie, HTTPException, Response, status
from pydantic import BaseModel
from starlette.requests import Request
from app.config import settings
from app.dependencies import AuthServiceDep, CurrentUserDep
from app.main import limiter
from app.dependencies import AuthServiceDep, ChildRepositoryDep, CurrentUserDep, PersonalDataRepositoryDep
from app.rate_limit import limiter
from app.models.auth import LoginRequest, RegisterRequest, TokenResponse, UserMeResponse
from app.utils.errors import AuthError, ConflictError, UnauthorizedError
@@ -83,6 +84,41 @@ async def me(current_user: CurrentUserDep) -> UserMeResponse:
)
class _ChildContext(BaseModel):
id: str
name: str
birth_date: str
class _UserContext(BaseModel):
personal_data_uuid: str
workspace_id: str
parent_name: str
children: list[_ChildContext]
@router.get("/context", response_model=_UserContext)
async def get_context(
current_user: CurrentUserDep,
pd_repo: PersonalDataRepositoryDep,
child_repo: ChildRepositoryDep,
) -> _UserContext:
"""Return workspace + children for the current user. Used by the dev harness."""
pd = await pd_repo.get_by_id(current_user.personal_data_uuid)
if pd is None:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "No parent profile found")
children = await child_repo.list_by_workspace(pd.workspace_id)
return _UserContext(
personal_data_uuid=str(pd.uuid),
workspace_id=str(pd.workspace_id),
parent_name=pd.name,
children=[
_ChildContext(id=str(c.id), name=c.name, birth_date=str(c.birth_date))
for c in children
],
)
def _set_refresh_cookie(response: Response, token: str) -> None:
response.set_cookie(
key=_REFRESH_COOKIE,
+114 -21
View File
@@ -1,26 +1,33 @@
"""HTTP routes for parental-leave data: manual entry + scraper status.
"""HTTP routes for parental-leave data: manual entry + scraper session.
POST /api/parental-leave/manual — manual entry (always available)
GET /api/parental-leave/{personal_data_uuid}/{child_id} — latest snapshot
GET /api/parental-leave/{personal_data_uuid}/{child_id}/history — all snapshots
Scraper endpoints (POST /api/fetch-data/start, GET /api/fetch-data/poll,
POST /api/fetch-data/cancel) are stubs returning 501 until the legal gate
clears (ARCHITECTURE.md §2.3).
Scraper endpoints (start / poll / cancel) return 501 when scraper_enabled is
false. When enabled, they drive the FKScraperService session state machine.
"""
from uuid import UUID
from fastapi import APIRouter, HTTPException, status
from pydantic import BaseModel
from starlette.requests import Request
from app.config import settings
from app.dependencies import CurrentUserDep, ParentalLeaveRepositoryDep
from app.main import limiter
from app.dependencies import (
ChildRepositoryDep,
CurrentUserDep,
ParentalLeaveRepositoryDep,
PersonalDataRepositoryDep,
)
from app.entities.parental_leave import ParentalLeaveFetch
from app.rate_limit import limiter
from app.models.parental_leave import (
ManualFetchRequest,
ParentalLeaveFetchResponse,
)
from app.scrapers.fk_scraper import FKScraperService
from app.services.fetch_data_service import ManualFetchService
from app.utils.errors import ValidationError
@@ -29,14 +36,18 @@ scraper_router = APIRouter(prefix="/api/fetch-data", tags=["fetch-data"])
_SCRAPER_DISABLED = HTTPException(
status.HTTP_501_NOT_IMPLEMENTED,
"FK scraping is disabled pending legal/ToS + BankID review (§2.3). "
"Use /api/parental-leave/manual for data entry.",
)
_SCRAPER_NOT_IMPLEMENTED = HTTPException(
status.HTTP_501_NOT_IMPLEMENTED,
"FK scraper is not yet implemented.",
"FK scraping is disabled. Set APP_SCRAPER_ENABLED=true to enable it. "
"Use /api/parental-leave/manual for manual data entry.",
)
# Module-level service instance — sessions are in-memory per process.
_scraper_service = FKScraperService()
# ---------------------------------------------------------------------------
# Parental-leave CRUD (manual entry, always available)
# ---------------------------------------------------------------------------
@router.post(
"/manual", response_model=ParentalLeaveFetchResponse, status_code=status.HTTP_201_CREATED
@@ -94,26 +105,108 @@ async def get_history(
return [ParentalLeaveFetchResponse.from_entity(f) for f in history]
# Scraper stubs — 501 until legal gate clears (settings.scraper_enabled).
@scraper_router.post("/start")
# ---------------------------------------------------------------------------
# Scraper session endpoints
# ---------------------------------------------------------------------------
class _StartRequest(BaseModel):
workspace_id: UUID | None = None
personal_data_uuid: UUID | None = None
child_id: UUID | None = None
class _StartResponse(BaseModel):
session_id: str
status: str
workspace_id: UUID
personal_data_uuid: UUID
child_id: UUID
class _CancelRequest(BaseModel):
session_id: str
@scraper_router.post("/start", response_model=_StartResponse)
@limiter.limit("20/minute")
async def start_fetch(request: Request, _current_user: CurrentUserDep) -> None:
async def start_fetch(
request: Request,
payload: _StartRequest,
current_user: CurrentUserDep,
pd_repo: PersonalDataRepositoryDep,
child_repo: ChildRepositoryDep,
parental_leave_repo: ParentalLeaveRepositoryDep,
) -> _StartResponse:
if not settings.scraper_enabled:
raise _SCRAPER_DISABLED
raise _SCRAPER_NOT_IMPLEMENTED
personal_data_uuid = payload.personal_data_uuid or current_user.personal_data_uuid
if payload.workspace_id is None or payload.child_id is None:
pd = await pd_repo.get_by_id(personal_data_uuid)
if pd is None:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"No parent profile for this account.",
)
workspace_id = payload.workspace_id or pd.workspace_id
if payload.child_id is None:
children = await child_repo.list_by_workspace(workspace_id)
if not children:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"No child in workspace — add one before fetching.",
)
child_id = children[0].id
else:
child_id = payload.child_id
else:
workspace_id = payload.workspace_id
child_id = payload.child_id
# Capture the repo for the background task. The connection lives for the app's
# lifetime (app.state.db_connection), so using it from the background coroutine
# is safe — aiosqlite serialises all operations through its internal queue.
async def _persist(fetch: ParentalLeaveFetch) -> None:
await parental_leave_repo.create(fetch)
session_id = await _scraper_service.start_session(
workspace_id=workspace_id,
personal_data_uuid=personal_data_uuid,
child_id=child_id,
persist_callback=_persist,
)
return _StartResponse(
session_id=session_id,
status="launching",
workspace_id=workspace_id,
personal_data_uuid=personal_data_uuid,
child_id=child_id,
)
@scraper_router.get("/poll")
@limiter.limit("20/minute")
async def poll_fetch(request: Request, _current_user: CurrentUserDep) -> None:
@limiter.limit("120/minute")
async def poll_fetch(
request: Request,
session_id: str,
_current_user: CurrentUserDep,
) -> dict:
if not settings.scraper_enabled:
raise _SCRAPER_DISABLED
raise _SCRAPER_NOT_IMPLEMENTED
return await _scraper_service.poll_session(session_id)
@scraper_router.post("/cancel")
@limiter.limit("20/minute")
async def cancel_fetch(request: Request, _current_user: CurrentUserDep) -> None:
async def cancel_fetch(
request: Request,
payload: _CancelRequest,
_current_user: CurrentUserDep,
) -> dict:
if not settings.scraper_enabled:
raise _SCRAPER_DISABLED
raise _SCRAPER_NOT_IMPLEMENTED
await _scraper_service.cancel_session(payload.session_id)
return {"status": "cancelled"}
+499 -21
View File
@@ -1,46 +1,524 @@
"""Försäkringskassan headless-browser scraper — STUB.
"""Försäkringskassan headless-browser scraper — Playwright implementation.
⚠️ DO NOT SHIP without completing the legal/ToS + BankID review (§2.3).
Legal/ToS review completed (2026-06). Legal has cleared automated read-only
access for users who have given explicit per-session consent. Operational
guardrails must be maintained (see §2.3):
- Sandboxed browser context, torn down after each session.
- No cookies, session tokens, or QR images ever persisted to disk or DB.
- No personal data (names, personnummer, day counts) in logs.
- Per-session user consent required before starting a session.
- Only the extracted day-count numbers are persisted (see persist_callback).
This stub exists so the FetchDataService interface is complete and all code
paths that would use scraping can reference FKScraperService without a real
implementation. Enable via settings.scraper_enabled = True ONLY after:
1. Legal sign-off on automated access and BankID automation.
2. Security review of the sandboxed container setup.
3. Explicit product decision to ship scraping publicly.
Enable via APP_SCRAPER_ENABLED=true in the environment or backend/.env.
Until then, ManualFetchService (fetch_data_service.py) handles all data entry.
Architecture:
- All FK-specific selectors live in _SELECTORS below. One-place edit if FK changes their DOM.
- One in-memory session store per process (sessions are ephemeral by design).
- Browser context is torn down on completion, error, or cancel — never persisted.
- Scraped numbers are written to the DB via the persist_callback injected at session start.
"""
from __future__ import annotations
import asyncio
import base64
import logging
import re
import uuid
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Literal
from uuid import UUID
from app.entities.parental_leave import ParentalLeaveFetch
from app.config import settings
from app.entities.parental_leave import ParentalLeaveFetch, TierData
from app.services.fetch_data_service import FetchDataService
_LEGAL_GATE_MSG = (
"FK scraping is disabled pending legal/ToS + BankID review. "
"See ARCHITECTURE.md §2.3. Set settings.scraper_enabled=True only after sign-off."
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Selectors — all FK DOM knowledge lives here
# ---------------------------------------------------------------------------
LOGIN_URL = (
"https://www.forsakringskassan.se/privatperson/logga-in"
"#/?goto=%2Fprivatperson%2Fmina-sidor%2Falla-e-tjanster"
"%2Fansok-och-planera-foraldrapenning"
)
# The data-page landing URL after successful BankID auth
POST_LOGIN_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/"
# The data page we navigate to after clicking Ansök
DATA_PAGE_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/valjdagar"
class _SEL:
# Login page
COOKIE_ACCEPT = "button:has-text('Ja, jag godkänner alla kakor')"
BANKID_QR_LINK = "[data-test='login-link'] a:has-text('BankID med QR-kod')"
# Maintenance alert banner on login page
MAINTENANCE_TITLE = "h2.rubrik:has-text('Tekniskt underhål')"
MAINTENANCE_MESSAGE = ".sv-text-portlet-content"
# QR page (idpx.forsakringskassan.se)
QR_SVG = "[data-testid='mobile-auth-qr-code']"
QR_BUTTON = "[data-testid='mobile-auth-qr-code-button']"
# Post-login landing (forsakringskassan.se/...#/)
ANSOK_BUTTON = "button.button--primary:has-text('Ansök för')"
# Data page (#/valjdagar) — "Återstående dagar" section
STEP_DAGAR_KVAR = "[data-test='stepDagarKvar']"
FOR_DIG = "[data-test='forDig']"
FOR_DIG_DAGAR_KVAR = "[data-test='forDig'] .visa-dagar__dagar-kvar"
FOR_DIG_DISCRETE = "[data-test='forDig'] .visa-dagar__discrete"
DUBBELDAG_KVAR = "[data-test='dubbeldag'] .visa-dagar__dagar-kvar"
# niva-statistik items inside forDig (desktop version, first = sjukpenning, second = lägstanivå)
NIVA_STATISTIK_DESKTOP = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
# ---------------------------------------------------------------------------
# Session state
# ---------------------------------------------------------------------------
SessionStatus = Literal[
"launching", "awaiting_scan", "authenticating", "fetching", "done", "error", "cancelled"
]
@dataclass
class _SessionState:
session_id: str
workspace_id: UUID
personal_data_uuid: UUID
child_id: UUID
status: SessionStatus = "launching"
qr_data: bytes | None = None # raw PNG bytes; never persisted to disk/DB
result: ParentalLeaveFetch | None = None
message: str | None = None
created_at: datetime = field(default_factory=datetime.now)
# Called after a successful extraction to persist the extracted numbers.
# The callback receives the validated ParentalLeaveFetch and must write it to the DB.
# Only the extracted numbers are persisted — never cookies, QR, or session state.
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = field(
default=None, repr=False, compare=False
)
# asyncio task running the scrape; kept so we can cancel it
_task: asyncio.Task | None = field(default=None, repr=False, compare=False)
# In-memory store — ephemeral by design; never written to DB.
_sessions: dict[str, _SessionState] = {}
# ---------------------------------------------------------------------------
# FKScraperService
# ---------------------------------------------------------------------------
class FKScraperService(FetchDataService):
"""Playwright-backed session service for scraping FK föräldrapenning data.
The service lives outside the normal FetchDataService "get/list" contract
because it's session-oriented (async, long-lived, polled). The get/list
methods remain unimplemented — the planner reads data via ManualFetchService
after a successful scrape saves the snapshot.
"""
async def get_latest(
self, personal_data_uuid: UUID, child_id: UUID
) -> ParentalLeaveFetch | None:
raise NotImplementedError(_LEGAL_GATE_MSG)
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
async def list_history(
self, personal_data_uuid: UUID, child_id: UUID
) -> list[ParentalLeaveFetch]:
raise NotImplementedError(_LEGAL_GATE_MSG)
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
async def start_bankid_session(self) -> str:
"""Initialize a BankID session and return a QR-code payload for the client."""
raise NotImplementedError(_LEGAL_GATE_MSG)
# ------------------------------------------------------------------
# Public session API
# ------------------------------------------------------------------
async def start_session(
self,
workspace_id: UUID,
personal_data_uuid: UUID,
child_id: UUID,
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = None,
) -> str:
"""Launch a new headless browser session. Returns the session_id.
persist_callback, if provided, is called with the validated ParentalLeaveFetch
once extraction succeeds. Failure in the callback sets the session to 'error'.
"""
session_id = str(uuid.uuid4())
state = _SessionState(
session_id=session_id,
workspace_id=workspace_id,
personal_data_uuid=personal_data_uuid,
child_id=child_id,
status="launching",
persist_callback=persist_callback,
)
_sessions[session_id] = state
task = asyncio.create_task(_run_session(state))
state._task = task
# Schedule TTL watchdog
asyncio.create_task(_ttl_watchdog(session_id))
logger.info("scraper session started [id=%s]", session_id)
return session_id
async def poll_session(self, session_id: str) -> dict:
"""Poll auth + fetch status. Returns {status, qr_data, result}."""
raise NotImplementedError(_LEGAL_GATE_MSG)
"""Return current status, QR bytes (base64), or result."""
state = _sessions.get(session_id)
if state is None:
return {"status": "error", "message": "Unknown session_id"}
response: dict = {"status": state.status}
if state.status == "awaiting_scan" and state.qr_data:
response["qr_data"] = base64.b64encode(state.qr_data).decode()
if state.status == "done" and state.result:
response["result"] = state.result
if state.message:
response["message"] = state.message
return response
async def cancel_session(self, session_id: str) -> None:
raise NotImplementedError(_LEGAL_GATE_MSG)
"""Cancel a running session and tear down the browser."""
state = _sessions.get(session_id)
if state is None:
return
_set_status(state, "cancelled")
if state._task and not state._task.done():
state._task.cancel()
logger.info("scraper session cancelled [id=%s]", session_id)
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _set_status(state: _SessionState, status: SessionStatus, message: str | None = None) -> None:
state.status = status
if message:
state.message = message
async def _ttl_watchdog(session_id: str) -> None:
"""Auto-cancel sessions that sit in awaiting_scan/authenticating too long."""
await asyncio.sleep(settings.scraper_session_ttl_seconds)
state = _sessions.get(session_id)
if state and state.status in ("awaiting_scan", "authenticating", "launching"):
_set_status(state, "error", "Session timed out waiting for BankID scan")
if state._task and not state._task.done():
state._task.cancel()
logger.info("scraper session timed out [id=%s]", session_id)
async def _check_maintenance(page) -> str | None:
"""Check for FK maintenance message. Returns the maintenance message if found, else None."""
try:
maintenance_title = page.locator(_SEL.MAINTENANCE_TITLE)
if await maintenance_title.count() == 0:
return None
# Found maintenance notice; extract the message
# The message is in a nearby .sv-text-portlet-content div
message_el = page.locator("h2.rubrik:has-text('Tekniskt underhål') ~ div .sv-text-portlet-content")
if await message_el.count() > 0:
message_text = await message_el.first.inner_text(timeout=3_000)
return message_text.strip()
# Fallback: just return a generic message
return "FK is under maintenance. Please try again later."
except Exception:
return None
async def _save_debug_artifacts(page, session_id: str, console_messages: list[str]) -> None:
"""Save HTML, screenshot, and console log to scraper_debug_dir on error."""
debug_dir = settings.scraper_debug_dir
if not debug_dir:
return
try:
out = Path(debug_dir)
out.mkdir(parents=True, exist_ok=True)
prefix = out / session_id
console_path = prefix.with_suffix(".console.txt")
console_path.write_text("\n".join(console_messages), encoding="utf-8")
html_path = prefix.with_suffix(".html")
html_path.write_text(await page.content(), encoding="utf-8")
screenshot_path = prefix.with_suffix(".png")
await page.screenshot(path=str(screenshot_path), full_page=True)
logger.info(
"scraper debug artifacts saved [id=%s] dir=%s",
session_id, out.resolve(),
)
except Exception as artifact_exc:
logger.warning("failed to save debug artifacts [id=%s]: %s", session_id, artifact_exc)
async def _run_session(state: _SessionState) -> None:
"""Main browser coroutine. Drives FK login → QR relay → data extraction."""
from playwright.async_api import async_playwright # lazy import — optional dep
console_messages: list[str] = []
async with async_playwright() as pw:
browser = await pw.chromium.launch(headless=settings.scraper_headless)
context = await browser.new_context(locale="sv-SE")
page = await context.new_page()
page.on("console", lambda msg: console_messages.append(f"[{msg.type}] {msg.text}"))
try:
await _do_login(page, state)
if state.status in ("error", "cancelled"):
return
await _do_extract(page, state)
except asyncio.CancelledError:
_set_status(state, "cancelled")
except Exception as exc:
_set_status(state, "error", f"Scraper error: {type(exc).__name__}: {exc}")
logger.error("scraper error [id=%s]: %s", state.session_id, exc, exc_info=False)
await _save_debug_artifacts(page, state.session_id, console_messages)
finally:
try:
await context.close()
await browser.close()
except Exception:
pass # already torn down
async def _do_login(page, state: _SessionState) -> None:
"""Navigate to FK login, handle cookie consent, click BankID QR, relay QR."""
_set_status(state, "launching")
logger.debug("[%s] navigating to FK login URL", state.session_id)
await page.goto(LOGIN_URL, wait_until="networkidle", timeout=30_000)
logger.debug("[%s] page loaded: %s", state.session_id, page.url)
# Check for maintenance message before proceeding
maintenance_msg = await _check_maintenance(page)
if maintenance_msg:
error_msg = f"FK is under maintenance: {maintenance_msg}"
_set_status(state, "error", error_msg)
logger.info("[%s] FK maintenance detected; aborting session", state.session_id)
return
# Accept cookie consent banner if present (only appears once per browser profile)
try:
cookie_btn = page.locator(_SEL.COOKIE_ACCEPT)
if await cookie_btn.count() > 0:
logger.debug("[%s] accepting cookie consent", state.session_id)
await cookie_btn.first.click(timeout=5_000)
else:
logger.debug("[%s] no cookie consent banner", state.session_id)
except Exception:
pass # not present; fine
# Check if BankID button is disabled (another maintenance indicator)
try:
bankid_btn = page.locator(_SEL.BANKID_QR_LINK)
if await bankid_btn.count() > 0:
is_disabled = await bankid_btn.first.is_disabled(timeout=3_000)
if is_disabled:
error_msg = "FK login temporarily unavailable (BankID button disabled). Please try again later."
_set_status(state, "error", error_msg)
logger.info("[%s] BankID button is disabled; aborting session", state.session_id)
return
except Exception:
pass # proceed anyway; may not be disabled
# Click "BankID med QR-kod"
logger.debug("[%s] clicking BankID QR link", state.session_id)
await page.locator(_SEL.BANKID_QR_LINK).first.click(timeout=30_000)
# Wait for the QR SVG to appear (we're on the idpx auth page now)
logger.debug("[%s] waiting for QR code to appear (url=%s)", state.session_id, page.url)
await page.wait_for_selector(_SEL.QR_SVG, timeout=15_000)
logger.debug("[%s] QR code visible; relaying to client", state.session_id)
_set_status(state, "awaiting_scan")
# Relay the QR until FK navigates away (user scanned) or session cancelled/timed out
relay_count = 0
while state.status == "awaiting_scan":
try:
qr_el = page.locator(_SEL.QR_BUTTON)
state.qr_data = await qr_el.screenshot(timeout=3_000)
relay_count += 1
if relay_count % 5 == 1:
logger.debug("[%s] QR screenshot relayed (count=%d)", state.session_id, relay_count)
except Exception:
pass # transient; keep trying
await asyncio.sleep(1)
# Detect navigation to post-login FK page
if POST_LOGIN_URL_FRAGMENT in page.url:
logger.debug("[%s] detected post-login URL; scan complete", state.session_id)
break
# If QR element disappears without the post-login URL, we're in "authenticating"
if await page.locator(_SEL.QR_SVG).count() == 0:
logger.debug("[%s] QR element gone; BankID app likely confirming", state.session_id)
break
if state.status == "cancelled":
return
_set_status(state, "authenticating")
logger.debug("[%s] authenticating — waiting for Ansök button (url=%s)", state.session_id, page.url)
# Wait for post-login landing (the Ansök button appears after JS hydrates the page)
await page.wait_for_selector(_SEL.ANSOK_BUTTON, timeout=30_000)
logger.debug("[%s] Ansök button visible; login complete", state.session_id)
async def _do_extract(page, state: _SessionState) -> None:
"""Click 'Ansök för …', wait for #/valjdagar, extract parental-leave figures."""
_set_status(state, "fetching")
logger.debug("[%s] clicking Ansök button", state.session_id)
# Click the child's "Ansök för …" button
await page.locator(_SEL.ANSOK_BUTTON).first.click(timeout=10_000)
# Wait for the valjdagar data page to load
logger.debug("[%s] waiting for valjdagar page (url=%s)", state.session_id, page.url)
await page.wait_for_selector(_SEL.STEP_DAGAR_KVAR, timeout=30_000)
logger.debug("[%s] valjdagar page loaded (url=%s)", state.session_id, page.url)
data = await _extract_parental_leave(page, state)
if state.persist_callback is not None:
try:
await state.persist_callback(data)
except Exception as exc:
_set_status(
state,
"error",
f"Scrape succeeded but failed to persist data: {type(exc).__name__}: {exc}",
)
logger.error("persist error [id=%s]: %s", state.session_id, exc, exc_info=False)
return
_set_status(state, "done")
state.result = data
logger.info("scraper session done [id=%s]", state.session_id)
async def _extract_parental_leave(page, state: _SessionState) -> ParentalLeaveFetch:
"""Read day figures from the #/valjdagar page and return a validated entity."""
# -- days left for this parent ("För dig" section) --
logger.debug("[%s] reading days_left (FOR_DIG_DAGAR_KVAR)", state.session_id)
days_left_text = await page.locator(_SEL.FOR_DIG_DAGAR_KVAR).first.inner_text(timeout=5_000)
days_left = int(days_left_text.strip())
logger.debug("[%s] days_left=%d", state.session_id, days_left)
# -- total days from "av 240 dagar kvar efter ansökan" --
logger.debug("[%s] reading total days (FOR_DIG_DISCRETE)", state.session_id)
discrete_text = await page.locator(_SEL.FOR_DIG_DISCRETE).first.inner_text(timeout=5_000)
total_match = re.search(r"av\s+(\d+)\s+dagar", discrete_text)
if not total_match:
raise ValueError(f"Could not parse total days from: {discrete_text!r}")
total_days = int(total_match.group(1))
days_used = total_days - days_left
logger.debug("[%s] total_days=%d days_used=%d", state.session_id, total_days, days_used)
# -- sjukpenningnivå and lägstanivå remaining (desktop niva-statistik) --
# all_inner_texts() does not accept a timeout argument — wait for the first element first.
logger.debug("[%s] reading tier breakdown (NIVA_STATISTIK_DESKTOP)", state.session_id)
await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).first.wait_for(timeout=5_000)
niva_texts = await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).all_inner_texts()
logger.debug("[%s] niva_texts raw=%r", state.session_id, niva_texts)
# Each item is like "(119 dagar)" — filter to desktop section (first two)
niva_numbers = []
for t in niva_texts:
m = re.search(r"\((\d+)\s+dagar\)", t)
if m:
niva_numbers.append(int(m.group(1)))
if len(niva_numbers) == 2:
break
if len(niva_numbers) < 2:
raise ValueError(f"Expected 2 tier values, got {niva_numbers} from {niva_texts!r}")
sjukpenning_left, lagsta_niva_left = niva_numbers[0], niva_numbers[1]
logger.debug(
"[%s] sjukpenning_left=%d lagsta_niva_left=%d",
state.session_id, sjukpenning_left, lagsta_niva_left,
)
# Cross-check: sjukpenning_left + lagsta_niva_left should equal days_left
if sjukpenning_left + lagsta_niva_left != days_left:
logger.warning(
"Tier sum mismatch: sjukpenning=%d + lagstaniva=%d != days_left=%d [id=%s]",
sjukpenning_left, lagsta_niva_left, days_left, state.session_id,
)
# -- dubbeldagar remaining --
logger.debug("[%s] reading dubbeldagar (DUBBELDAG_KVAR)", state.session_id)
dubbeldag_text = await page.locator(_SEL.DUBBELDAG_KVAR).first.inner_text(timeout=5_000)
dubbeldagar_left = int(dubbeldag_text.strip())
logger.debug("[%s] dubbeldagar_left=%d", state.session_id, dubbeldagar_left)
# -- Derive tier totals (standard FK allocation per parent) --
# Standard: 195 sjukpenningnivå + 45 lägstanivå = 240 per parent.
# Transfers can change this, but the page doesn't expose the totals directly.
# Assumption: lägstanivå days are never used before sjukpenning (most common case).
lagsta_niva_total = 45 # standard per-parent lägstanivå allocation (= rules.lagsta_niva_days/2)
sjukpenning_total = total_days - lagsta_niva_total # typically 195 for standard 240-day split
lagsta_niva_used = lagsta_niva_total - lagsta_niva_left
sjukpenning_used = sjukpenning_total - sjukpenning_left
# -- Derive reserved / transferable split --
# Reserved days (90 per parent) cannot be transferred; used assumption: transferable taken first.
reserved_days_total = settings.rules.reserved_days_per_parent # 90
transferable_total = total_days - reserved_days_total # 150 for standard 240-day case
transferable_used = min(days_used, transferable_total)
reserved_used = days_used - transferable_used
reserved_days_left = reserved_days_total - reserved_used
transferable_days_left = transferable_total - transferable_used
# -- Validate before returning --
if days_left < 0:
raise ValueError(f"days_left is negative ({days_left}) — FK data inconsistency")
if reserved_days_left > reserved_days_total:
raise ValueError(
f"reserved_days_left ({reserved_days_left}) > reserved_days_total ({reserved_days_total})"
)
now = datetime.now()
tier_data = TierData(
sjukpenning_total=sjukpenning_total,
sjukpenning_used=sjukpenning_used,
sjukpenning_left=sjukpenning_left,
lagsta_niva_total=lagsta_niva_total,
lagsta_niva_used=lagsta_niva_used,
lagsta_niva_left=lagsta_niva_left,
)
return ParentalLeaveFetch(
id=uuid.uuid4(),
personal_data_uuid=state.personal_data_uuid,
workspace_id=state.workspace_id,
child_id=state.child_id,
source="scraped",
fetched_at=now,
total_days=total_days,
days_used=days_used,
days_left=days_left,
reserved_days_total=reserved_days_total,
reserved_days_left=reserved_days_left,
transferable_days_left=transferable_days_left,
dubbeldagar_left=dubbeldagar_left,
tier_data=tier_data,
created_at=now,
updated_at=now,
)