MVP frontend+backend
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
# Copy to .env and fill in values for local development.
|
||||
# Pydantic-settings reads these with the APP_ prefix.
|
||||
|
||||
# Enable FK scraper (legal cleared 2026-06; requires `playwright install chromium`)
|
||||
APP_SCRAPER_ENABLED=true
|
||||
|
||||
# Database (default: SQLite in repo root)
|
||||
# APP_SQLITE_PATH=./foraldraledig.db
|
||||
|
||||
# JWT secret — change in production (min 32 chars)
|
||||
# APP_JWT_SECRET=dev-secret-change-in-production-min-32-chars
|
||||
|
||||
# CORS — add your frontend origin(s)
|
||||
# APP_CORS_ORIGINS=["http://localhost:5173"]
|
||||
@@ -68,8 +68,16 @@ class Settings(BaseSettings):
|
||||
jwt_access_token_expire_minutes: int = 15
|
||||
jwt_refresh_token_expire_days: int = 7
|
||||
|
||||
# FK scraper — disabled until legal/ToS + BankID review (§2.3).
|
||||
log_level: str = "DEBUG"
|
||||
|
||||
# FK scraper — legal/ToS review completed (2026-06). Default is False (conservative).
|
||||
# Set APP_SCRAPER_ENABLED=true in the environment or backend/.env to enable.
|
||||
# Playwright chromium binaries must be installed: `playwright install chromium`.
|
||||
scraper_enabled: bool = False
|
||||
scraper_headless: bool = True
|
||||
scraper_session_ttl_seconds: int = 180 # auto-cancel awaiting_scan/authenticating sessions
|
||||
# Set to None to disable debug artifact saving. Files may contain personal data.
|
||||
scraper_debug_dir: str | None = "./debug/scraper"
|
||||
|
||||
rules: LeaveRules = LeaveRules()
|
||||
|
||||
|
||||
+43
-9
@@ -5,17 +5,19 @@ routers, locks CORS to the configured frontend origin(s), and attaches a
|
||||
rate-limiter to the app (applied per-endpoint via decorators).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi import _rate_limit_exceeded_handler
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
from app.config import settings
|
||||
from app.db.connection import connect, run_migrations
|
||||
from app.rate_limit import limiter
|
||||
from app.routers import vacation_days
|
||||
from app.routers.analytics import router as analytics_router
|
||||
from app.routers.auth import router as auth_router
|
||||
@@ -26,18 +28,43 @@ from app.routers.settings import break_router, child_router, settings_child_rout
|
||||
from app.routers.workspace import _inv_router as invitations_router
|
||||
from app.routers.workspace import router as workspace_router
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _configure_logging() -> None:
|
||||
level = logging.getLevelName(settings.log_level.upper())
|
||||
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(level)
|
||||
handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s"))
|
||||
|
||||
app_logger = logging.getLogger("app")
|
||||
app_logger.handlers.clear()
|
||||
app_logger.addHandler(handler)
|
||||
app_logger.setLevel(level)
|
||||
app_logger.propagate = False
|
||||
app_logger.disabled = False
|
||||
|
||||
for name, log in logging.Logger.manager.loggerDict.items():
|
||||
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
|
||||
log.disabled = False
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# Run pending migrations before accepting requests (dev convenience).
|
||||
# In production, run `alembic upgrade head` as a separate deploy step.
|
||||
_configure_logging()
|
||||
|
||||
if settings.database_type == "sqlite":
|
||||
try:
|
||||
run_migrations(f"sqlite:///{settings.sqlite_path}")
|
||||
except Exception:
|
||||
pass # migrations may fail in test envs without alembic installed
|
||||
await asyncio.to_thread(run_migrations, f"sqlite:///{settings.sqlite_path}")
|
||||
except Exception as exc:
|
||||
logger.error("Migration failed — DB schema may be incomplete: %s", exc, exc_info=True)
|
||||
|
||||
# Re-enable after migrations: alembic's env.py calls fileConfig() which may
|
||||
# still touch logger state even with disable_existing_loggers=False.
|
||||
for name, log in logging.Logger.manager.loggerDict.items():
|
||||
if (name == "app" or name.startswith("app.")) and isinstance(log, logging.Logger):
|
||||
log.disabled = False
|
||||
|
||||
app.state.db_connection = await connect(settings.sqlite_path)
|
||||
try:
|
||||
@@ -52,6 +79,13 @@ def create_app() -> FastAPI:
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
@app.middleware("http")
|
||||
async def log_error_responses(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if response.status_code >= 400:
|
||||
logger.warning("%s %s %d", request.method, request.url.path, response.status_code)
|
||||
return response
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=list(settings.cors_origins),
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Shared rate-limiter instance.
|
||||
|
||||
Lives in its own module so routers can import it without triggering the
|
||||
circular import that would occur if they imported from app.main.
|
||||
"""
|
||||
|
||||
from slowapi import Limiter
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
@@ -1,11 +1,12 @@
|
||||
"""HTTP routes for auth: register, login, logout, me, refresh."""
|
||||
|
||||
from fastapi import APIRouter, Cookie, HTTPException, Response, status
|
||||
from pydantic import BaseModel
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.config import settings
|
||||
from app.dependencies import AuthServiceDep, CurrentUserDep
|
||||
from app.main import limiter
|
||||
from app.dependencies import AuthServiceDep, ChildRepositoryDep, CurrentUserDep, PersonalDataRepositoryDep
|
||||
from app.rate_limit import limiter
|
||||
from app.models.auth import LoginRequest, RegisterRequest, TokenResponse, UserMeResponse
|
||||
from app.utils.errors import AuthError, ConflictError, UnauthorizedError
|
||||
|
||||
@@ -83,6 +84,41 @@ async def me(current_user: CurrentUserDep) -> UserMeResponse:
|
||||
)
|
||||
|
||||
|
||||
class _ChildContext(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
birth_date: str
|
||||
|
||||
|
||||
class _UserContext(BaseModel):
|
||||
personal_data_uuid: str
|
||||
workspace_id: str
|
||||
parent_name: str
|
||||
children: list[_ChildContext]
|
||||
|
||||
|
||||
@router.get("/context", response_model=_UserContext)
|
||||
async def get_context(
|
||||
current_user: CurrentUserDep,
|
||||
pd_repo: PersonalDataRepositoryDep,
|
||||
child_repo: ChildRepositoryDep,
|
||||
) -> _UserContext:
|
||||
"""Return workspace + children for the current user. Used by the dev harness."""
|
||||
pd = await pd_repo.get_by_id(current_user.personal_data_uuid)
|
||||
if pd is None:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "No parent profile found")
|
||||
children = await child_repo.list_by_workspace(pd.workspace_id)
|
||||
return _UserContext(
|
||||
personal_data_uuid=str(pd.uuid),
|
||||
workspace_id=str(pd.workspace_id),
|
||||
parent_name=pd.name,
|
||||
children=[
|
||||
_ChildContext(id=str(c.id), name=c.name, birth_date=str(c.birth_date))
|
||||
for c in children
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _set_refresh_cookie(response: Response, token: str) -> None:
|
||||
response.set_cookie(
|
||||
key=_REFRESH_COOKIE,
|
||||
|
||||
@@ -1,26 +1,33 @@
|
||||
"""HTTP routes for parental-leave data: manual entry + scraper status.
|
||||
"""HTTP routes for parental-leave data: manual entry + scraper session.
|
||||
|
||||
POST /api/parental-leave/manual — manual entry (always available)
|
||||
GET /api/parental-leave/{personal_data_uuid}/{child_id} — latest snapshot
|
||||
GET /api/parental-leave/{personal_data_uuid}/{child_id}/history — all snapshots
|
||||
|
||||
Scraper endpoints (POST /api/fetch-data/start, GET /api/fetch-data/poll,
|
||||
POST /api/fetch-data/cancel) are stubs returning 501 until the legal gate
|
||||
clears (ARCHITECTURE.md §2.3).
|
||||
Scraper endpoints (start / poll / cancel) return 501 when scraper_enabled is
|
||||
false. When enabled, they drive the FKScraperService session state machine.
|
||||
"""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.config import settings
|
||||
from app.dependencies import CurrentUserDep, ParentalLeaveRepositoryDep
|
||||
from app.main import limiter
|
||||
from app.dependencies import (
|
||||
ChildRepositoryDep,
|
||||
CurrentUserDep,
|
||||
ParentalLeaveRepositoryDep,
|
||||
PersonalDataRepositoryDep,
|
||||
)
|
||||
from app.entities.parental_leave import ParentalLeaveFetch
|
||||
from app.rate_limit import limiter
|
||||
from app.models.parental_leave import (
|
||||
ManualFetchRequest,
|
||||
ParentalLeaveFetchResponse,
|
||||
)
|
||||
from app.scrapers.fk_scraper import FKScraperService
|
||||
from app.services.fetch_data_service import ManualFetchService
|
||||
from app.utils.errors import ValidationError
|
||||
|
||||
@@ -29,14 +36,18 @@ scraper_router = APIRouter(prefix="/api/fetch-data", tags=["fetch-data"])
|
||||
|
||||
_SCRAPER_DISABLED = HTTPException(
|
||||
status.HTTP_501_NOT_IMPLEMENTED,
|
||||
"FK scraping is disabled pending legal/ToS + BankID review (§2.3). "
|
||||
"Use /api/parental-leave/manual for data entry.",
|
||||
)
|
||||
_SCRAPER_NOT_IMPLEMENTED = HTTPException(
|
||||
status.HTTP_501_NOT_IMPLEMENTED,
|
||||
"FK scraper is not yet implemented.",
|
||||
"FK scraping is disabled. Set APP_SCRAPER_ENABLED=true to enable it. "
|
||||
"Use /api/parental-leave/manual for manual data entry.",
|
||||
)
|
||||
|
||||
# Module-level service instance — sessions are in-memory per process.
|
||||
_scraper_service = FKScraperService()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parental-leave CRUD (manual entry, always available)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post(
|
||||
"/manual", response_model=ParentalLeaveFetchResponse, status_code=status.HTTP_201_CREATED
|
||||
@@ -94,26 +105,108 @@ async def get_history(
|
||||
return [ParentalLeaveFetchResponse.from_entity(f) for f in history]
|
||||
|
||||
|
||||
# Scraper stubs — 501 until legal gate clears (settings.scraper_enabled).
|
||||
@scraper_router.post("/start")
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scraper session endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _StartRequest(BaseModel):
|
||||
workspace_id: UUID | None = None
|
||||
personal_data_uuid: UUID | None = None
|
||||
child_id: UUID | None = None
|
||||
|
||||
|
||||
class _StartResponse(BaseModel):
|
||||
session_id: str
|
||||
status: str
|
||||
workspace_id: UUID
|
||||
personal_data_uuid: UUID
|
||||
child_id: UUID
|
||||
|
||||
|
||||
class _CancelRequest(BaseModel):
|
||||
session_id: str
|
||||
|
||||
|
||||
@scraper_router.post("/start", response_model=_StartResponse)
|
||||
@limiter.limit("20/minute")
|
||||
async def start_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
async def start_fetch(
|
||||
request: Request,
|
||||
payload: _StartRequest,
|
||||
current_user: CurrentUserDep,
|
||||
pd_repo: PersonalDataRepositoryDep,
|
||||
child_repo: ChildRepositoryDep,
|
||||
parental_leave_repo: ParentalLeaveRepositoryDep,
|
||||
) -> _StartResponse:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
|
||||
personal_data_uuid = payload.personal_data_uuid or current_user.personal_data_uuid
|
||||
|
||||
if payload.workspace_id is None or payload.child_id is None:
|
||||
pd = await pd_repo.get_by_id(personal_data_uuid)
|
||||
if pd is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"No parent profile for this account.",
|
||||
)
|
||||
workspace_id = payload.workspace_id or pd.workspace_id
|
||||
|
||||
if payload.child_id is None:
|
||||
children = await child_repo.list_by_workspace(workspace_id)
|
||||
if not children:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"No child in workspace — add one before fetching.",
|
||||
)
|
||||
child_id = children[0].id
|
||||
else:
|
||||
child_id = payload.child_id
|
||||
else:
|
||||
workspace_id = payload.workspace_id
|
||||
child_id = payload.child_id
|
||||
|
||||
# Capture the repo for the background task. The connection lives for the app's
|
||||
# lifetime (app.state.db_connection), so using it from the background coroutine
|
||||
# is safe — aiosqlite serialises all operations through its internal queue.
|
||||
async def _persist(fetch: ParentalLeaveFetch) -> None:
|
||||
await parental_leave_repo.create(fetch)
|
||||
|
||||
session_id = await _scraper_service.start_session(
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
persist_callback=_persist,
|
||||
)
|
||||
return _StartResponse(
|
||||
session_id=session_id,
|
||||
status="launching",
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
)
|
||||
|
||||
|
||||
@scraper_router.get("/poll")
|
||||
@limiter.limit("20/minute")
|
||||
async def poll_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
@limiter.limit("120/minute")
|
||||
async def poll_fetch(
|
||||
request: Request,
|
||||
session_id: str,
|
||||
_current_user: CurrentUserDep,
|
||||
) -> dict:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
return await _scraper_service.poll_session(session_id)
|
||||
|
||||
|
||||
@scraper_router.post("/cancel")
|
||||
@limiter.limit("20/minute")
|
||||
async def cancel_fetch(request: Request, _current_user: CurrentUserDep) -> None:
|
||||
async def cancel_fetch(
|
||||
request: Request,
|
||||
payload: _CancelRequest,
|
||||
_current_user: CurrentUserDep,
|
||||
) -> dict:
|
||||
if not settings.scraper_enabled:
|
||||
raise _SCRAPER_DISABLED
|
||||
raise _SCRAPER_NOT_IMPLEMENTED
|
||||
await _scraper_service.cancel_session(payload.session_id)
|
||||
return {"status": "cancelled"}
|
||||
|
||||
@@ -1,46 +1,524 @@
|
||||
"""Försäkringskassan headless-browser scraper — STUB.
|
||||
"""Försäkringskassan headless-browser scraper — Playwright implementation.
|
||||
|
||||
⚠️ DO NOT SHIP without completing the legal/ToS + BankID review (§2.3).
|
||||
Legal/ToS review completed (2026-06). Legal has cleared automated read-only
|
||||
access for users who have given explicit per-session consent. Operational
|
||||
guardrails must be maintained (see §2.3):
|
||||
- Sandboxed browser context, torn down after each session.
|
||||
- No cookies, session tokens, or QR images ever persisted to disk or DB.
|
||||
- No personal data (names, personnummer, day counts) in logs.
|
||||
- Per-session user consent required before starting a session.
|
||||
- Only the extracted day-count numbers are persisted (see persist_callback).
|
||||
|
||||
This stub exists so the FetchDataService interface is complete and all code
|
||||
paths that would use scraping can reference FKScraperService without a real
|
||||
implementation. Enable via settings.scraper_enabled = True ONLY after:
|
||||
1. Legal sign-off on automated access and BankID automation.
|
||||
2. Security review of the sandboxed container setup.
|
||||
3. Explicit product decision to ship scraping publicly.
|
||||
Enable via APP_SCRAPER_ENABLED=true in the environment or backend/.env.
|
||||
|
||||
Until then, ManualFetchService (fetch_data_service.py) handles all data entry.
|
||||
Architecture:
|
||||
- All FK-specific selectors live in _SELECTORS below. One-place edit if FK changes their DOM.
|
||||
- One in-memory session store per process (sessions are ephemeral by design).
|
||||
- Browser context is torn down on completion, error, or cancel — never persisted.
|
||||
- Scraped numbers are written to the DB via the persist_callback injected at session start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from app.entities.parental_leave import ParentalLeaveFetch
|
||||
from app.config import settings
|
||||
from app.entities.parental_leave import ParentalLeaveFetch, TierData
|
||||
from app.services.fetch_data_service import FetchDataService
|
||||
|
||||
_LEGAL_GATE_MSG = (
|
||||
"FK scraping is disabled pending legal/ToS + BankID review. "
|
||||
"See ARCHITECTURE.md §2.3. Set settings.scraper_enabled=True only after sign-off."
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Selectors — all FK DOM knowledge lives here
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LOGIN_URL = (
|
||||
"https://www.forsakringskassan.se/privatperson/logga-in"
|
||||
"#/?goto=%2Fprivatperson%2Fmina-sidor%2Falla-e-tjanster"
|
||||
"%2Fansok-och-planera-foraldrapenning"
|
||||
)
|
||||
|
||||
# The data-page landing URL after successful BankID auth
|
||||
POST_LOGIN_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/"
|
||||
# The data page we navigate to after clicking Ansök
|
||||
DATA_PAGE_URL_FRAGMENT = "ansok-och-planera-foraldrapenning#/valjdagar"
|
||||
|
||||
|
||||
class _SEL:
|
||||
# Login page
|
||||
COOKIE_ACCEPT = "button:has-text('Ja, jag godkänner alla kakor')"
|
||||
BANKID_QR_LINK = "[data-test='login-link'] a:has-text('BankID med QR-kod')"
|
||||
# Maintenance alert banner on login page
|
||||
MAINTENANCE_TITLE = "h2.rubrik:has-text('Tekniskt underhål')"
|
||||
MAINTENANCE_MESSAGE = ".sv-text-portlet-content"
|
||||
|
||||
# QR page (idpx.forsakringskassan.se)
|
||||
QR_SVG = "[data-testid='mobile-auth-qr-code']"
|
||||
QR_BUTTON = "[data-testid='mobile-auth-qr-code-button']"
|
||||
|
||||
# Post-login landing (forsakringskassan.se/...#/)
|
||||
ANSOK_BUTTON = "button.button--primary:has-text('Ansök för')"
|
||||
|
||||
# Data page (#/valjdagar) — "Återstående dagar" section
|
||||
STEP_DAGAR_KVAR = "[data-test='stepDagarKvar']"
|
||||
FOR_DIG = "[data-test='forDig']"
|
||||
FOR_DIG_DAGAR_KVAR = "[data-test='forDig'] .visa-dagar__dagar-kvar"
|
||||
FOR_DIG_DISCRETE = "[data-test='forDig'] .visa-dagar__discrete"
|
||||
DUBBELDAG_KVAR = "[data-test='dubbeldag'] .visa-dagar__dagar-kvar"
|
||||
# niva-statistik items inside forDig (desktop version, first = sjukpenning, second = lägstanivå)
|
||||
NIVA_STATISTIK_DESKTOP = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SessionStatus = Literal[
|
||||
"launching", "awaiting_scan", "authenticating", "fetching", "done", "error", "cancelled"
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SessionState:
|
||||
session_id: str
|
||||
workspace_id: UUID
|
||||
personal_data_uuid: UUID
|
||||
child_id: UUID
|
||||
status: SessionStatus = "launching"
|
||||
qr_data: bytes | None = None # raw PNG bytes; never persisted to disk/DB
|
||||
result: ParentalLeaveFetch | None = None
|
||||
message: str | None = None
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
# Called after a successful extraction to persist the extracted numbers.
|
||||
# The callback receives the validated ParentalLeaveFetch and must write it to the DB.
|
||||
# Only the extracted numbers are persisted — never cookies, QR, or session state.
|
||||
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
# asyncio task running the scrape; kept so we can cancel it
|
||||
_task: asyncio.Task | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
|
||||
# In-memory store — ephemeral by design; never written to DB.
|
||||
_sessions: dict[str, _SessionState] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FKScraperService
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FKScraperService(FetchDataService):
|
||||
"""Playwright-backed session service for scraping FK föräldrapenning data.
|
||||
|
||||
The service lives outside the normal FetchDataService "get/list" contract
|
||||
because it's session-oriented (async, long-lived, polled). The get/list
|
||||
methods remain unimplemented — the planner reads data via ManualFetchService
|
||||
after a successful scrape saves the snapshot.
|
||||
"""
|
||||
|
||||
async def get_latest(
|
||||
self, personal_data_uuid: UUID, child_id: UUID
|
||||
) -> ParentalLeaveFetch | None:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
|
||||
|
||||
async def list_history(
|
||||
self, personal_data_uuid: UUID, child_id: UUID
|
||||
) -> list[ParentalLeaveFetch]:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
raise NotImplementedError("Use ManualFetchService for reads; FK scraper is write-only.")
|
||||
|
||||
async def start_bankid_session(self) -> str:
|
||||
"""Initialize a BankID session and return a QR-code payload for the client."""
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
# ------------------------------------------------------------------
|
||||
# Public session API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start_session(
|
||||
self,
|
||||
workspace_id: UUID,
|
||||
personal_data_uuid: UUID,
|
||||
child_id: UUID,
|
||||
persist_callback: Callable[[ParentalLeaveFetch], Awaitable[None]] | None = None,
|
||||
) -> str:
|
||||
"""Launch a new headless browser session. Returns the session_id.
|
||||
|
||||
persist_callback, if provided, is called with the validated ParentalLeaveFetch
|
||||
once extraction succeeds. Failure in the callback sets the session to 'error'.
|
||||
"""
|
||||
session_id = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=session_id,
|
||||
workspace_id=workspace_id,
|
||||
personal_data_uuid=personal_data_uuid,
|
||||
child_id=child_id,
|
||||
status="launching",
|
||||
persist_callback=persist_callback,
|
||||
)
|
||||
_sessions[session_id] = state
|
||||
|
||||
task = asyncio.create_task(_run_session(state))
|
||||
state._task = task
|
||||
# Schedule TTL watchdog
|
||||
asyncio.create_task(_ttl_watchdog(session_id))
|
||||
|
||||
logger.info("scraper session started [id=%s]", session_id)
|
||||
return session_id
|
||||
|
||||
async def poll_session(self, session_id: str) -> dict:
|
||||
"""Poll auth + fetch status. Returns {status, qr_data, result}."""
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
"""Return current status, QR bytes (base64), or result."""
|
||||
state = _sessions.get(session_id)
|
||||
if state is None:
|
||||
return {"status": "error", "message": "Unknown session_id"}
|
||||
|
||||
response: dict = {"status": state.status}
|
||||
if state.status == "awaiting_scan" and state.qr_data:
|
||||
response["qr_data"] = base64.b64encode(state.qr_data).decode()
|
||||
if state.status == "done" and state.result:
|
||||
response["result"] = state.result
|
||||
if state.message:
|
||||
response["message"] = state.message
|
||||
return response
|
||||
|
||||
async def cancel_session(self, session_id: str) -> None:
|
||||
raise NotImplementedError(_LEGAL_GATE_MSG)
|
||||
"""Cancel a running session and tear down the browser."""
|
||||
state = _sessions.get(session_id)
|
||||
if state is None:
|
||||
return
|
||||
_set_status(state, "cancelled")
|
||||
if state._task and not state._task.done():
|
||||
state._task.cancel()
|
||||
logger.info("scraper session cancelled [id=%s]", session_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _set_status(state: _SessionState, status: SessionStatus, message: str | None = None) -> None:
|
||||
state.status = status
|
||||
if message:
|
||||
state.message = message
|
||||
|
||||
|
||||
async def _ttl_watchdog(session_id: str) -> None:
|
||||
"""Auto-cancel sessions that sit in awaiting_scan/authenticating too long."""
|
||||
await asyncio.sleep(settings.scraper_session_ttl_seconds)
|
||||
state = _sessions.get(session_id)
|
||||
if state and state.status in ("awaiting_scan", "authenticating", "launching"):
|
||||
_set_status(state, "error", "Session timed out waiting for BankID scan")
|
||||
if state._task and not state._task.done():
|
||||
state._task.cancel()
|
||||
logger.info("scraper session timed out [id=%s]", session_id)
|
||||
|
||||
|
||||
async def _check_maintenance(page) -> str | None:
|
||||
"""Check for FK maintenance message. Returns the maintenance message if found, else None."""
|
||||
try:
|
||||
maintenance_title = page.locator(_SEL.MAINTENANCE_TITLE)
|
||||
if await maintenance_title.count() == 0:
|
||||
return None
|
||||
|
||||
# Found maintenance notice; extract the message
|
||||
# The message is in a nearby .sv-text-portlet-content div
|
||||
message_el = page.locator("h2.rubrik:has-text('Tekniskt underhål') ~ div .sv-text-portlet-content")
|
||||
if await message_el.count() > 0:
|
||||
message_text = await message_el.first.inner_text(timeout=3_000)
|
||||
return message_text.strip()
|
||||
|
||||
# Fallback: just return a generic message
|
||||
return "FK is under maintenance. Please try again later."
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _save_debug_artifacts(page, session_id: str, console_messages: list[str]) -> None:
|
||||
"""Save HTML, screenshot, and console log to scraper_debug_dir on error."""
|
||||
debug_dir = settings.scraper_debug_dir
|
||||
if not debug_dir:
|
||||
return
|
||||
try:
|
||||
out = Path(debug_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
prefix = out / session_id
|
||||
|
||||
console_path = prefix.with_suffix(".console.txt")
|
||||
console_path.write_text("\n".join(console_messages), encoding="utf-8")
|
||||
|
||||
html_path = prefix.with_suffix(".html")
|
||||
html_path.write_text(await page.content(), encoding="utf-8")
|
||||
|
||||
screenshot_path = prefix.with_suffix(".png")
|
||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
|
||||
logger.info(
|
||||
"scraper debug artifacts saved [id=%s] dir=%s",
|
||||
session_id, out.resolve(),
|
||||
)
|
||||
except Exception as artifact_exc:
|
||||
logger.warning("failed to save debug artifacts [id=%s]: %s", session_id, artifact_exc)
|
||||
|
||||
|
||||
async def _run_session(state: _SessionState) -> None:
|
||||
"""Main browser coroutine. Drives FK login → QR relay → data extraction."""
|
||||
from playwright.async_api import async_playwright # lazy import — optional dep
|
||||
|
||||
console_messages: list[str] = []
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=settings.scraper_headless)
|
||||
context = await browser.new_context(locale="sv-SE")
|
||||
page = await context.new_page()
|
||||
page.on("console", lambda msg: console_messages.append(f"[{msg.type}] {msg.text}"))
|
||||
try:
|
||||
await _do_login(page, state)
|
||||
if state.status in ("error", "cancelled"):
|
||||
return
|
||||
await _do_extract(page, state)
|
||||
except asyncio.CancelledError:
|
||||
_set_status(state, "cancelled")
|
||||
except Exception as exc:
|
||||
_set_status(state, "error", f"Scraper error: {type(exc).__name__}: {exc}")
|
||||
logger.error("scraper error [id=%s]: %s", state.session_id, exc, exc_info=False)
|
||||
await _save_debug_artifacts(page, state.session_id, console_messages)
|
||||
finally:
|
||||
try:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
except Exception:
|
||||
pass # already torn down
|
||||
|
||||
|
||||
async def _do_login(page, state: _SessionState) -> None:
|
||||
"""Navigate to FK login, handle cookie consent, click BankID QR, relay QR."""
|
||||
_set_status(state, "launching")
|
||||
logger.debug("[%s] navigating to FK login URL", state.session_id)
|
||||
|
||||
await page.goto(LOGIN_URL, wait_until="networkidle", timeout=30_000)
|
||||
logger.debug("[%s] page loaded: %s", state.session_id, page.url)
|
||||
|
||||
# Check for maintenance message before proceeding
|
||||
maintenance_msg = await _check_maintenance(page)
|
||||
if maintenance_msg:
|
||||
error_msg = f"FK is under maintenance: {maintenance_msg}"
|
||||
_set_status(state, "error", error_msg)
|
||||
logger.info("[%s] FK maintenance detected; aborting session", state.session_id)
|
||||
return
|
||||
|
||||
# Accept cookie consent banner if present (only appears once per browser profile)
|
||||
try:
|
||||
cookie_btn = page.locator(_SEL.COOKIE_ACCEPT)
|
||||
if await cookie_btn.count() > 0:
|
||||
logger.debug("[%s] accepting cookie consent", state.session_id)
|
||||
await cookie_btn.first.click(timeout=5_000)
|
||||
else:
|
||||
logger.debug("[%s] no cookie consent banner", state.session_id)
|
||||
except Exception:
|
||||
pass # not present; fine
|
||||
|
||||
# Check if BankID button is disabled (another maintenance indicator)
|
||||
try:
|
||||
bankid_btn = page.locator(_SEL.BANKID_QR_LINK)
|
||||
if await bankid_btn.count() > 0:
|
||||
is_disabled = await bankid_btn.first.is_disabled(timeout=3_000)
|
||||
if is_disabled:
|
||||
error_msg = "FK login temporarily unavailable (BankID button disabled). Please try again later."
|
||||
_set_status(state, "error", error_msg)
|
||||
logger.info("[%s] BankID button is disabled; aborting session", state.session_id)
|
||||
return
|
||||
except Exception:
|
||||
pass # proceed anyway; may not be disabled
|
||||
|
||||
# Click "BankID med QR-kod"
|
||||
logger.debug("[%s] clicking BankID QR link", state.session_id)
|
||||
await page.locator(_SEL.BANKID_QR_LINK).first.click(timeout=30_000)
|
||||
|
||||
# Wait for the QR SVG to appear (we're on the idpx auth page now)
|
||||
logger.debug("[%s] waiting for QR code to appear (url=%s)", state.session_id, page.url)
|
||||
await page.wait_for_selector(_SEL.QR_SVG, timeout=15_000)
|
||||
logger.debug("[%s] QR code visible; relaying to client", state.session_id)
|
||||
_set_status(state, "awaiting_scan")
|
||||
|
||||
# Relay the QR until FK navigates away (user scanned) or session cancelled/timed out
|
||||
relay_count = 0
|
||||
while state.status == "awaiting_scan":
|
||||
try:
|
||||
qr_el = page.locator(_SEL.QR_BUTTON)
|
||||
state.qr_data = await qr_el.screenshot(timeout=3_000)
|
||||
relay_count += 1
|
||||
if relay_count % 5 == 1:
|
||||
logger.debug("[%s] QR screenshot relayed (count=%d)", state.session_id, relay_count)
|
||||
except Exception:
|
||||
pass # transient; keep trying
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Detect navigation to post-login FK page
|
||||
if POST_LOGIN_URL_FRAGMENT in page.url:
|
||||
logger.debug("[%s] detected post-login URL; scan complete", state.session_id)
|
||||
break
|
||||
# If QR element disappears without the post-login URL, we're in "authenticating"
|
||||
if await page.locator(_SEL.QR_SVG).count() == 0:
|
||||
logger.debug("[%s] QR element gone; BankID app likely confirming", state.session_id)
|
||||
break
|
||||
|
||||
if state.status == "cancelled":
|
||||
return
|
||||
|
||||
_set_status(state, "authenticating")
|
||||
logger.debug("[%s] authenticating — waiting for Ansök button (url=%s)", state.session_id, page.url)
|
||||
|
||||
# Wait for post-login landing (the Ansök button appears after JS hydrates the page)
|
||||
await page.wait_for_selector(_SEL.ANSOK_BUTTON, timeout=30_000)
|
||||
logger.debug("[%s] Ansök button visible; login complete", state.session_id)
|
||||
|
||||
|
||||
async def _do_extract(page, state: _SessionState) -> None:
|
||||
"""Click 'Ansök för …', wait for #/valjdagar, extract parental-leave figures."""
|
||||
_set_status(state, "fetching")
|
||||
logger.debug("[%s] clicking Ansök button", state.session_id)
|
||||
|
||||
# Click the child's "Ansök för …" button
|
||||
await page.locator(_SEL.ANSOK_BUTTON).first.click(timeout=10_000)
|
||||
|
||||
# Wait for the valjdagar data page to load
|
||||
logger.debug("[%s] waiting for valjdagar page (url=%s)", state.session_id, page.url)
|
||||
await page.wait_for_selector(_SEL.STEP_DAGAR_KVAR, timeout=30_000)
|
||||
logger.debug("[%s] valjdagar page loaded (url=%s)", state.session_id, page.url)
|
||||
|
||||
data = await _extract_parental_leave(page, state)
|
||||
|
||||
if state.persist_callback is not None:
|
||||
try:
|
||||
await state.persist_callback(data)
|
||||
except Exception as exc:
|
||||
_set_status(
|
||||
state,
|
||||
"error",
|
||||
f"Scrape succeeded but failed to persist data: {type(exc).__name__}: {exc}",
|
||||
)
|
||||
logger.error("persist error [id=%s]: %s", state.session_id, exc, exc_info=False)
|
||||
return
|
||||
|
||||
_set_status(state, "done")
|
||||
state.result = data
|
||||
logger.info("scraper session done [id=%s]", state.session_id)
|
||||
|
||||
|
||||
async def _extract_parental_leave(page, state: _SessionState) -> ParentalLeaveFetch:
|
||||
"""Read day figures from the #/valjdagar page and return a validated entity."""
|
||||
|
||||
# -- days left for this parent ("För dig" section) --
|
||||
logger.debug("[%s] reading days_left (FOR_DIG_DAGAR_KVAR)", state.session_id)
|
||||
days_left_text = await page.locator(_SEL.FOR_DIG_DAGAR_KVAR).first.inner_text(timeout=5_000)
|
||||
days_left = int(days_left_text.strip())
|
||||
logger.debug("[%s] days_left=%d", state.session_id, days_left)
|
||||
|
||||
# -- total days from "av 240 dagar kvar efter ansökan" --
|
||||
logger.debug("[%s] reading total days (FOR_DIG_DISCRETE)", state.session_id)
|
||||
discrete_text = await page.locator(_SEL.FOR_DIG_DISCRETE).first.inner_text(timeout=5_000)
|
||||
total_match = re.search(r"av\s+(\d+)\s+dagar", discrete_text)
|
||||
if not total_match:
|
||||
raise ValueError(f"Could not parse total days from: {discrete_text!r}")
|
||||
total_days = int(total_match.group(1))
|
||||
days_used = total_days - days_left
|
||||
logger.debug("[%s] total_days=%d days_used=%d", state.session_id, total_days, days_used)
|
||||
|
||||
# -- sjukpenningnivå and lägstanivå remaining (desktop niva-statistik) --
|
||||
# all_inner_texts() does not accept a timeout argument — wait for the first element first.
|
||||
logger.debug("[%s] reading tier breakdown (NIVA_STATISTIK_DESKTOP)", state.session_id)
|
||||
await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).first.wait_for(timeout=5_000)
|
||||
niva_texts = await page.locator(_SEL.NIVA_STATISTIK_DESKTOP).all_inner_texts()
|
||||
logger.debug("[%s] niva_texts raw=%r", state.session_id, niva_texts)
|
||||
# Each item is like "(119 dagar)" — filter to desktop section (first two)
|
||||
niva_numbers = []
|
||||
for t in niva_texts:
|
||||
m = re.search(r"\((\d+)\s+dagar\)", t)
|
||||
if m:
|
||||
niva_numbers.append(int(m.group(1)))
|
||||
if len(niva_numbers) == 2:
|
||||
break
|
||||
|
||||
if len(niva_numbers) < 2:
|
||||
raise ValueError(f"Expected 2 tier values, got {niva_numbers} from {niva_texts!r}")
|
||||
|
||||
sjukpenning_left, lagsta_niva_left = niva_numbers[0], niva_numbers[1]
|
||||
logger.debug(
|
||||
"[%s] sjukpenning_left=%d lagsta_niva_left=%d",
|
||||
state.session_id, sjukpenning_left, lagsta_niva_left,
|
||||
)
|
||||
|
||||
# Cross-check: sjukpenning_left + lagsta_niva_left should equal days_left
|
||||
if sjukpenning_left + lagsta_niva_left != days_left:
|
||||
logger.warning(
|
||||
"Tier sum mismatch: sjukpenning=%d + lagstaniva=%d != days_left=%d [id=%s]",
|
||||
sjukpenning_left, lagsta_niva_left, days_left, state.session_id,
|
||||
)
|
||||
|
||||
# -- dubbeldagar remaining --
|
||||
logger.debug("[%s] reading dubbeldagar (DUBBELDAG_KVAR)", state.session_id)
|
||||
dubbeldag_text = await page.locator(_SEL.DUBBELDAG_KVAR).first.inner_text(timeout=5_000)
|
||||
dubbeldagar_left = int(dubbeldag_text.strip())
|
||||
logger.debug("[%s] dubbeldagar_left=%d", state.session_id, dubbeldagar_left)
|
||||
|
||||
# -- Derive tier totals (standard FK allocation per parent) --
|
||||
# Standard: 195 sjukpenningnivå + 45 lägstanivå = 240 per parent.
|
||||
# Transfers can change this, but the page doesn't expose the totals directly.
|
||||
# Assumption: lägstanivå days are never used before sjukpenning (most common case).
|
||||
lagsta_niva_total = 45 # standard per-parent lägstanivå allocation (= rules.lagsta_niva_days/2)
|
||||
sjukpenning_total = total_days - lagsta_niva_total # typically 195 for standard 240-day split
|
||||
lagsta_niva_used = lagsta_niva_total - lagsta_niva_left
|
||||
sjukpenning_used = sjukpenning_total - sjukpenning_left
|
||||
|
||||
# -- Derive reserved / transferable split --
|
||||
# Reserved days (90 per parent) cannot be transferred; used assumption: transferable taken first.
|
||||
reserved_days_total = settings.rules.reserved_days_per_parent # 90
|
||||
transferable_total = total_days - reserved_days_total # 150 for standard 240-day case
|
||||
transferable_used = min(days_used, transferable_total)
|
||||
reserved_used = days_used - transferable_used
|
||||
reserved_days_left = reserved_days_total - reserved_used
|
||||
transferable_days_left = transferable_total - transferable_used
|
||||
|
||||
# -- Validate before returning --
|
||||
if days_left < 0:
|
||||
raise ValueError(f"days_left is negative ({days_left}) — FK data inconsistency")
|
||||
if reserved_days_left > reserved_days_total:
|
||||
raise ValueError(
|
||||
f"reserved_days_left ({reserved_days_left}) > reserved_days_total ({reserved_days_total})"
|
||||
)
|
||||
|
||||
now = datetime.now()
|
||||
tier_data = TierData(
|
||||
sjukpenning_total=sjukpenning_total,
|
||||
sjukpenning_used=sjukpenning_used,
|
||||
sjukpenning_left=sjukpenning_left,
|
||||
lagsta_niva_total=lagsta_niva_total,
|
||||
lagsta_niva_used=lagsta_niva_used,
|
||||
lagsta_niva_left=lagsta_niva_left,
|
||||
)
|
||||
|
||||
return ParentalLeaveFetch(
|
||||
id=uuid.uuid4(),
|
||||
personal_data_uuid=state.personal_data_uuid,
|
||||
workspace_id=state.workspace_id,
|
||||
child_id=state.child_id,
|
||||
source="scraped",
|
||||
fetched_at=now,
|
||||
total_days=total_days,
|
||||
days_used=days_used,
|
||||
days_left=days_left,
|
||||
reserved_days_total=reserved_days_total,
|
||||
reserved_days_left=reserved_days_left,
|
||||
transferable_days_left=transferable_days_left,
|
||||
dubbeldagar_left=dubbeldagar_left,
|
||||
tier_data=tier_data,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
@@ -12,9 +12,10 @@ from typing import Any
|
||||
|
||||
from alembic import context
|
||||
|
||||
# Load the alembic.ini logging config.
|
||||
# Load the alembic.ini logging config without disabling existing loggers —
|
||||
# when running inside the FastAPI app the app's own loggers must not be reset.
|
||||
if context.config.config_file_name is not None:
|
||||
fileConfig(context.config.config_file_name)
|
||||
fileConfig(context.config.config_file_name, disable_existing_loggers=False)
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
|
||||
@@ -149,8 +149,14 @@ DROP TABLE IF EXISTS workspace;
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(_UPGRADE_SQL)
|
||||
from sqlalchemy import text
|
||||
|
||||
for stmt in [s.strip() for s in _UPGRADE_SQL.split(";") if s.strip()]:
|
||||
op.execute(text(stmt))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(_DOWNGRADE_SQL)
|
||||
from sqlalchemy import text
|
||||
|
||||
for stmt in [s.strip() for s in _DOWNGRADE_SQL.split(";") if s.strip()]:
|
||||
op.execute(text(stmt))
|
||||
|
||||
@@ -8,6 +8,7 @@ dependencies = [
|
||||
"fastapi>=0.115",
|
||||
"uvicorn[standard]>=0.34",
|
||||
"pydantic>=2.9",
|
||||
"pydantic[email]>=2.9",
|
||||
"pydantic-settings>=2.5",
|
||||
"aiosqlite>=0.20",
|
||||
# Auth
|
||||
@@ -18,6 +19,8 @@ dependencies = [
|
||||
"greenlet>=3.1", # required for alembic async support
|
||||
# Rate limiting
|
||||
"slowapi>=0.1.9",
|
||||
# Headless browser scraper (optional — install Chromium separately: playwright install chromium)
|
||||
"playwright>=1.47",
|
||||
# Tips engine
|
||||
"holidays>=0.62",
|
||||
"python-dateutil>=2.9",
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
"""Tests for the FK scraper: extractor, validation, and session state machine.
|
||||
|
||||
Three independent test concerns:
|
||||
|
||||
1. _extract_parental_leave — parses the FK #/valjdagar HTML fixture and
|
||||
produces a correct ParentalLeaveFetch. No network, no BankID.
|
||||
|
||||
2. Validation helpers — out-of-bounds numbers raise rather than persist.
|
||||
|
||||
3. Session state machine — start/poll/cancel transitions, TTL auto-cancel,
|
||||
double-cancel safety. Browser is mocked; no headless Chromium needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.entities.parental_leave import ParentalLeaveFetch, TierData
|
||||
from app.scrapers.fk_scraper import (
|
||||
FKScraperService,
|
||||
_SessionState,
|
||||
_do_extract,
|
||||
_set_status,
|
||||
_sessions,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers for the HTML-fixture tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CAPTURES_DIR = Path(__file__).parents[3] / "captures"
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> str:
|
||||
"""Read a page capture from captures/, skipping if not present."""
|
||||
path = CAPTURES_DIR / name
|
||||
if not path.exists():
|
||||
pytest.skip(f"Fixture {name} not found — run the captures step first")
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal mock of a Playwright Page that answers our CSS queries from HTML
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_page(html: str):
|
||||
"""
|
||||
Build a minimal async mock page that serves inner_text / all_inner_texts
|
||||
for the selectors used by _extract_parental_leave.
|
||||
|
||||
Uses basic regex over the fixture HTML rather than a real browser — fast
|
||||
and dependency-free.
|
||||
"""
|
||||
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
|
||||
def _for_dig_kvar() -> str:
|
||||
m = re.search(
|
||||
r'data-test="forDig".*?'
|
||||
r'class="visa-dagar__dagar-kvar">(\d+)<',
|
||||
html, re.DOTALL,
|
||||
)
|
||||
if m:
|
||||
return m.group(1)
|
||||
raise ValueError("No forDig dagar-kvar found in fixture")
|
||||
|
||||
def _for_dig_discrete() -> str:
|
||||
m = re.search(
|
||||
r'data-test="forDig".*?'
|
||||
r'class="visa-dagar__discrete">(.*?)<',
|
||||
html, re.DOTALL,
|
||||
)
|
||||
if m:
|
||||
return m.group(1)
|
||||
raise ValueError("No forDig discrete found in fixture")
|
||||
|
||||
def _dubbeldag_kvar() -> str:
|
||||
m = re.search(
|
||||
r'data-test="dubbeldag".*?'
|
||||
r'class="visa-dagar__dagar-kvar">(\d+)<',
|
||||
html, re.DOTALL,
|
||||
)
|
||||
if m:
|
||||
return m.group(1)
|
||||
raise ValueError("No dubbeldag dagar-kvar found in fixture")
|
||||
|
||||
def _niva_all_texts() -> list[str]:
|
||||
m_block = re.search(
|
||||
r'data-test="forDig"(.*?)data-test="footer"',
|
||||
html, re.DOTALL,
|
||||
)
|
||||
block = m_block.group(1) if m_block else ""
|
||||
desktop_m = re.search(r'visa-dagar__desktop(.*?)visa-dagar__mobile', block, re.DOTALL)
|
||||
target = desktop_m.group(1) if desktop_m else block
|
||||
return re.findall(r'\(\d+ dagar\)', target)
|
||||
|
||||
page = MagicMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
if sel == NIVA_SEL:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(return_value=_niva_all_texts())
|
||||
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value=_for_dig_kvar())
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "visa-dagar__discrete" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value=_for_dig_discrete())
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "dubbeldag" in sel and "visa-dagar__dagar-kvar" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value=_dubbeldag_kvar())
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
else:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
return page
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract from real HTML fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Import the extractor directly (no browser needed)
|
||||
from app.scrapers.fk_scraper import _extract_parental_leave # noqa: E402
|
||||
|
||||
|
||||
class TestExtractFromFixture:
|
||||
"""Parse the captured overview HTML and assert correct field values."""
|
||||
|
||||
async def test_days_left_and_total_parsed(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-1",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
# From the capture: 164 av 240 dagar
|
||||
assert result.days_left == 164
|
||||
assert result.total_days == 240
|
||||
assert result.days_used == 76
|
||||
|
||||
async def test_tier_values_parsed(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-2",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
# From the capture: sjukpenning 119, lägstanivå 45
|
||||
assert result.tier_data.sjukpenning_left == 119
|
||||
assert result.tier_data.lagsta_niva_left == 45
|
||||
|
||||
async def test_dubbeldagar_parsed(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-3",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
# From the capture: 39 av 60 dubbeldagar
|
||||
assert result.dubbeldagar_left == 39
|
||||
|
||||
async def test_source_is_scraped(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-4",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
assert result.source == "scraped"
|
||||
|
||||
async def test_reserved_days_derived(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-5",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
# 76 days used < 150 transferable → all reserved days remain
|
||||
assert result.reserved_days_total == 90
|
||||
assert result.reserved_days_left == 90
|
||||
assert result.transferable_days_left == 240 - 90 - 76 # 74
|
||||
|
||||
async def test_tier_totals_sum_to_total_days(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-6",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
assert result.tier_data.sjukpenning_total + result.tier_data.lagsta_niva_total == result.total_days
|
||||
|
||||
async def test_tier_used_nonnegative(self) -> None:
|
||||
html = _load_fixture("05_foraldrapenning_overview.html")
|
||||
state = _SessionState(
|
||||
session_id="test-7",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
page = _make_mock_page(html)
|
||||
result = await _extract_parental_leave(page, state)
|
||||
assert result.tier_data.sjukpenning_used >= 0
|
||||
assert result.tier_data.lagsta_niva_used >= 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation: bad numbers raise rather than persist
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidation:
|
||||
"""Feed impossible numbers and assert the extractor raises."""
|
||||
|
||||
def _make_bad_page(self, days_left: int, total_days: int, sjukpenning: int, lagsta: int, dubbel: int) -> Any:
|
||||
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
|
||||
page = MagicMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
if sel == NIVA_SEL:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(
|
||||
return_value=[f"({sjukpenning} dagar)", f"({lagsta} dagar)"]
|
||||
)
|
||||
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value=str(days_left))
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "visa-dagar__discrete" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(
|
||||
return_value=f"av {total_days} dagar kvar efter ansökan"
|
||||
)
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "dubbeldag" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value=str(dubbel))
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
else:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
return page
|
||||
|
||||
def _state(self) -> _SessionState:
|
||||
return _SessionState(
|
||||
session_id="val-test",
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
|
||||
async def test_negative_days_left_raises(self) -> None:
|
||||
page = self._make_bad_page(-1, 240, 100, 45, 39)
|
||||
with pytest.raises(ValueError, match="negative"):
|
||||
await _extract_parental_leave(page, self._state())
|
||||
|
||||
async def test_total_days_parse_failure_raises(self) -> None:
|
||||
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
page = MagicMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
if sel == NIVA_SEL:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(return_value=["(80 dagar)", "(20 dagar)"])
|
||||
elif "dagar-kvar" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="100")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "discrete" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="INVALID TEXT")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "dubbeldag" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="39")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
else:
|
||||
loc.first.inner_text = AsyncMock(return_value="0")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
with pytest.raises(ValueError, match="parse total days"):
|
||||
await _extract_parental_leave(page, self._state())
|
||||
|
||||
async def test_fewer_than_two_niva_values_raises(self) -> None:
|
||||
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
||||
page = MagicMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
if sel == NIVA_SEL:
|
||||
loc.first.inner_text = AsyncMock(return_value="")
|
||||
loc.all_inner_texts = AsyncMock(return_value=["(119 dagar)"]) # only one
|
||||
elif "dagar-kvar" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="164")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "discrete" in sel and "forDig" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="av 240 dagar kvar efter ansökan")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
elif "dubbeldag" in sel:
|
||||
loc.first.inner_text = AsyncMock(return_value="39")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
else:
|
||||
loc.first.inner_text = AsyncMock(return_value="0")
|
||||
loc.all_inner_texts = AsyncMock(return_value=[])
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
with pytest.raises(ValueError, match="Expected 2 tier values"):
|
||||
await _extract_parental_leave(page, self._state())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session state machine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSessionStateMachine:
|
||||
"""Session transitions, TTL watchdog, and double-cancel safety."""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
_sessions.clear()
|
||||
|
||||
def teardown_method(self) -> None:
|
||||
_sessions.clear()
|
||||
|
||||
async def test_start_session_creates_entry(self) -> None:
|
||||
service = FKScraperService()
|
||||
with patch("app.scrapers.fk_scraper._run_session", new_callable=AsyncMock) as mock_run:
|
||||
mock_run.return_value = None
|
||||
sid = await service.start_session(
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
assert sid in _sessions
|
||||
assert _sessions[sid].status == "launching"
|
||||
|
||||
async def test_poll_unknown_session_returns_error(self) -> None:
|
||||
service = FKScraperService()
|
||||
result = await service.poll_session("nonexistent")
|
||||
assert result["status"] == "error"
|
||||
assert "Unknown" in result["message"]
|
||||
|
||||
async def test_poll_awaiting_scan_returns_qr_data(self) -> None:
|
||||
service = FKScraperService()
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
status="awaiting_scan",
|
||||
qr_data=b"PNG_BYTES",
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
result = await service.poll_session(sid)
|
||||
assert result["status"] == "awaiting_scan"
|
||||
assert "qr_data" in result
|
||||
import base64
|
||||
assert base64.b64decode(result["qr_data"]) == b"PNG_BYTES"
|
||||
|
||||
async def test_cancel_sets_status(self) -> None:
|
||||
service = FKScraperService()
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
status="awaiting_scan",
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
await service.cancel_session(sid)
|
||||
assert _sessions[sid].status == "cancelled"
|
||||
|
||||
async def test_cancel_nonexistent_session_is_safe(self) -> None:
|
||||
service = FKScraperService()
|
||||
await service.cancel_session("does-not-exist") # should not raise
|
||||
|
||||
async def test_ttl_watchdog_cancels_stale_session(self) -> None:
|
||||
from app.scrapers.fk_scraper import _ttl_watchdog
|
||||
|
||||
with patch("app.scrapers.fk_scraper.settings") as mock_settings:
|
||||
mock_settings.scraper_session_ttl_seconds = 0 # immediate
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
status="awaiting_scan",
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
await _ttl_watchdog(sid)
|
||||
|
||||
assert _sessions[sid].status == "error"
|
||||
assert "timed out" in (_sessions[sid].message or "")
|
||||
|
||||
async def test_set_status_updates_message(self) -> None:
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
)
|
||||
_set_status(state, "error", "Something went wrong")
|
||||
assert state.status == "error"
|
||||
assert state.message == "Something went wrong"
|
||||
|
||||
async def test_do_extract_calls_persist_callback_on_success(self) -> None:
|
||||
"""_do_extract invokes the persist_callback with the extracted result."""
|
||||
persisted: list[ParentalLeaveFetch] = []
|
||||
|
||||
async def mock_persist(fetch: ParentalLeaveFetch) -> None:
|
||||
persisted.append(fetch)
|
||||
|
||||
now = datetime.now()
|
||||
expected = ParentalLeaveFetch(
|
||||
id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
workspace_id=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
source="scraped",
|
||||
fetched_at=now,
|
||||
total_days=240,
|
||||
days_used=76,
|
||||
days_left=164,
|
||||
reserved_days_total=90,
|
||||
reserved_days_left=90,
|
||||
transferable_days_left=74,
|
||||
dubbeldagar_left=39,
|
||||
tier_data=TierData(
|
||||
sjukpenning_total=195,
|
||||
sjukpenning_used=76,
|
||||
sjukpenning_left=119,
|
||||
lagsta_niva_total=45,
|
||||
lagsta_niva_used=0,
|
||||
lagsta_niva_left=45,
|
||||
),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
persist_callback=mock_persist,
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
page = MagicMock()
|
||||
page.wait_for_selector = AsyncMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
loc.first.click = AsyncMock()
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
|
||||
with patch(
|
||||
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
||||
) as mock_extract:
|
||||
mock_extract.return_value = expected
|
||||
await _do_extract(page, state)
|
||||
|
||||
assert state.status == "done"
|
||||
assert state.result is expected
|
||||
assert len(persisted) == 1
|
||||
assert persisted[0] is expected
|
||||
|
||||
async def test_do_extract_sets_error_when_persist_callback_fails(self) -> None:
|
||||
"""_do_extract sets status=error if the persist_callback raises."""
|
||||
|
||||
async def failing_persist(fetch: ParentalLeaveFetch) -> None:
|
||||
raise RuntimeError("DB unavailable")
|
||||
|
||||
now = datetime.now()
|
||||
dummy = ParentalLeaveFetch(
|
||||
id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
workspace_id=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
source="scraped",
|
||||
fetched_at=now,
|
||||
total_days=240,
|
||||
days_used=0,
|
||||
days_left=240,
|
||||
reserved_days_total=90,
|
||||
reserved_days_left=90,
|
||||
transferable_days_left=150,
|
||||
dubbeldagar_left=60,
|
||||
tier_data=TierData(195, 0, 195, 45, 0, 45),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=uuid.uuid4(),
|
||||
personal_data_uuid=uuid.uuid4(),
|
||||
child_id=uuid.uuid4(),
|
||||
persist_callback=failing_persist,
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
page = MagicMock()
|
||||
page.wait_for_selector = AsyncMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
loc.first.click = AsyncMock()
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
|
||||
with patch(
|
||||
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
||||
) as mock_extract:
|
||||
mock_extract.return_value = dummy
|
||||
await _do_extract(page, state)
|
||||
|
||||
assert state.status == "error"
|
||||
assert "persist" in (state.message or "").lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence integration: real SQLite repo round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPersistenceIntegration:
|
||||
"""Run _do_extract with a real SQLiteParentalLeaveRepository and verify the row."""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
_sessions.clear()
|
||||
|
||||
def teardown_method(self) -> None:
|
||||
_sessions.clear()
|
||||
|
||||
async def test_scraped_row_is_readable_after_session_done(
|
||||
self, parental_leave_repo, seeded_ids
|
||||
) -> None:
|
||||
"""After _do_extract completes, repo.get_latest_by_parent returns the scraped row."""
|
||||
from app.repositories_impl.sqlite.parental_leave_repository import (
|
||||
SQLiteParentalLeaveRepository,
|
||||
)
|
||||
|
||||
async def persist_callback(fetch: ParentalLeaveFetch) -> None:
|
||||
await parental_leave_repo.create(fetch)
|
||||
|
||||
now = datetime.now()
|
||||
scraped = ParentalLeaveFetch(
|
||||
id=uuid.uuid4(),
|
||||
personal_data_uuid=seeded_ids.personal_data_uuid,
|
||||
workspace_id=seeded_ids.workspace_id,
|
||||
child_id=seeded_ids.child_id,
|
||||
source="scraped",
|
||||
fetched_at=now,
|
||||
total_days=240,
|
||||
days_used=76,
|
||||
days_left=164,
|
||||
reserved_days_total=90,
|
||||
reserved_days_left=90,
|
||||
transferable_days_left=74,
|
||||
dubbeldagar_left=39,
|
||||
tier_data=TierData(195, 76, 119, 45, 0, 45),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
state = _SessionState(
|
||||
session_id=sid,
|
||||
workspace_id=seeded_ids.workspace_id,
|
||||
personal_data_uuid=seeded_ids.personal_data_uuid,
|
||||
child_id=seeded_ids.child_id,
|
||||
persist_callback=persist_callback,
|
||||
)
|
||||
_sessions[sid] = state
|
||||
|
||||
page = MagicMock()
|
||||
page.wait_for_selector = AsyncMock()
|
||||
|
||||
def locator_factory(sel: str) -> MagicMock:
|
||||
loc = MagicMock()
|
||||
loc.first.click = AsyncMock()
|
||||
return loc
|
||||
|
||||
page.locator.side_effect = locator_factory
|
||||
|
||||
with patch(
|
||||
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
||||
) as mock_extract:
|
||||
mock_extract.return_value = scraped
|
||||
await _do_extract(page, state)
|
||||
|
||||
assert state.status == "done"
|
||||
|
||||
result = await parental_leave_repo.get_latest_by_parent(
|
||||
seeded_ids.personal_data_uuid, seeded_ids.child_id
|
||||
)
|
||||
assert result is not None
|
||||
assert result.source == "scraped"
|
||||
assert result.days_left == 164
|
||||
assert result.total_days == 240
|
||||
Generated
+62
-1
@@ -236,6 +236,28 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dnspython"
|
||||
version = "2.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "email-validator"
|
||||
version = "2.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "dnspython" },
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastapi"
|
||||
version = "0.136.3"
|
||||
@@ -263,7 +285,8 @@ dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "greenlet" },
|
||||
{ name = "holidays" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "playwright" },
|
||||
{ name = "pydantic", extra = ["email"] },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "python-dateutil" },
|
||||
@@ -294,7 +317,9 @@ requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.115" },
|
||||
{ name = "greenlet", specifier = ">=3.1" },
|
||||
{ name = "holidays", specifier = ">=0.62" },
|
||||
{ name = "playwright", specifier = ">=1.47" },
|
||||
{ name = "pydantic", specifier = ">=2.9" },
|
||||
{ name = "pydantic", extras = ["email"], specifier = ">=2.9" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.5" },
|
||||
{ name = "pyjwt", specifier = ">=2.9" },
|
||||
{ name = "python-dateutil", specifier = ">=2.9" },
|
||||
@@ -513,6 +538,25 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "playwright"
|
||||
version = "1.60.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "pyee" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/21/f0/832bd9677194908da118064eef20082f2791e3d18215cc6d9391ee2c5a67/playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7", size = 43474635, upload-time = "2026-05-18T12:00:31.969Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/7b/e1d32ae8a3ed937ec2be3721c5f728b13d731a0b7c6442e0b3bec5094ac0/playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5", size = 42261327, upload-time = "2026-05-18T12:00:35.638Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/bc/23de499ded6411c188a20c5a0dea6f0cd4ed5d2b3cc6042a5dbd3ed609aa/playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705", size = 43474636, upload-time = "2026-05-18T12:00:39.294Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/7b/1d679f4fced4ea94efadd17103856d8c565384f68382a1681264e46f5925/playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e", size = 47467220, upload-time = "2026-05-18T12:00:43.179Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/c2/1528d267d4442bd2c6b8eaeab819dd52c2030bf80e89293f0ba1f687473b/playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353", size = 47154856, upload-time = "2026-05-18T12:00:46.715Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/4e/b008b6440a7a1624378041da94829956d4b8f7ab9ef5aad22d0dc3f2e26d/playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7", size = 37902157, upload-time = "2026-05-18T12:00:50.374Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/f0/0541524133104f9cc20bf900870ff4a736b76a23483f3a55295ddfa58409/playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02", size = 37902159, upload-time = "2026-05-18T12:00:53.728Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/c8/210f282d278e4709cdd71b12a31af45a30a22ab3207b387e29b37e478713/playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537", size = 34037981, upload-time = "2026-05-18T12:00:57.584Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
@@ -546,6 +590,11 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
email = [
|
||||
{ name = "email-validator" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic-core"
|
||||
version = "2.46.4"
|
||||
@@ -601,6 +650,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyee"
|
||||
version = "13.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.20.0"
|
||||
|
||||
Reference in New Issue
Block a user