jacobh2/F-r-ldraledigplannerare
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Styling fix and scraper bug fix

Browse Source
This commit is contained in:
Jacob Hagstedt
2026-06-14 18:39:55 +02:00
parent 6ed2a3ae0c
commit 566595984f
21 changed files with 2150 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/scrapers/fk_scraper.py
+36 -5
View File
@@ -229,7 +229,9 @@ async def _check_maintenance(page) -> str | None:
# Found maintenance notice; extract the message
# The message is in a nearby .sv-text-portlet-content div
message_el = page.locator("h2.rubrik:has-text('Tekniskt underhål') ~ div .sv-text-portlet-content")
message_el = page.locator(
"h2.rubrik:has-text('Tekniskt underhål') ~ div .sv-text-portlet-content"
)
if await message_el.count() > 0:
message_text = await message_el.first.inner_text(timeout=3_000)
return message_text.strip()
@@ -261,7 +263,8 @@ async def _save_debug_artifacts(page, session_id: str, console_messages: list[st
logger.info(
"scraper debug artifacts saved [id=%s] dir=%s",
session_id, out.resolve(),
session_id,
out.resolve(),
)
except Exception as artifact_exc:
logger.warning("failed to save debug artifacts [id=%s]: %s", session_id, artifact_exc)
@@ -373,7 +376,9 @@ async def _do_login(page, state: _SessionState) -> None:
return
_set_status(state, "authenticating")
logger.debug("[%s] authenticating — waiting for Ansök button (url=%s)", state.session_id, page.url)
logger.debug(
"[%s] authenticating — waiting for Ansök button (url=%s)", state.session_id, page.url
)
# Wait for post-login landing (the Ansök button appears after JS hydrates the page)
await page.wait_for_selector(_SEL.ANSOK_BUTTON, timeout=30_000)
@@ -393,6 +398,22 @@ async def _do_extract(page, state: _SessionState) -> None:
await page.wait_for_selector(_SEL.STEP_DAGAR_KVAR, timeout=30_000)
logger.debug("[%s] valjdagar page loaded (url=%s)", state.session_id, page.url)
# The panel mounts with placeholder zeros, then hydrates from an async fetch.
# Wait until the "av N dagar" total is positive — total is always 240/480, never 0,
# so this is a reliable "real data has arrived" signal (days_left can legitimately be 0).
logger.debug("[%s] waiting for valjdagar data to hydrate", state.session_id)
await page.wait_for_function(
"""(sel) => {
const el = document.querySelector(sel);
if (!el) return false;
const m = el.textContent.match(/av\\s+(\\d+)\\s+dagar/);
return m && parseInt(m[1], 10) > 0;
}""",
arg=_SEL.FOR_DIG_DISCRETE,
timeout=30_000,
)
logger.debug("[%s] valjdagar data hydrated (url=%s)", state.session_id, page.url)
data = await _extract_parental_leave(page, state)
if state.persist_callback is not None:
@@ -428,6 +449,11 @@ async def _extract_parental_leave(page, state: _SessionState) -> ParentalLeaveFe
if not total_match:
raise ValueError(f"Could not parse total days from: {discrete_text!r}")
total_days = int(total_match.group(1))
if total_days == 0:
raise ValueError(
"FK panel returned total_days=0 — data had not hydrated when read "
"(placeholder state). The hydration wait in _do_extract should prevent this."
)
days_used = total_days - days_left
logger.debug("[%s] total_days=%d days_used=%d", state.session_id, total_days, days_used)
@@ -452,14 +478,19 @@ async def _extract_parental_leave(page, state: _SessionState) -> ParentalLeaveFe
sjukpenning_left, lagsta_niva_left = niva_numbers[0], niva_numbers[1]
logger.debug(
"[%s] sjukpenning_left=%d lagsta_niva_left=%d",
state.session_id, sjukpenning_left, lagsta_niva_left,
state.session_id,
sjukpenning_left,
lagsta_niva_left,
)
# Cross-check: sjukpenning_left + lagsta_niva_left should equal days_left
if sjukpenning_left + lagsta_niva_left != days_left:
logger.warning(
"Tier sum mismatch: sjukpenning=%d + lagstaniva=%d != days_left=%d [id=%s]",
sjukpenning_left, lagsta_niva_left, days_left, state.session_id,
sjukpenning_left,
lagsta_niva_left,
days_left,
state.session_id,
)
# -- dubbeldagar remaining --
backend/tests/unit/test_fk_scraper.py
+102 -12
View File
@@ -15,7 +15,7 @@ from __future__ import annotations
import re
import uuid
from datetime import date, datetime
from datetime import datetime
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -65,7 +65,8 @@ def _make_mock_page(html: str):
m = re.search(
r'data-test="forDig".*?'
r'class="visa-dagar__dagar-kvar">(\d+)<',
html, re.DOTALL,
html,
re.DOTALL,
)
if m:
return m.group(1)
@@ -75,7 +76,8 @@ def _make_mock_page(html: str):
m = re.search(
r'data-test="forDig".*?'
r'class="visa-dagar__discrete">(.*?)<',
html, re.DOTALL,
html,
re.DOTALL,
)
if m:
return m.group(1)
@@ -85,7 +87,8 @@ def _make_mock_page(html: str):
m = re.search(
r'data-test="dubbeldag".*?'
r'class="visa-dagar__dagar-kvar">(\d+)<',
html, re.DOTALL,
html,
re.DOTALL,
)
if m:
return m.group(1)
@@ -94,12 +97,13 @@ def _make_mock_page(html: str):
def _niva_all_texts() -> list[str]:
m_block = re.search(
r'data-test="forDig"(.*?)data-test="footer"',
html, re.DOTALL,
html,
re.DOTALL,
)
block = m_block.group(1) if m_block else ""
desktop_m = re.search(r'visa-dagar__desktop(.*?)visa-dagar__mobile', block, re.DOTALL)
desktop_m = re.search(r"visa-dagar__desktop(.*?)visa-dagar__mobile", block, re.DOTALL)
target = desktop_m.group(1) if desktop_m else block
return re.findall(r'\(\d+ dagar\)', target)
return re.findall(r"\(\d+ dagar\)", target)
page = MagicMock()
@@ -107,18 +111,23 @@ def _make_mock_page(html: str):
loc = MagicMock()
if sel == NIVA_SEL:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=_niva_all_texts())
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value=_for_dig_kvar())
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "visa-dagar__discrete" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value=_for_dig_discrete())
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "dubbeldag" in sel and "visa-dagar__dagar-kvar" in sel:
loc.first.inner_text = AsyncMock(return_value=_dubbeldag_kvar())
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
else:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
return loc
@@ -216,7 +225,10 @@ class TestExtractFromFixture:
)
page = _make_mock_page(html)
result = await _extract_parental_leave(page, state)
assert result.tier_data.sjukpenning_total + result.tier_data.lagsta_niva_total == result.total_days
assert (
result.tier_data.sjukpenning_total + result.tier_data.lagsta_niva_total
== result.total_days
)
async def test_tier_used_nonnegative(self) -> None:
html = _load_fixture("05_foraldrapenning_overview.html")
@@ -240,7 +252,9 @@ class TestExtractFromFixture:
class TestValidation:
"""Feed impossible numbers and assert the extractor raises."""
def _make_bad_page(self, days_left: int, total_days: int, sjukpenning: int, lagsta: int, dubbel: int) -> Any:
def _make_bad_page(
self, days_left: int, total_days: int, sjukpenning: int, lagsta: int, dubbel: int
) -> Any:
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
page = MagicMock()
@@ -249,22 +263,27 @@ class TestValidation:
loc = MagicMock()
if sel == NIVA_SEL:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(
return_value=[f"({sjukpenning} dagar)", f"({lagsta} dagar)"]
)
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value=str(days_left))
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "visa-dagar__discrete" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(
return_value=f"av {total_days} dagar kvar efter ansökan"
)
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "dubbeldag" in sel:
loc.first.inner_text = AsyncMock(return_value=str(dubbel))
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
else:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
return loc
@@ -292,18 +311,23 @@ class TestValidation:
loc = MagicMock()
if sel == NIVA_SEL:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=["(80 dagar)", "(20 dagar)"])
elif "dagar-kvar" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value="100")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "discrete" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value="INVALID TEXT")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "dubbeldag" in sel:
loc.first.inner_text = AsyncMock(return_value="39")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
else:
loc.first.inner_text = AsyncMock(return_value="0")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
return loc
@@ -319,18 +343,23 @@ class TestValidation:
loc = MagicMock()
if sel == NIVA_SEL:
loc.first.inner_text = AsyncMock(return_value="")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=["(119 dagar)"]) # only one
elif "dagar-kvar" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value="164")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "discrete" in sel and "forDig" in sel:
loc.first.inner_text = AsyncMock(return_value="av 240 dagar kvar efter ansökan")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
elif "dubbeldag" in sel:
loc.first.inner_text = AsyncMock(return_value="39")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
else:
loc.first.inner_text = AsyncMock(return_value="0")
loc.first.wait_for = AsyncMock()
loc.all_inner_texts = AsyncMock(return_value=[])
return loc
@@ -338,6 +367,12 @@ class TestValidation:
with pytest.raises(ValueError, match="Expected 2 tier values"):
await _extract_parental_leave(page, self._state())
async def test_zero_total_days_raises(self) -> None:
"""Guard against placeholder zero data: total_days=0 is impossible and indicates hydration failure."""
page = self._make_bad_page(164, 0, 119, 45, 39)
with pytest.raises(ValueError, match="total_days=0.*had not hydrated"):
await _extract_parental_leave(page, self._state())
# ---------------------------------------------------------------------------
# Session state machine
@@ -388,6 +423,7 @@ class TestSessionStateMachine:
assert result["status"] == "awaiting_scan"
assert "qr_data" in result
import base64
assert base64.b64decode(result["qr_data"]) == b"PNG_BYTES"
async def test_cancel_sets_status(self) -> None:
@@ -487,6 +523,7 @@ class TestSessionStateMachine:
page = MagicMock()
page.wait_for_selector = AsyncMock()
page.wait_for_function = AsyncMock()
def locator_factory(sel: str) -> MagicMock:
loc = MagicMock()
@@ -544,6 +581,7 @@ class TestSessionStateMachine:
page = MagicMock()
page.wait_for_selector = AsyncMock()
page.wait_for_function = AsyncMock()
def locator_factory(sel: str) -> MagicMock:
loc = MagicMock()
@@ -561,6 +599,60 @@ class TestSessionStateMachine:
assert state.status == "error"
assert "persist" in (state.message or "").lower()
async def test_do_extract_waits_for_hydration_before_extracting(self) -> None:
"""_do_extract calls wait_for_function to ensure data hydrates before extraction."""
now = datetime.now()
expected = ParentalLeaveFetch(
id=uuid.uuid4(),
personal_data_uuid=uuid.uuid4(),
workspace_id=uuid.uuid4(),
child_id=uuid.uuid4(),
source="scraped",
fetched_at=now,
total_days=240,
days_used=0,
days_left=240,
reserved_days_total=90,
reserved_days_left=90,
transferable_days_left=150,
dubbeldagar_left=60,
tier_data=TierData(195, 0, 195, 45, 0, 45),
created_at=now,
updated_at=now,
)
sid = str(uuid.uuid4())
state = _SessionState(
session_id=sid,
workspace_id=uuid.uuid4(),
personal_data_uuid=uuid.uuid4(),
child_id=uuid.uuid4(),
)
_sessions[sid] = state
page = MagicMock()
page.wait_for_selector = AsyncMock()
page.wait_for_function = AsyncMock()
def locator_factory(sel: str) -> MagicMock:
loc = MagicMock()
loc.first.click = AsyncMock()
return loc
page.locator.side_effect = locator_factory
with patch(
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
) as mock_extract:
mock_extract.return_value = expected
await _do_extract(page, state)
# Verify wait_for_function was called with the correct selector
page.wait_for_function.assert_called_once()
call_args = page.wait_for_function.call_args
assert call_args[1]["arg"] == "[data-test='forDig'] .visa-dagar__discrete"
assert call_args[1]["timeout"] == 30_000
# ---------------------------------------------------------------------------
# Persistence integration: real SQLite repo round-trip
@@ -580,9 +672,6 @@ class TestPersistenceIntegration:
self, parental_leave_repo, seeded_ids
) -> None:
"""After _do_extract completes, repo.get_latest_by_parent returns the scraped row."""
from app.repositories_impl.sqlite.parental_leave_repository import (
SQLiteParentalLeaveRepository,
)
async def persist_callback(fetch: ParentalLeaveFetch) -> None:
await parental_leave_repo.create(fetch)
@@ -619,6 +708,7 @@ class TestPersistenceIntegration:
page = MagicMock()
page.wait_for_selector = AsyncMock()
page.wait_for_function = AsyncMock()
def locator_factory(sel: str) -> MagicMock:
loc = MagicMock()
backend/tests/unit/test_planning_service.py
+2 -2
View File
@@ -154,14 +154,14 @@ class TestPlanningService:
ws = seeded_ids.workspace_id
child_id = seeded_ids.child_id
d1 = await planning_service.add_planned_day(
await planning_service.add_planned_day(
workspace_id=ws,
child_id=child_id,
parent_personal_data_uuid=seeded_ids.personal_data_uuid,
date=date(2026, 6, 15),
day_type="vacation",
)
d2 = await planning_service.add_planned_day(
await planning_service.add_planned_day(
workspace_id=ws,
child_id=child_id,
parent_personal_data_uuid=seeded_ids.personal_data_uuid,