735 lines
27 KiB
Python
735 lines
27 KiB
Python
"""Tests for the FK scraper: extractor, validation, and session state machine.
|
|
|
|
Three independent test concerns:
|
|
|
|
1. _extract_parental_leave — parses the FK #/valjdagar HTML fixture and
|
|
produces a correct ParentalLeaveFetch. No network, no BankID.
|
|
|
|
2. Validation helpers — out-of-bounds numbers raise rather than persist.
|
|
|
|
3. Session state machine — start/poll/cancel transitions, TTL auto-cancel,
|
|
double-cancel safety. Browser is mocked; no headless Chromium needed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import uuid
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from app.entities.parental_leave import ParentalLeaveFetch, TierData
|
|
from app.scrapers.fk_scraper import (
|
|
FKScraperService,
|
|
_SessionState,
|
|
_do_extract,
|
|
_set_status,
|
|
_sessions,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers for the HTML-fixture tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CAPTURES_DIR = Path(__file__).parents[3] / "captures"
|
|
|
|
|
|
def _load_fixture(name: str) -> str:
|
|
"""Read a page capture from captures/, skipping if not present."""
|
|
path = CAPTURES_DIR / name
|
|
if not path.exists():
|
|
pytest.skip(f"Fixture {name} not found — run the captures step first")
|
|
return path.read_text(encoding="utf-8")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Minimal mock of a Playwright Page that answers our CSS queries from HTML
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_mock_page(html: str):
|
|
"""
|
|
Build a minimal async mock page that serves inner_text / all_inner_texts
|
|
for the selectors used by _extract_parental_leave.
|
|
|
|
Uses basic regex over the fixture HTML rather than a real browser — fast
|
|
and dependency-free.
|
|
"""
|
|
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
|
|
|
def _for_dig_kvar() -> str:
|
|
m = re.search(
|
|
r'data-test="forDig".*?'
|
|
r'class="visa-dagar__dagar-kvar">(\d+)<',
|
|
html,
|
|
re.DOTALL,
|
|
)
|
|
if m:
|
|
return m.group(1)
|
|
raise ValueError("No forDig dagar-kvar found in fixture")
|
|
|
|
def _for_dig_discrete() -> str:
|
|
m = re.search(
|
|
r'data-test="forDig".*?'
|
|
r'class="visa-dagar__discrete">(.*?)<',
|
|
html,
|
|
re.DOTALL,
|
|
)
|
|
if m:
|
|
return m.group(1)
|
|
raise ValueError("No forDig discrete found in fixture")
|
|
|
|
def _dubbeldag_kvar() -> str:
|
|
m = re.search(
|
|
r'data-test="dubbeldag".*?'
|
|
r'class="visa-dagar__dagar-kvar">(\d+)<',
|
|
html,
|
|
re.DOTALL,
|
|
)
|
|
if m:
|
|
return m.group(1)
|
|
raise ValueError("No dubbeldag dagar-kvar found in fixture")
|
|
|
|
def _niva_all_texts() -> list[str]:
|
|
m_block = re.search(
|
|
r'data-test="forDig"(.*?)data-test="footer"',
|
|
html,
|
|
re.DOTALL,
|
|
)
|
|
block = m_block.group(1) if m_block else ""
|
|
desktop_m = re.search(r"visa-dagar__desktop(.*?)visa-dagar__mobile", block, re.DOTALL)
|
|
target = desktop_m.group(1) if desktop_m else block
|
|
return re.findall(r"\(\d+ dagar\)", target)
|
|
|
|
page = MagicMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
if sel == NIVA_SEL:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=_niva_all_texts())
|
|
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value=_for_dig_kvar())
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "visa-dagar__discrete" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value=_for_dig_discrete())
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "dubbeldag" in sel and "visa-dagar__dagar-kvar" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value=_dubbeldag_kvar())
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
else:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
return page
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extract from real HTML fixture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Import the extractor directly (no browser needed)
|
|
from app.scrapers.fk_scraper import _extract_parental_leave # noqa: E402
|
|
|
|
|
|
class TestExtractFromFixture:
|
|
"""Parse the captured overview HTML and assert correct field values."""
|
|
|
|
async def test_days_left_and_total_parsed(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-1",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
# From the capture: 164 av 240 dagar
|
|
assert result.days_left == 164
|
|
assert result.total_days == 240
|
|
assert result.days_used == 76
|
|
|
|
async def test_tier_values_parsed(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-2",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
# From the capture: sjukpenning 119, lägstanivå 45
|
|
assert result.tier_data.sjukpenning_left == 119
|
|
assert result.tier_data.lagsta_niva_left == 45
|
|
|
|
async def test_dubbeldagar_parsed(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-3",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
# From the capture: 39 av 60 dubbeldagar
|
|
assert result.dubbeldagar_left == 39
|
|
|
|
async def test_source_is_scraped(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-4",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
assert result.source == "scraped"
|
|
|
|
async def test_reserved_days_derived(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-5",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
# 76 days used < 150 transferable → all reserved days remain
|
|
assert result.reserved_days_total == 90
|
|
assert result.reserved_days_left == 90
|
|
assert result.transferable_days_left == 240 - 90 - 76 # 74
|
|
|
|
async def test_tier_totals_sum_to_total_days(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-6",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
assert (
|
|
result.tier_data.sjukpenning_total + result.tier_data.lagsta_niva_total
|
|
== result.total_days
|
|
)
|
|
|
|
async def test_tier_used_nonnegative(self) -> None:
|
|
html = _load_fixture("05_foraldrapenning_overview.html")
|
|
state = _SessionState(
|
|
session_id="test-7",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
page = _make_mock_page(html)
|
|
result = await _extract_parental_leave(page, state)
|
|
assert result.tier_data.sjukpenning_used >= 0
|
|
assert result.tier_data.lagsta_niva_used >= 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation: bad numbers raise rather than persist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidation:
|
|
"""Feed impossible numbers and assert the extractor raises."""
|
|
|
|
def _make_bad_page(
|
|
self, days_left: int, total_days: int, sjukpenning: int, lagsta: int, dubbel: int
|
|
) -> Any:
|
|
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
|
|
|
page = MagicMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
if sel == NIVA_SEL:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(
|
|
return_value=[f"({sjukpenning} dagar)", f"({lagsta} dagar)"]
|
|
)
|
|
elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value=str(days_left))
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "visa-dagar__discrete" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(
|
|
return_value=f"av {total_days} dagar kvar efter ansökan"
|
|
)
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "dubbeldag" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value=str(dubbel))
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
else:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
return page
|
|
|
|
def _state(self) -> _SessionState:
|
|
return _SessionState(
|
|
session_id="val-test",
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
|
|
async def test_negative_days_left_raises(self) -> None:
|
|
page = self._make_bad_page(-1, 240, 100, 45, 39)
|
|
with pytest.raises(ValueError, match="negative"):
|
|
await _extract_parental_leave(page, self._state())
|
|
|
|
async def test_total_days_parse_failure_raises(self) -> None:
|
|
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
|
page = MagicMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
if sel == NIVA_SEL:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=["(80 dagar)", "(20 dagar)"])
|
|
elif "dagar-kvar" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="100")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "discrete" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="INVALID TEXT")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "dubbeldag" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="39")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
else:
|
|
loc.first.inner_text = AsyncMock(return_value="0")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
with pytest.raises(ValueError, match="parse total days"):
|
|
await _extract_parental_leave(page, self._state())
|
|
|
|
async def test_fewer_than_two_niva_values_raises(self) -> None:
|
|
NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span"
|
|
page = MagicMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
if sel == NIVA_SEL:
|
|
loc.first.inner_text = AsyncMock(return_value="")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=["(119 dagar)"]) # only one
|
|
elif "dagar-kvar" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="164")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "discrete" in sel and "forDig" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="av 240 dagar kvar efter ansökan")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
elif "dubbeldag" in sel:
|
|
loc.first.inner_text = AsyncMock(return_value="39")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
else:
|
|
loc.first.inner_text = AsyncMock(return_value="0")
|
|
loc.first.wait_for = AsyncMock()
|
|
loc.all_inner_texts = AsyncMock(return_value=[])
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
with pytest.raises(ValueError, match="Expected 2 tier values"):
|
|
await _extract_parental_leave(page, self._state())
|
|
|
|
async def test_zero_total_days_raises(self) -> None:
|
|
"""Guard against placeholder zero data: total_days=0 is impossible and indicates hydration failure."""
|
|
page = self._make_bad_page(164, 0, 119, 45, 39)
|
|
with pytest.raises(ValueError, match="total_days=0.*had not hydrated"):
|
|
await _extract_parental_leave(page, self._state())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Session state machine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSessionStateMachine:
|
|
"""Session transitions, TTL watchdog, and double-cancel safety."""
|
|
|
|
def setup_method(self) -> None:
|
|
_sessions.clear()
|
|
|
|
def teardown_method(self) -> None:
|
|
_sessions.clear()
|
|
|
|
async def test_start_session_creates_entry(self) -> None:
|
|
service = FKScraperService()
|
|
with patch("app.scrapers.fk_scraper._run_session", new_callable=AsyncMock) as mock_run:
|
|
mock_run.return_value = None
|
|
sid = await service.start_session(
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
assert sid in _sessions
|
|
assert _sessions[sid].status == "launching"
|
|
|
|
async def test_poll_unknown_session_returns_error(self) -> None:
|
|
service = FKScraperService()
|
|
result = await service.poll_session("nonexistent")
|
|
assert result["status"] == "error"
|
|
assert "Unknown" in result["message"]
|
|
|
|
async def test_poll_awaiting_scan_returns_qr_data(self) -> None:
|
|
service = FKScraperService()
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
status="awaiting_scan",
|
|
qr_data=b"PNG_BYTES",
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
result = await service.poll_session(sid)
|
|
assert result["status"] == "awaiting_scan"
|
|
assert "qr_data" in result
|
|
import base64
|
|
|
|
assert base64.b64decode(result["qr_data"]) == b"PNG_BYTES"
|
|
|
|
async def test_cancel_sets_status(self) -> None:
|
|
service = FKScraperService()
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
status="awaiting_scan",
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
await service.cancel_session(sid)
|
|
assert _sessions[sid].status == "cancelled"
|
|
|
|
async def test_cancel_nonexistent_session_is_safe(self) -> None:
|
|
service = FKScraperService()
|
|
await service.cancel_session("does-not-exist") # should not raise
|
|
|
|
async def test_ttl_watchdog_cancels_stale_session(self) -> None:
|
|
from app.scrapers.fk_scraper import _ttl_watchdog
|
|
|
|
with patch("app.scrapers.fk_scraper.settings") as mock_settings:
|
|
mock_settings.scraper_session_ttl_seconds = 0 # immediate
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
status="awaiting_scan",
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
await _ttl_watchdog(sid)
|
|
|
|
assert _sessions[sid].status == "error"
|
|
assert "timed out" in (_sessions[sid].message or "")
|
|
|
|
async def test_set_status_updates_message(self) -> None:
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
_set_status(state, "error", "Something went wrong")
|
|
assert state.status == "error"
|
|
assert state.message == "Something went wrong"
|
|
|
|
async def test_do_extract_calls_persist_callback_on_success(self) -> None:
|
|
"""_do_extract invokes the persist_callback with the extracted result."""
|
|
persisted: list[ParentalLeaveFetch] = []
|
|
|
|
async def mock_persist(fetch: ParentalLeaveFetch) -> None:
|
|
persisted.append(fetch)
|
|
|
|
now = datetime.now()
|
|
expected = ParentalLeaveFetch(
|
|
id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
workspace_id=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
source="scraped",
|
|
fetched_at=now,
|
|
total_days=240,
|
|
days_used=76,
|
|
days_left=164,
|
|
reserved_days_total=90,
|
|
reserved_days_left=90,
|
|
transferable_days_left=74,
|
|
dubbeldagar_left=39,
|
|
tier_data=TierData(
|
|
sjukpenning_total=195,
|
|
sjukpenning_used=76,
|
|
sjukpenning_left=119,
|
|
lagsta_niva_total=45,
|
|
lagsta_niva_used=0,
|
|
lagsta_niva_left=45,
|
|
),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
persist_callback=mock_persist,
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
page = MagicMock()
|
|
page.wait_for_selector = AsyncMock()
|
|
page.wait_for_function = AsyncMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
loc.first.click = AsyncMock()
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
|
|
with patch(
|
|
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
|
) as mock_extract:
|
|
mock_extract.return_value = expected
|
|
await _do_extract(page, state)
|
|
|
|
assert state.status == "done"
|
|
assert state.result is expected
|
|
assert len(persisted) == 1
|
|
assert persisted[0] is expected
|
|
|
|
async def test_do_extract_sets_error_when_persist_callback_fails(self) -> None:
|
|
"""_do_extract sets status=error if the persist_callback raises."""
|
|
|
|
async def failing_persist(fetch: ParentalLeaveFetch) -> None:
|
|
raise RuntimeError("DB unavailable")
|
|
|
|
now = datetime.now()
|
|
dummy = ParentalLeaveFetch(
|
|
id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
workspace_id=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
source="scraped",
|
|
fetched_at=now,
|
|
total_days=240,
|
|
days_used=0,
|
|
days_left=240,
|
|
reserved_days_total=90,
|
|
reserved_days_left=90,
|
|
transferable_days_left=150,
|
|
dubbeldagar_left=60,
|
|
tier_data=TierData(195, 0, 195, 45, 0, 45),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
persist_callback=failing_persist,
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
page = MagicMock()
|
|
page.wait_for_selector = AsyncMock()
|
|
page.wait_for_function = AsyncMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
loc.first.click = AsyncMock()
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
|
|
with patch(
|
|
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
|
) as mock_extract:
|
|
mock_extract.return_value = dummy
|
|
await _do_extract(page, state)
|
|
|
|
assert state.status == "error"
|
|
assert "persist" in (state.message or "").lower()
|
|
|
|
async def test_do_extract_waits_for_hydration_before_extracting(self) -> None:
|
|
"""_do_extract calls wait_for_function to ensure data hydrates before extraction."""
|
|
now = datetime.now()
|
|
expected = ParentalLeaveFetch(
|
|
id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
workspace_id=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
source="scraped",
|
|
fetched_at=now,
|
|
total_days=240,
|
|
days_used=0,
|
|
days_left=240,
|
|
reserved_days_total=90,
|
|
reserved_days_left=90,
|
|
transferable_days_left=150,
|
|
dubbeldagar_left=60,
|
|
tier_data=TierData(195, 0, 195, 45, 0, 45),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=uuid.uuid4(),
|
|
personal_data_uuid=uuid.uuid4(),
|
|
child_id=uuid.uuid4(),
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
page = MagicMock()
|
|
page.wait_for_selector = AsyncMock()
|
|
page.wait_for_function = AsyncMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
loc.first.click = AsyncMock()
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
|
|
with patch(
|
|
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
|
) as mock_extract:
|
|
mock_extract.return_value = expected
|
|
await _do_extract(page, state)
|
|
|
|
# Verify wait_for_function was called with the correct selector
|
|
page.wait_for_function.assert_called_once()
|
|
call_args = page.wait_for_function.call_args
|
|
assert call_args[1]["arg"] == "[data-test='forDig'] .visa-dagar__discrete"
|
|
assert call_args[1]["timeout"] == 30_000
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Persistence integration: real SQLite repo round-trip
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPersistenceIntegration:
|
|
"""Run _do_extract with a real SQLiteParentalLeaveRepository and verify the row."""
|
|
|
|
def setup_method(self) -> None:
|
|
_sessions.clear()
|
|
|
|
def teardown_method(self) -> None:
|
|
_sessions.clear()
|
|
|
|
async def test_scraped_row_is_readable_after_session_done(
|
|
self, parental_leave_repo, seeded_ids
|
|
) -> None:
|
|
"""After _do_extract completes, repo.get_latest_by_parent returns the scraped row."""
|
|
|
|
async def persist_callback(fetch: ParentalLeaveFetch) -> None:
|
|
await parental_leave_repo.create(fetch)
|
|
|
|
now = datetime.now()
|
|
scraped = ParentalLeaveFetch(
|
|
id=uuid.uuid4(),
|
|
personal_data_uuid=seeded_ids.personal_data_uuid,
|
|
workspace_id=seeded_ids.workspace_id,
|
|
child_id=seeded_ids.child_id,
|
|
source="scraped",
|
|
fetched_at=now,
|
|
total_days=240,
|
|
days_used=76,
|
|
days_left=164,
|
|
reserved_days_total=90,
|
|
reserved_days_left=90,
|
|
transferable_days_left=74,
|
|
dubbeldagar_left=39,
|
|
tier_data=TierData(195, 76, 119, 45, 0, 45),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
sid = str(uuid.uuid4())
|
|
state = _SessionState(
|
|
session_id=sid,
|
|
workspace_id=seeded_ids.workspace_id,
|
|
personal_data_uuid=seeded_ids.personal_data_uuid,
|
|
child_id=seeded_ids.child_id,
|
|
persist_callback=persist_callback,
|
|
)
|
|
_sessions[sid] = state
|
|
|
|
page = MagicMock()
|
|
page.wait_for_selector = AsyncMock()
|
|
page.wait_for_function = AsyncMock()
|
|
|
|
def locator_factory(sel: str) -> MagicMock:
|
|
loc = MagicMock()
|
|
loc.first.click = AsyncMock()
|
|
return loc
|
|
|
|
page.locator.side_effect = locator_factory
|
|
|
|
with patch(
|
|
"app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock
|
|
) as mock_extract:
|
|
mock_extract.return_value = scraped
|
|
await _do_extract(page, state)
|
|
|
|
assert state.status == "done"
|
|
|
|
result = await parental_leave_repo.get_latest_by_parent(
|
|
seeded_ids.personal_data_uuid, seeded_ids.child_id
|
|
)
|
|
assert result is not None
|
|
assert result.source == "scraped"
|
|
assert result.days_left == 164
|
|
assert result.total_days == 240
|