"""Tests for the FK scraper: extractor, validation, and session state machine. Three independent test concerns: 1. _extract_parental_leave — parses the FK #/valjdagar HTML fixture and produces a correct ParentalLeaveFetch. No network, no BankID. 2. Validation helpers — out-of-bounds numbers raise rather than persist. 3. Session state machine — start/poll/cancel transitions, TTL auto-cancel, double-cancel safety. Browser is mocked; no headless Chromium needed. """ from __future__ import annotations import re import uuid from datetime import datetime from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.entities.parental_leave import ParentalLeaveFetch, TierData from app.scrapers.fk_scraper import ( FamilySummary, FKScraperService, PartnerDeduction, TransferableDays, _apply_transferable, _deduce_partner, _do_extract, _do_login, _extract_family_summary, _extract_transferable_days, _SEL, _SessionState, _set_status, _sessions, ) from app.utils.errors import ValidationError # --------------------------------------------------------------------------- # Helpers for the HTML-fixture tests # --------------------------------------------------------------------------- FIXTURES_DIR = Path(__file__).parents[1] / "fixtures" def _load_fixture(name: str) -> str: """Read a synthetic HTML fixture from tests/fixtures/. Fixtures here are hand-built (never real FK page captures — see F1 of the scraper/config/ops security audit) and always present in the repo. """ return (FIXTURES_DIR / name).read_text(encoding="utf-8") # --------------------------------------------------------------------------- # Minimal mock of a Playwright Page that answers our CSS queries from HTML # --------------------------------------------------------------------------- def _make_mock_page(html: str): """ Build a minimal async mock page that serves inner_text / all_inner_texts for the selectors used by _extract_parental_leave. Uses basic regex over the fixture HTML rather than a real browser — fast and dependency-free. """ NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span" FAMILY_SEL = "[data-test='forDigOchAnnanForalder']" FAMILY_KVAR_SEL = FAMILY_SEL + " .visa-dagar__dagar-kvar" FAMILY_DISCRETE_SEL = FAMILY_SEL + " .visa-dagar__discrete" FAMILY_NIVA_SEL = FAMILY_SEL + " .visa-dagar__desktop .niva-statistik__dagar span" def _family_panel_present() -> bool: return 'data-test="forDigOchAnnanForalder"' in html def _family_kvar() -> str | None: m = re.search( r'data-test="forDigOchAnnanForalder".*?' r'class="visa-dagar__dagar-kvar">(\d+)<', html, re.DOTALL, ) return m.group(1) if m else None def _family_discrete() -> str | None: m = re.search( r'data-test="forDigOchAnnanForalder".*?' r'class="visa-dagar__discrete">(.*?)<', html, re.DOTALL, ) return m.group(1) if m else None def _family_niva_all_texts() -> list[str]: m_block = re.search( r'data-test="forDigOchAnnanForalder"(.*?)$', html, re.DOTALL, ) block = m_block.group(1) if m_block else "" return re.findall(r"\(\d+ dagar\)", block) def _for_dig_kvar() -> str: m = re.search( r'data-test="forDig".*?' r'class="visa-dagar__dagar-kvar">(\d+)<', html, re.DOTALL, ) if m: return m.group(1) raise ValueError("No forDig dagar-kvar found in fixture") def _for_dig_discrete() -> str: m = re.search( r'data-test="forDig".*?' r'class="visa-dagar__discrete">(.*?)<', html, re.DOTALL, ) if m: return m.group(1) raise ValueError("No forDig discrete found in fixture") def _dubbeldag_kvar() -> str: m = re.search( r'data-test="dubbeldag".*?' r'class="visa-dagar__dagar-kvar">(\d+)<', html, re.DOTALL, ) if m: return m.group(1) raise ValueError("No dubbeldag dagar-kvar found in fixture") def _niva_all_texts() -> list[str]: m_block = re.search( r'data-test="forDig"(.*?)data-test="footer"', html, re.DOTALL, ) block = m_block.group(1) if m_block else "" desktop_m = re.search(r"visa-dagar__desktop(.*?)visa-dagar__mobile", block, re.DOTALL) target = desktop_m.group(1) if desktop_m else block return re.findall(r"\(\d+ dagar\)", target) page = MagicMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() if sel == NIVA_SEL: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=_niva_all_texts()) loc.count = AsyncMock(return_value=len(_niva_all_texts())) elif sel == FAMILY_SEL: loc.count = AsyncMock(return_value=1 if _family_panel_present() else 0) elif sel == FAMILY_KVAR_SEL: val = _family_kvar() loc.count = AsyncMock(return_value=1 if val is not None else 0) loc.first.inner_text = AsyncMock(return_value=val or "") elif sel == FAMILY_DISCRETE_SEL: val = _family_discrete() loc.count = AsyncMock(return_value=1 if val is not None else 0) loc.first.inner_text = AsyncMock(return_value=val or "") elif sel == FAMILY_NIVA_SEL: texts = _family_niva_all_texts() loc.count = AsyncMock(return_value=len(texts)) loc.all_inner_texts = AsyncMock(return_value=texts) elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value=_for_dig_kvar()) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) loc.count = AsyncMock(return_value=1) elif "visa-dagar__discrete" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value=_for_dig_discrete()) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) loc.count = AsyncMock(return_value=1) elif "dubbeldag" in sel and "visa-dagar__dagar-kvar" in sel: loc.first.inner_text = AsyncMock(return_value=_dubbeldag_kvar()) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) loc.count = AsyncMock(return_value=1) else: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) loc.count = AsyncMock(return_value=0) return loc page.locator.side_effect = locator_factory return page # --------------------------------------------------------------------------- # Extract from real HTML fixture # --------------------------------------------------------------------------- # Import the extractor directly (no browser needed) from app.scrapers.fk_scraper import _extract_parental_leave # noqa: E402 class TestExtractFromFixture: """Parse the synthetic overview HTML fixture and assert correct field values.""" async def test_days_left_and_total_parsed(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-1", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) # From the synthetic fixture: 100 av 240 dagar assert result.days_left == 100 assert result.total_days == 240 assert result.days_used == 140 async def test_tier_values_parsed(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-2", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) # From the synthetic fixture: sjukpenning 60, lägstanivå 40 assert result.tier_data.sjukpenning_left == 60 assert result.tier_data.lagsta_niva_left == 40 async def test_dubbeldagar_parsed(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-3", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) # From the synthetic fixture: 20 av 60 dubbeldagar assert result.dubbeldagar_left == 20 async def test_source_is_scraped(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-4", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) assert result.source == "scraped" async def test_reserved_days_derived(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-5", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) # 140 days used < 150 transferable → all reserved days remain assert result.reserved_days_total == 90 assert result.reserved_days_left == 90 assert result.transferable_days_left == 240 - 90 - 140 # 10 async def test_tier_totals_sum_to_total_days(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-6", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) assert ( result.tier_data.sjukpenning_total + result.tier_data.lagsta_niva_total == result.total_days ) async def test_tier_used_nonnegative(self) -> None: html = _load_fixture("fk_overview_synthetic.html") state = _SessionState( session_id="test-7", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = _make_mock_page(html) result = await _extract_parental_leave(page, state) assert result.tier_data.sjukpenning_used >= 0 assert result.tier_data.lagsta_niva_used >= 0 # --------------------------------------------------------------------------- # Validation: bad numbers raise rather than persist # --------------------------------------------------------------------------- class TestValidation: """Feed impossible numbers and assert the extractor raises.""" def _make_bad_page( self, days_left: int, total_days: int, sjukpenning: int, lagsta: int, dubbel: int ) -> Any: NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span" page = MagicMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() if sel == NIVA_SEL: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock( return_value=[f"({sjukpenning} dagar)", f"({lagsta} dagar)"] ) elif "visa-dagar__dagar-kvar" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value=str(days_left)) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "visa-dagar__discrete" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock( return_value=f"av {total_days} dagar kvar efter ansökan" ) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "dubbeldag" in sel: loc.first.inner_text = AsyncMock(return_value=str(dubbel)) loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) else: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) return loc page.locator.side_effect = locator_factory return page def _state(self) -> _SessionState: return _SessionState( session_id="val-test", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) async def test_negative_dubbeldagar_left_raises(self) -> None: """Regression coverage moved here from a "negative days_left" scenario. Derivation now lives in `app.services.leave_snapshot.build_snapshot` (see its module docstring), which derives `days_left` from the tier breakdown (`sjukpenning_left + lagsta_niva_left`) rather than the page's separate "För dig, dagar kvar" text — and that breakdown is parsed with a digits-only regex, so it can never itself go negative. `dubbeldagar_left` is still parsed with a plain `int()`, so it's the remaining reachable path to a negative-number guard here; the `days_left`/tier negative-value guards themselves are unit-tested directly against `build_snapshot` in `test_leave_snapshot.py`. `build_snapshot` raises `ValidationError` (a `DomainError`), not a bare `ValueError` like this function's own page-parsing code does. """ page = self._make_bad_page(164, 240, 119, 45, -1) with pytest.raises(ValidationError, match="negative"): await _extract_parental_leave(page, self._state()) async def test_total_days_parse_failure_raises(self) -> None: NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span" page = MagicMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() if sel == NIVA_SEL: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=["(80 dagar)", "(20 dagar)"]) elif "dagar-kvar" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value="100") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "discrete" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value="INVALID TEXT") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "dubbeldag" in sel: loc.first.inner_text = AsyncMock(return_value="39") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) else: loc.first.inner_text = AsyncMock(return_value="0") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) return loc page.locator.side_effect = locator_factory with pytest.raises(ValueError, match="parse total days"): await _extract_parental_leave(page, self._state()) async def test_fewer_than_two_niva_values_raises(self) -> None: NIVA_SEL = "[data-test='forDig'] .visa-dagar__desktop .niva-statistik__dagar span" page = MagicMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() if sel == NIVA_SEL: loc.first.inner_text = AsyncMock(return_value="") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=["(119 dagar)"]) # only one elif "dagar-kvar" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value="164") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "discrete" in sel and "forDig" in sel: loc.first.inner_text = AsyncMock(return_value="av 240 dagar kvar efter ansökan") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) elif "dubbeldag" in sel: loc.first.inner_text = AsyncMock(return_value="39") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) else: loc.first.inner_text = AsyncMock(return_value="0") loc.first.wait_for = AsyncMock() loc.all_inner_texts = AsyncMock(return_value=[]) return loc page.locator.side_effect = locator_factory with pytest.raises(ValueError, match="Expected 2 tier values"): await _extract_parental_leave(page, self._state()) async def test_zero_total_days_raises(self) -> None: """Guard against placeholder zero data: total_days=0 is impossible and indicates hydration failure.""" page = self._make_bad_page(164, 0, 119, 45, 39) with pytest.raises(ValueError, match="total_days=0.*had not hydrated"): await _extract_parental_leave(page, self._state()) # --------------------------------------------------------------------------- # Family panel -> partner deduction # --------------------------------------------------------------------------- _NO_FAMILY_PANEL_HTML = """
100 av 240 dagar kvar efter ansökan
(60 dagar) (40 dagar)
20
""" _FAMILY_PANEL_NO_TIERS_HTML = """
100 av 240 dagar kvar efter ansökan
(60 dagar) (40 dagar)
250 av 480 dagar kvar efter ansökan
20
""" def _state() -> _SessionState: return _SessionState( session_id="family-test", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) class TestExtractFamilySummary: """`_extract_family_summary` parses the (assumed) family-panel DOM defensively.""" async def test_family_summary_parsed_from_fixture(self) -> None: html = _load_fixture("fk_overview_synthetic.html") page = _make_mock_page(html) family = await _extract_family_summary(page, _state()) assert family == FamilySummary( total_days=480, days_left=250, sjukpenning_left=190, lagsta_niva_left=60 ) async def test_absent_family_panel_returns_none(self) -> None: page = _make_mock_page(_NO_FAMILY_PANEL_HTML) family = await _extract_family_summary(page, _state()) assert family is None async def test_absent_family_panel_does_not_affect_main_extraction(self) -> None: """The caller's own extraction must succeed exactly as without this feature.""" page = _make_mock_page(_NO_FAMILY_PANEL_HTML) result = await _extract_parental_leave(page, _state()) assert result.days_left == 100 assert result.total_days == 240 async def test_tierless_family_panel_returns_summary_with_none_tiers(self) -> None: page = _make_mock_page(_FAMILY_PANEL_NO_TIERS_HTML) family = await _extract_family_summary(page, _state()) assert family == FamilySummary( total_days=480, days_left=250, sjukpenning_left=None, lagsta_niva_left=None ) class TestDeducePartner: """`_deduce_partner` derives the other parent's balance as family - mine.""" def _mine(self, **overrides: object) -> ParentalLeaveFetch: now = datetime.now() base = dict( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=140, days_left=100, reserved_days_total=90, reserved_days_left=90, transferable_days_left=10, dubbeldagar_left=20, tier_data=TierData( sjukpenning_total=195, sjukpenning_used=135, sjukpenning_left=60, lagsta_niva_total=45, lagsta_niva_used=5, lagsta_niva_left=40, ), created_at=now, updated_at=now, ) base.update(overrides) return ParentalLeaveFetch(**base) def test_deduction_with_family_tiers_present(self) -> None: mine = self._mine() family = FamilySummary( total_days=480, days_left=250, sjukpenning_left=190, lagsta_niva_left=60 ) deduction = _deduce_partner(mine, family) assert deduction == PartnerDeduction( total_days=240, days_left=150, sjukpenning_left=130, lagsta_niva_left=20, dubbeldagar_left=20, ) def test_deduction_falls_back_to_proportional_split_without_family_tiers(self) -> None: mine = self._mine() family = FamilySummary( total_days=480, days_left=250, sjukpenning_left=None, lagsta_niva_left=None ) deduction = _deduce_partner(mine, family) # total=240, left=150; lagsta_share = 90*240 // 480 = 45 assert deduction == PartnerDeduction( total_days=240, days_left=150, sjukpenning_left=105, lagsta_niva_left=45, dubbeldagar_left=20, ) def test_family_left_below_mine_returns_none(self) -> None: """family.days_left < mine.days_left can only mean stale/inconsistent data.""" mine = self._mine(total_days=240, days_left=200) family = FamilySummary( total_days=480, days_left=150, sjukpenning_left=None, lagsta_niva_left=None ) assert _deduce_partner(mine, family) is None def test_family_total_not_greater_than_mine_returns_none(self) -> None: mine = self._mine(total_days=480) family = FamilySummary( total_days=480, days_left=300, sjukpenning_left=None, lagsta_niva_left=None ) assert _deduce_partner(mine, family) is None def test_negative_tier_deduction_falls_back_to_proportional_split(self) -> None: """Family-level tiers that would go negative per-parent fall back rather than producing a nonsensical negative PartnerDeduction field.""" mine = self._mine() # family.sjukpenning_left (50) < mine.sjukpenning_left (60) -> negative naive diff family = FamilySummary( total_days=480, days_left=150, sjukpenning_left=50, lagsta_niva_left=100 ) deduction = _deduce_partner(mine, family) assert deduction is not None assert deduction.sjukpenning_left >= 0 assert deduction.lagsta_niva_left >= 0 # --------------------------------------------------------------------------- # Transferable days ("Dagar ni kan föra över") -> min-foraldrapenning page # --------------------------------------------------------------------------- # Small balanced-
HTML toolkit for the mock page below. Regex alone # (as `_make_mock_page` above uses) can't safely slice `.barn-sektion` or # `.fora-over-group`, since they contain further nested
s (the tooltip, # in particular) — a naive ".*?
" would stop at the first nested close. def _balanced_div_slice(html: str, open_tag_start: int) -> str: """From the index of a '/
pairs.""" depth = 0 for m in re.finditer(r"]*>|", html[open_tag_start:], re.IGNORECASE): if m.group(0).lower().startswith(" in fixture") def _find_divs_by_class(html: str, class_name: str) -> list[str]: """Balanced outer-HTML slices for every
in document order (non-overlapping — resumes the search after each match's own closing tag, so nested divs of the same class, unused here, aren't double-counted).""" pattern = re.compile(rf']*\bclass="[^"]*\b{re.escape(class_name)}\b[^"]*"[^>]*>') results: list[str] = [] pos = 0 while True: m = pattern.search(html, pos) if not m: break sl = _balanced_div_slice(html, m.start()) results.append(sl) pos = m.start() + len(sl) return results def _find_barn_rubrik_text(section_html: str) -> str | None: idx = section_html.find("barn-rubrik") if idx == -1: return None m = re.search(r"(.*?)", section_html[idx:], re.DOTALL) return m.group(1).strip() if m else None def _direct_child_p_texts(div_slice: str) -> list[str]: """Text of

elements that are direct children of div_slice's own root element — i.e. not nested inside any further

— mirroring Playwright's `:scope > p`.""" open_m = re.match(r"]*>", div_slice, re.IGNORECASE) inner = div_slice[open_m.end() : -len("
")] texts: list[str] = [] depth = 0 for m in re.finditer(r"<(/?)(div|p)\b[^>]*>", inner, re.IGNORECASE): closing = m.group(1) == "/" tag = m.group(2).lower() if tag == "div": depth += -1 if closing else 1 continue if tag == "p" and not closing and depth == 0: end_m = re.search(r"", inner[m.end() :], re.IGNORECASE) if end_m: raw = inner[m.end() : m.end() + end_m.start()] texts.append(re.sub(r"<[^>]+>", "", raw).strip()) return texts def _make_min_fp_mock_page(html: str): """Build a minimal async mock Page for the min-foraldrapenning "Dagar ni kan föra över" panel selectors. Balanced-
slicing + regex over the fixture HTML, not a real browser — see `_find_divs_by_class` above. """ min_fp_slices = _find_divs_by_class(html, "min-fp") min_fp_html = min_fp_slices[0] if min_fp_slices else "" barn_sektioner = _find_divs_by_class(min_fp_html, "barn-sektion") if min_fp_html else [] def _make_group_locator(group_html: str) -> MagicMock: loc = MagicMock() def sub_locator(sel: str) -> MagicMock: if sel == _SEL.MINFP_LABEL: m = re.search( r']*class="[^"]*minfp-label[^"]*"[^>]*>(.*?)', group_html, re.DOTALL, ) sub = MagicMock() sub.count = AsyncMock(return_value=1 if m else 0) if m: text = re.sub(r"<[^>]+>", "", m.group(1)).strip() sub.first.inner_text = AsyncMock(return_value=text) return sub if sel == ":scope > p": sub = MagicMock() sub.all_inner_texts = AsyncMock(return_value=_direct_child_p_texts(group_html)) return sub raise AssertionError(f"unexpected selector on group locator: {sel!r}") loc.locator.side_effect = sub_locator return loc def _make_panel_locator(panel_html: str) -> MagicMock: loc = MagicMock() loc.count = AsyncMock(return_value=1) def sub_locator(sel: str) -> MagicMock: if sel == _SEL.EXPANDABLE_BUTTON: m = re.search( r']*class="[^"]*expandable-paragraph__button[^"]*"[^>]*>', panel_html, ) sub = MagicMock() sub.count = AsyncMock(return_value=1 if m else 0) if m: aria_m = re.search(r'aria-expanded="(true|false)"', m.group(0)) sub.first.get_attribute = AsyncMock( return_value=aria_m.group(1) if aria_m else None ) sub.first.click = AsyncMock() return sub if sel == _SEL.FORA_OVER_GROUP: groups = _find_divs_by_class(panel_html, "fora-over-group") sub = MagicMock() sub.count = AsyncMock(return_value=len(groups)) sub.nth.side_effect = lambda i: _make_group_locator(groups[i]) return sub raise AssertionError(f"unexpected selector on panel locator: {sel!r}") loc.locator.side_effect = sub_locator return loc def _make_section_locator(section_html: str) -> MagicMock: loc = MagicMock() def sub_locator(sel: str) -> MagicMock: if sel == _SEL.BARN_RUBRIK: text = _find_barn_rubrik_text(section_html) sub = MagicMock() sub.count = AsyncMock(return_value=1 if text is not None else 0) sub.first.inner_text = AsyncMock(return_value=text or "") return sub if sel == _SEL.PANEL_FORA_OVER_DAGAR: panels = _find_divs_by_class(section_html, "panel-fora-over-dagar") sub = MagicMock() sub.count = AsyncMock(return_value=len(panels)) if panels: panel_loc = _make_panel_locator(panels[0]) sub.locator.side_effect = panel_loc.locator.side_effect return sub raise AssertionError(f"unexpected selector on section locator: {sel!r}") loc.locator.side_effect = sub_locator return loc def _make_sections_root_locator() -> MagicMock: loc = MagicMock() loc.count = AsyncMock(return_value=len(barn_sektioner)) if barn_sektioner: loc.first = _make_section_locator(barn_sektioner[0]) loc.nth.side_effect = lambda i: _make_section_locator(barn_sektioner[i]) return loc page = MagicMock() page.goto = AsyncMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() def locator_factory(sel: str) -> MagicMock: if sel == _SEL.MIN_FP_BARN_SEKTION: return _make_sections_root_locator() raise AssertionError(f"unexpected top-level selector: {sel!r}") page.locator.side_effect = locator_factory return page _MULTI_CHILD_MIN_FP_HTML = """

Testbarn

Du kan föra över

84 dagar på sjukpenningnivå

45 dagar på lägstanivå

Annan förälder kan föra över

26 dagar på sjukpenningnivå

45 dagar på lägstanivå

Andrabarn

Du kan föra över

10 dagar på sjukpenningnivå

5 dagar på lägstanivå

""" _SOLO_PARENT_MIN_FP_HTML = """

Testbarn

Du kan föra över

84 dagar på sjukpenningnivå

45 dagar på lägstanivå

""" _NO_TRANSFER_PANEL_MIN_FP_HTML = """

Testbarn

""" _INCOMPLETE_MINE_MIN_FP_HTML = """

Testbarn

Du kan föra över

84 dagar på sjukpenningnivå

""" def _min_fp_state() -> _SessionState: return _SessionState( session_id="min-fp-test", workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) class TestExtractTransferableDays: """`_extract_transferable_days` reads FK's own "Dagar ni kan föra över" figures off the min-foraldrapenning page — the fix for the bug where `transferable_days_left` could never be reconstructed from #/valjdagar alone (see facts.md: FK reports 129/71, the old derivation gave 49/0).""" async def test_both_groups_parsed_golden_regression(self) -> None: """Pinning regression: mine 84+45=129, partner 26+45=71 — the exact `facts.md` figures FK itself reports, which the old arithmetic derivation could never reproduce.""" html = _load_fixture("fk_min_foraldrapenning_synthetic.html") page = _make_min_fp_mock_page(html) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is not None assert result.mine_total == 129 assert result.partner_total == 71 async def test_tooltip_decoys_ignored(self) -> None: """The tooltip's decoy "45 dagar"/"90 dagar" text must not leak into `mine_sjukpenning`/`mine_lagsta_niva` via a non-scoped read.""" html = _load_fixture("fk_min_foraldrapenning_synthetic.html") page = _make_min_fp_mock_page(html) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is not None assert result.mine_sjukpenning == 84 assert result.mine_lagsta_niva == 45 async def test_collapsed_accordion_still_yields_correct_numbers(self) -> None: html = _load_fixture("fk_min_foraldrapenning_synthetic.html").replace( 'aria-expanded="true"', 'aria-expanded="false"' ) page = _make_min_fp_mock_page(html) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is not None assert result.mine_total == 129 assert result.partner_total == 71 async def test_panel_absent_returns_none(self) -> None: page = _make_min_fp_mock_page(_NO_TRANSFER_PANEL_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is None async def test_single_section_used_without_name_match(self) -> None: """A single `.barn-sektion` is used as-is — no `child_name` needed.""" page = _make_min_fp_mock_page(_SOLO_PARENT_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), None) assert result is not None assert result.mine_total == 129 async def test_multi_child_correct_section_chosen_by_name(self) -> None: page = _make_min_fp_mock_page(_MULTI_CHILD_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), "Andrabarn") assert result is not None assert result.mine_sjukpenning == 10 assert result.mine_lagsta_niva == 5 async def test_multi_child_unmatched_name_returns_none(self) -> None: page = _make_min_fp_mock_page(_MULTI_CHILD_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), "Okänt Barn") assert result is None async def test_multi_child_no_name_returns_none(self) -> None: """Never guess a section when there's more than one and no name to match.""" page = _make_min_fp_mock_page(_MULTI_CHILD_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), None) assert result is None async def test_solo_parent_partner_fields_none(self) -> None: page = _make_min_fp_mock_page(_SOLO_PARENT_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is not None assert result.partner_sjukpenning is None assert result.partner_lagsta_niva is None assert result.partner_total is None async def test_incomplete_mine_group_returns_none(self) -> None: page = _make_min_fp_mock_page(_INCOMPLETE_MINE_MIN_FP_HTML) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is None async def test_exception_during_parse_returns_none(self) -> None: """Any unexpected failure (e.g. a locator raising) degrades to None rather than propagating — mirrors `_extract_family_summary`.""" page = _make_min_fp_mock_page(_load_fixture("fk_min_foraldrapenning_synthetic.html")) page.wait_for_selector = AsyncMock(side_effect=RuntimeError("boom")) result = await _extract_transferable_days(page, _min_fp_state(), "Testbarn") assert result is None class TestApplyTransferable: """`_apply_transferable` re-derives `data` with FK's own reported split.""" def _mine(self, **overrides: object) -> ParentalLeaveFetch: now = datetime.now() base = dict( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=140, days_left=100, reserved_days_total=90, reserved_days_left=90, transferable_days_left=10, dubbeldagar_left=20, tier_data=TierData( sjukpenning_total=195, sjukpenning_used=135, sjukpenning_left=60, lagsta_niva_total=45, lagsta_niva_used=5, lagsta_niva_left=40, ), created_at=now, updated_at=now, ) base.update(overrides) return ParentalLeaveFetch(**base) def test_valid_transferable_updates_reserved_split(self) -> None: mine = self._mine() transferable = TransferableDays(mine_sjukpenning=50, mine_lagsta_niva=10) result = _apply_transferable(mine, transferable) assert result.transferable_days_left == 60 assert result.reserved_days_left == 40 # days_left(100) - transferable(60) assert result.transferable_days_reported is True def test_implausible_transferable_returns_original_unchanged(self) -> None: """transferable_days_left > days_left is impossible — build_snapshot raises ValidationError (reserved_days_left would go negative), and the caller's original estimate must survive unchanged, never raise.""" mine = self._mine() transferable = TransferableDays(mine_sjukpenning=500, mine_lagsta_niva=500) result = _apply_transferable(mine, transferable) assert result is mine class TestDoExtractTransferableIntegration: """`_do_extract`'s new transferable-days block, end to end. `_mine`'s figures mirror `facts.md` exactly: days_left=94+45=139, and the old transferable-first derivation (no FK figure available) computes transferable_days_left=49 — the bug this feature fixes (FK itself reports 129). The family fixture below likewise mirrors facts.md's "210 av 480 dagar kvar totalt" / "Annan förälder: 71 dagar kvar". """ _FAMILY_HTML = """
210 av 480 dagar kvar efter ansökan
(120 dagar) (90 dagar)
""" def setup_method(self) -> None: _sessions.clear() def teardown_method(self) -> None: _sessions.clear() def _mine(self, **overrides: object) -> ParentalLeaveFetch: now = datetime.now() base = dict( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=101, days_left=139, reserved_days_total=90, reserved_days_left=90, transferable_days_left=49, dubbeldagar_left=39, tier_data=TierData( sjukpenning_total=195, sjukpenning_used=101, sjukpenning_left=94, lagsta_niva_total=45, lagsta_niva_used=0, lagsta_niva_left=45, ), created_at=now, updated_at=now, ) base.update(overrides) return ParentalLeaveFetch(**base) async def _run( self, mine: ParentalLeaveFetch, min_fp_html: str, family_html: str = "", child_name: str | None = "Testbarn", ) -> _SessionState: sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), child_name=child_name, ) _sessions[sid] = state min_fp_page = _make_min_fp_mock_page(min_fp_html) family_page = _make_mock_page(family_html) page = MagicMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() page.goto = min_fp_page.goto def locator_factory(sel: str) -> MagicMock: if sel == _SEL.MIN_FP_BARN_SEKTION: return min_fp_page.locator(sel) if sel.startswith(_SEL.FAMILY): return family_page.locator(sel) loc = MagicMock() loc.first.click = AsyncMock() loc.count = AsyncMock(return_value=0) return loc page.locator.side_effect = locator_factory with patch( "app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock ) as mock_extract: mock_extract.return_value = mine await _do_extract(page, state) return state async def test_transferable_days_applied_to_result(self) -> None: """Golden regression via `_do_extract`: the persisted result carries FK's own 129, not the old broken derivation's 49.""" mine = self._mine() state = await self._run(mine, _load_fixture("fk_min_foraldrapenning_synthetic.html")) assert state.status == "done" assert state.result is not None assert state.result.transferable_days_left == 129 assert state.result.reserved_days_left == 10 # 139 - 129 async def test_partner_deduction_picks_up_transferable_days(self) -> None: mine = self._mine() state = await self._run( mine, _load_fixture("fk_min_foraldrapenning_synthetic.html"), family_html=self._FAMILY_HTML, ) assert state.partner_deduction is not None assert state.partner_deduction.days_left == 71 assert state.partner_deduction.transferable_days_left == 71 async def test_exception_in_block_leaves_data_untouched(self) -> None: """Mirrors `test_absent_family_panel_does_not_affect_main_extraction`: a failure in the new block must never affect the caller's own, already-validated `data`.""" mine = self._mine() with patch( "app.scrapers.fk_scraper._extract_transferable_days", new_callable=AsyncMock, side_effect=RuntimeError("boom"), ): state = await self._run(mine, _load_fixture("fk_min_foraldrapenning_synthetic.html")) assert state.status == "done" assert state.result is mine assert state.result.transferable_days_left == 49 # unchanged from the original estimate # --------------------------------------------------------------------------- # Maintenance detection — client-facing message must not echo scraped text # --------------------------------------------------------------------------- class TestMaintenanceDetection: """R3: `_do_login`'s maintenance branch must return a fixed message — never the scraped FK banner text — to the poll client.""" async def test_maintenance_banner_text_not_leaked_to_client(self) -> None: sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) page = MagicMock() page.goto = AsyncMock() page.url = "https://www.forsakringskassan.se/privatperson/logga-in" sensitive_banner = ( "Planerat underhall pagar for personer med personnummer som slutar pa " "1-5, tjansten ar otillganglig till kl 23." ) with patch( "app.scrapers.fk_scraper._check_maintenance", new_callable=AsyncMock ) as mock_check: mock_check.return_value = sensitive_banner await _do_login(page, state) assert state.status == "error" assert state.message is not None assert sensitive_banner not in state.message assert "maintenance" in state.message.lower() # --------------------------------------------------------------------------- # Session state machine # --------------------------------------------------------------------------- class TestSessionStateMachine: """Session transitions, TTL watchdog, and double-cancel safety.""" def setup_method(self) -> None: _sessions.clear() def teardown_method(self) -> None: _sessions.clear() async def test_start_session_creates_entry(self) -> None: service = FKScraperService() owner_user_id = uuid.uuid4() with patch("app.scrapers.fk_scraper._run_session", new_callable=AsyncMock) as mock_run: mock_run.return_value = None sid = await service.start_session( workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, ) assert sid in _sessions assert _sessions[sid].status == "launching" assert _sessions[sid].owner_user_id == owner_user_id async def test_poll_unknown_session_returns_error(self) -> None: service = FKScraperService() result = await service.poll_session("nonexistent", uuid.uuid4()) assert result["status"] == "error" assert "Unknown" in result["message"] async def test_poll_done_includes_partner_deduction_when_present(self) -> None: service = FKScraperService() sid = str(uuid.uuid4()) owner_user_id = uuid.uuid4() now = datetime.now() result_fetch = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=140, days_left=100, reserved_days_total=90, reserved_days_left=90, transferable_days_left=10, dubbeldagar_left=20, tier_data=TierData(195, 135, 60, 45, 5, 40), created_at=now, updated_at=now, ) deduction = PartnerDeduction( total_days=240, days_left=150, sjukpenning_left=130, lagsta_niva_left=20, dubbeldagar_left=20, ) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, status="done", result=result_fetch, partner_deduction=deduction, ) _sessions[sid] = state result = await service.poll_session(sid, owner_user_id) assert result["status"] == "done" assert result["partner_deduction"] == { "total_days": 240, "days_left": 150, "sjukpenning_left": 130, "lagsta_niva_left": 20, "dubbeldagar_left": 20, "transferable_days_left": None, } async def test_poll_done_partner_deduction_is_none_when_absent(self) -> None: service = FKScraperService() sid = str(uuid.uuid4()) owner_user_id = uuid.uuid4() now = datetime.now() result_fetch = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=480, days_used=0, days_left=480, reserved_days_total=90, reserved_days_left=90, transferable_days_left=390, dubbeldagar_left=60, tier_data=TierData(390, 0, 390, 90, 0, 90), created_at=now, updated_at=now, ) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, status="done", result=result_fetch, partner_deduction=None, ) _sessions[sid] = state result = await service.poll_session(sid, owner_user_id) assert result["status"] == "done" assert result["partner_deduction"] is None async def test_poll_awaiting_scan_returns_qr_data(self) -> None: service = FKScraperService() sid = str(uuid.uuid4()) owner_user_id = uuid.uuid4() state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, status="awaiting_scan", qr_data=b"PNG_BYTES", ) _sessions[sid] = state result = await service.poll_session(sid, owner_user_id) assert result["status"] == "awaiting_scan" assert "qr_data" in result import base64 assert base64.b64decode(result["qr_data"]) == b"PNG_BYTES" async def test_poll_with_wrong_owner_is_treated_as_unknown_session(self) -> None: """F4 (T1 authz audit): a session_id known to a caller who did not start it must behave EXACTLY like a made-up session_id — never leak that the session exists (no QR data, no distinct error).""" service = FKScraperService() sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), status="awaiting_scan", qr_data=b"PNG_BYTES", ) _sessions[sid] = state result = await service.poll_session(sid, uuid.uuid4()) # different owner assert result == {"status": "error", "message": "Unknown session_id"} async def test_cancel_sets_status(self) -> None: service = FKScraperService() sid = str(uuid.uuid4()) owner_user_id = uuid.uuid4() state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, status="awaiting_scan", ) _sessions[sid] = state await service.cancel_session(sid, owner_user_id) assert _sessions[sid].status == "cancelled" async def test_cancel_clears_qr_data(self) -> None: """F5: cancelling a session must not leave the last QR frame in RAM.""" service = FKScraperService() sid = str(uuid.uuid4()) owner_user_id = uuid.uuid4() state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, status="awaiting_scan", qr_data=b"PNG_BYTES", ) _sessions[sid] = state await service.cancel_session(sid, owner_user_id) assert _sessions[sid].qr_data is None async def test_cancel_with_wrong_owner_is_a_no_op(self) -> None: """F4: cancelling with the wrong owner must not affect the session — exactly like cancelling an unknown session_id.""" service = FKScraperService() sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), status="awaiting_scan", ) _sessions[sid] = state await service.cancel_session(sid, uuid.uuid4()) # different owner assert _sessions[sid].status == "awaiting_scan" async def test_cancel_nonexistent_session_is_safe(self) -> None: service = FKScraperService() await service.cancel_session("does-not-exist", uuid.uuid4()) # should not raise async def test_ttl_watchdog_cancels_stale_session(self) -> None: from app.scrapers.fk_scraper import _ttl_watchdog with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_session_ttl_seconds = 0 # immediate sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), status="awaiting_scan", ) _sessions[sid] = state await _ttl_watchdog(sid) assert _sessions[sid].status == "error" assert "timed out" in (_sessions[sid].message or "") async def test_ttl_watchdog_purges_already_terminal_session(self) -> None: """F5: if the session already finished by the time the watchdog fires, it's a safety net that purges it immediately rather than leaving it for `_purge_later`.""" from app.scrapers.fk_scraper import _ttl_watchdog with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_session_ttl_seconds = 0 # immediate sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), status="done", ) _sessions[sid] = state await _ttl_watchdog(sid) assert sid not in _sessions async def test_purge_later_removes_session_after_retention_window(self) -> None: """F5: a finished session is removed from `_sessions` after `scraper_result_retention_seconds`.""" from app.scrapers.fk_scraper import _purge_later with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_result_retention_seconds = 0 # immediate sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), status="done", result=None, ) _sessions[sid] = state await _purge_later(sid) assert sid not in _sessions async def test_purge_later_is_a_no_op_if_already_purged(self) -> None: from app.scrapers.fk_scraper import _purge_later with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_result_retention_seconds = 0 await _purge_later("does-not-exist") # should not raise async def test_start_session_cancels_existing_session_for_same_owner(self) -> None: """F5: at most one live session per owner — starting a new one cancels any non-terminal session already in flight for that owner.""" service = FKScraperService() owner_user_id = uuid.uuid4() with patch("app.scrapers.fk_scraper._run_session", new_callable=AsyncMock): first_sid = await service.start_session( workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, ) _sessions[first_sid].status = "awaiting_scan" second_sid = await service.start_session( workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_user_id, ) assert _sessions[first_sid].status == "cancelled" assert _sessions[second_sid].status == "launching" assert first_sid != second_sid async def test_start_session_does_not_cancel_other_owners_session(self) -> None: service = FKScraperService() owner_a = uuid.uuid4() owner_b = uuid.uuid4() with patch("app.scrapers.fk_scraper._run_session", new_callable=AsyncMock): sid_a = await service.start_session( workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_a, ) _sessions[sid_a].status = "awaiting_scan" await service.start_session( workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=owner_b, ) assert _sessions[sid_a].status == "awaiting_scan" async def test_set_status_updates_message(self) -> None: sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) _set_status(state, "error", "Something went wrong") assert state.status == "error" assert state.message == "Something went wrong" async def test_do_extract_calls_persist_callback_on_success(self) -> None: """_do_extract invokes the persist_callback with the extracted result.""" persisted: list[ParentalLeaveFetch] = [] async def mock_persist(fetch: ParentalLeaveFetch, partner: PartnerDeduction | None) -> None: persisted.append(fetch) now = datetime.now() expected = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=76, days_left=164, reserved_days_total=90, reserved_days_left=90, transferable_days_left=74, dubbeldagar_left=39, tier_data=TierData( sjukpenning_total=195, sjukpenning_used=76, sjukpenning_left=119, lagsta_niva_total=45, lagsta_niva_used=0, lagsta_niva_left=45, ), created_at=now, updated_at=now, ) sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), persist_callback=mock_persist, ) _sessions[sid] = state page = MagicMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() loc.first.click = AsyncMock() return loc page.locator.side_effect = locator_factory with patch( "app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock ) as mock_extract: mock_extract.return_value = expected await _do_extract(page, state) assert state.status == "done" assert state.result is expected assert len(persisted) == 1 assert persisted[0] is expected async def test_do_extract_sets_error_when_persist_callback_fails(self) -> None: """_do_extract sets status=error if the persist_callback raises.""" async def failing_persist( fetch: ParentalLeaveFetch, partner: PartnerDeduction | None ) -> None: raise RuntimeError("DB unavailable") now = datetime.now() dummy = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=0, days_left=240, reserved_days_total=90, reserved_days_left=90, transferable_days_left=150, dubbeldagar_left=60, tier_data=TierData(195, 0, 195, 45, 0, 45), created_at=now, updated_at=now, ) sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), persist_callback=failing_persist, ) _sessions[sid] = state page = MagicMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() loc.first.click = AsyncMock() return loc page.locator.side_effect = locator_factory with patch( "app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock ) as mock_extract: mock_extract.return_value = dummy await _do_extract(page, state) assert state.status == "error" assert "persist" in (state.message or "").lower() # R2/R3: the raw exception text must never reach the poll client. assert "DB unavailable" not in (state.message or "") async def test_do_extract_waits_for_hydration_before_extracting(self) -> None: """_do_extract calls wait_for_function to ensure data hydrates before extraction.""" now = datetime.now() expected = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), workspace_id=uuid.uuid4(), child_id=uuid.uuid4(), source="scraped", fetched_at=now, total_days=240, days_used=0, days_left=240, reserved_days_total=90, reserved_days_left=90, transferable_days_left=150, dubbeldagar_left=60, tier_data=TierData(195, 0, 195, 45, 0, 45), created_at=now, updated_at=now, ) sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=uuid.uuid4(), personal_data_uuid=uuid.uuid4(), child_id=uuid.uuid4(), owner_user_id=uuid.uuid4(), ) _sessions[sid] = state page = MagicMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() loc.first.click = AsyncMock() return loc page.locator.side_effect = locator_factory with patch( "app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock ) as mock_extract: mock_extract.return_value = expected await _do_extract(page, state) # Verify wait_for_function was called with the correct selector page.wait_for_function.assert_called_once() call_args = page.wait_for_function.call_args assert call_args[1]["arg"] == "[data-test='forDig'] .visa-dagar__discrete" assert call_args[1]["timeout"] == 30_000 # --------------------------------------------------------------------------- # F3: debug artifacts are opt-in (contain personal data when written) # --------------------------------------------------------------------------- class TestSaveDebugArtifacts: def _make_page(self) -> MagicMock: async def _fake_screenshot(path: str, full_page: bool = True) -> None: # Real Playwright writes the PNG to `path` itself; mirror that so # the "both files got written" assertion is meaningful. Path(path).write_bytes(b"fake-png-bytes") page = MagicMock() page.content = AsyncMock(return_value="fake") page.screenshot = AsyncMock(side_effect=_fake_screenshot) return page async def test_noop_when_debug_disabled_even_with_dir_set(self, tmp_path) -> None: from app.scrapers.fk_scraper import _save_debug_artifacts with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_debug_dir = str(tmp_path) mock_settings.scraper_debug_enabled = False await _save_debug_artifacts(self._make_page(), "sid-1", ["[log] hi"]) assert list(tmp_path.iterdir()) == [] async def test_noop_when_dir_unset_even_if_enabled(self, tmp_path) -> None: from app.scrapers.fk_scraper import _save_debug_artifacts with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_debug_dir = None mock_settings.scraper_debug_enabled = True await _save_debug_artifacts(self._make_page(), "sid-2", ["[log] hi"]) assert list(tmp_path.iterdir()) == [] async def test_writes_artifacts_when_both_enabled_and_dir_set(self, tmp_path) -> None: from app.scrapers.fk_scraper import _save_debug_artifacts with patch("app.scrapers.fk_scraper.settings") as mock_settings: mock_settings.scraper_debug_dir = str(tmp_path) mock_settings.scraper_debug_enabled = True await _save_debug_artifacts(self._make_page(), "sid-3", ["[log] hi"]) written = {p.name for p in tmp_path.iterdir()} assert written == {"sid-3.html", "sid-3.png", "sid-3.console.txt"} # --------------------------------------------------------------------------- # Persistence integration: real SQLite repo round-trip # --------------------------------------------------------------------------- class TestPersistenceIntegration: """Run _do_extract with a real SQLiteParentalLeaveRepository and verify the row.""" def setup_method(self) -> None: _sessions.clear() def teardown_method(self) -> None: _sessions.clear() async def test_scraped_row_is_readable_after_session_done( self, parental_leave_repo, seeded_ids ) -> None: """After _do_extract completes, repo.get_latest_by_parent returns the scraped row.""" async def persist_callback( fetch: ParentalLeaveFetch, partner: PartnerDeduction | None ) -> None: await parental_leave_repo.create(fetch) now = datetime.now() scraped = ParentalLeaveFetch( id=uuid.uuid4(), personal_data_uuid=seeded_ids.personal_data_uuid, workspace_id=seeded_ids.workspace_id, child_id=seeded_ids.child_id, source="scraped", fetched_at=now, total_days=240, days_used=76, days_left=164, reserved_days_total=90, reserved_days_left=90, transferable_days_left=74, dubbeldagar_left=39, tier_data=TierData(195, 76, 119, 45, 0, 45), created_at=now, updated_at=now, ) sid = str(uuid.uuid4()) state = _SessionState( session_id=sid, workspace_id=seeded_ids.workspace_id, personal_data_uuid=seeded_ids.personal_data_uuid, child_id=seeded_ids.child_id, owner_user_id=uuid.uuid4(), persist_callback=persist_callback, ) _sessions[sid] = state page = MagicMock() page.wait_for_selector = AsyncMock() page.wait_for_function = AsyncMock() def locator_factory(sel: str) -> MagicMock: loc = MagicMock() loc.first.click = AsyncMock() return loc page.locator.side_effect = locator_factory with patch( "app.scrapers.fk_scraper._extract_parental_leave", new_callable=AsyncMock ) as mock_extract: mock_extract.return_value = scraped await _do_extract(page, state) assert state.status == "done" result = await parental_leave_repo.get_latest_by_parent( seeded_ids.personal_data_uuid, seeded_ids.child_id ) assert result is not None assert result.source == "scraped" assert result.days_left == 164 assert result.total_days == 240