kvitt/server/crates/kvitt-server/tests/cards_api/fetch.rs

//! Fetching (SPEC §9.12 CP-7, CP-8, CP-14; D-178, D-179): the windowed walk, the split at the 100-row cap, the
//! de-duplication across windows, continuation keys, the end of history, the mapping, the matching of pending and
//! booked transactions, deletion of vanished pending purchases and the recorded bank errors.

use axum::http::Method;
use chrono::{Days, NaiveDate};
use serde_json::{Value, json};

use crate::common::*;
use crate::stub::eb_error;
use crate::support::*;

fn date(text: &str) -> NaiveDate {
    NaiveDate::parse_from_str(text, "%Y-%m-%d").unwrap()
}

fn windows_asked(app: &CardsApp) -> Vec<(String, String)> {
    app.stub
        .transaction_requests()
        .iter()
        .map(|r| (r.query["date_from"].clone(), r.query["date_to"].clone()))
        .collect()
}

#[tokio::test]
async fn a_180_day_first_fetch_asks_six_windows_newest_first_without_strategy() {
    let app = CardsApp::new().await;
    app.stub.set_transactions(vec![
        booked("r-new", &days_ago(1), "10.00", "Testkaféet Ankan"),
        booked("r-old", &days_ago(170), "20.00", "Gamla Testbutiken"),
    ]);
    app.connect(&admin_token(), 180).await;
    let asked = windows_asked(&app);
    assert_eq!(asked.len(), 6, "{asked:?}");
    assert_eq!(asked[0], (days_ago(29), days_ago(0)));
    assert_eq!(asked[5].0, days_ago(179), "180 days, today included");
    for pair in asked.windows(2) {
        assert_eq!(
            date(&pair[1].1).checked_add_days(Days::new(1)).unwrap(),
            date(&pair[0].0),
            "consecutive, non-overlapping, newest first"
        );
    }
    for (from, to) in &asked {
        assert_eq!((date(to) - date(from)).num_days(), 29, "30 days each");
    }
    for r in app.stub.transaction_requests() {
        assert!(
            !r.query.contains_key("strategy"),
            "strategy is never sent (D-178)"
        );
        assert!(!r.query.contains_key("continuation_key"));
    }
    assert_eq!(app.purchases(&admin_token()).await.len(), 2);

    // The next fetch is incremental: 10 days before the latest stored booking date, one window.
    app.stub.clear();
    let connection = app.connection(&admin_token()).await["id"]
        .as_str()
        .unwrap()
        .to_owned();
    app.refetch(&admin_token(), &connection).await;
    assert_eq!(windows_asked(&app), [(days_ago(11), days_ago(0))]);
}

#[tokio::test]
async fn a_window_at_the_cap_is_split_down_to_a_truncated_day_and_deletes_nothing() {
    let app = CardsApp::new().await;
    app.stub.set_transactions(vec![pending(
        "p-stays",
        &days_ago(20),
        "77.00",
        "Testbutiken Väntar",
    )]);
    let connection = app.connect(&admin_token(), 30).await;
    assert_eq!(app.purchases(&admin_token()).await.len(), 1);

    // 150 rows on one day (more than the cap), 5 elsewhere; the pending purchase has vanished.
    let mut rows = many("busy", &days_ago(7), 150);
    rows.extend(many("calm", &days_ago(27), 5));
    app.stub.set_transactions(rows);
    app.stub.clear();
    app.refetch(&admin_token(), &connection).await;

    let asked = windows_asked(&app);
    let busy = days_ago(7);
    assert_eq!(
        asked.first().unwrap(),
        &(days_ago(29), days_ago(0)),
        "{asked:?}"
    );
    assert!(
        asked.contains(&(busy.clone(), busy.clone())),
        "split down to the one day: {asked:?}"
    );
    assert_eq!(asked.len(), 11, "{asked:?}");
    let purchases = app.purchases(&admin_token()).await;
    assert_eq!(
        purchases.len(),
        1 + 100 + 5,
        "the truncated day kept as returned"
    );
    assert!(
        purchases
            .iter()
            .any(|p| p["merchant"] == "Testbutiken Väntar"),
        "an incomplete fetch deletes no pending purchase"
    );
    assert_eq!(
        app.connection(&admin_token()).await["last_fetch_error"],
        Value::Null
    );
}

#[tokio::test]
async fn rows_returned_outside_their_window_are_deduplicated_by_reference() {
    let app = CardsApp::new().await;
    let mut late = booked("dup", &days_ago(30), "11.00", "Gränsbutiken");
    late["_bank_date"] = json!(days_ago(29));
    let mut early = booked("dup", &days_ago(30), "99.00", "Gränsbutiken");
    early["_bank_date"] = json!(days_ago(30));
    app.stub.set_transactions(vec![late, early]);
    app.connect(&admin_token(), 90).await;
    assert_eq!(windows_asked(&app).len(), 3);
    let purchases = app.purchases(&admin_token()).await;
    assert_eq!(purchases.len(), 1, "{purchases:#?}");
    assert_eq!(
        purchases[0]["amount_ore"], 1100,
        "the newer window's occurrence wins"
    );
    assert_eq!(
        purchases[0]["purchase_date"],
        days_ago(30),
        "kept as returned"
    );
}

#[tokio::test]
async fn a_continuation_key_is_still_followed() {
    let app = CardsApp::new().await;
    *app.stub.state.paginate.lock().unwrap() = true;
    app.stub.set_transactions(many("page", &days_ago(3), 250));
    app.connect(&admin_token(), 7).await;
    let requests = app.stub.transaction_requests();
    assert_eq!(requests.len(), 3);
    assert_eq!(requests[1].query["continuation_key"], "100");
    assert_eq!(
        requests[2].query["date_from"],
        days_ago(6),
        "within its window"
    );
    assert_eq!(app.purchases(&admin_token()).await.len(), 250);
}

#[tokio::test]
async fn wrong_transactions_period_ends_the_walk_back_and_the_fetch_succeeds() {
    let app = CardsApp::new().await;
    *app.stub.state.history_from.lock().unwrap() =
        Some(today().checked_sub_days(Days::new(75)).unwrap());
    app.stub.set_transactions(vec![
        booked("a", &days_ago(5), "10.00", "Butik A"),
        booked("b", &days_ago(65), "20.00", "Butik B"),
    ]);
    app.connect(&admin_token(), 180).await;
    assert_eq!(
        windows_asked(&app).len(),
        4,
        "the fourth window answered WRONG_TRANSACTIONS_PERIOD"
    );
    let c = app.connection(&admin_token()).await;
    assert_eq!(c["last_fetch_error"], Value::Null);
    assert!(c["last_fetch_at"].is_string());
    assert_eq!(app.purchases(&admin_token()).await.len(), 2);
}

#[tokio::test]
async fn transactions_map_per_cp_7_and_personal_fields_are_never_stored_or_served() {
    let app = CardsApp::new().await;
    let mut fee = booked("fee", &days_ago(2), "25.00", "Årsavgift");
    fee["bank_transaction_code"] = json!({"code": "FEE"});
    let mut foreign = booked("eur", &days_ago(3), "-12.50", "Museum Example");
    foreign["transaction_amount"]["currency"] = json!("EUR");
    let mut no_ref = booked("x", &days_ago(4), "8.00", "Utan Referens");
    no_ref.as_object_mut().unwrap().remove("entry_reference");
    let mut twin = no_ref.clone();
    twin["transaction_id"] = json!("tid-twin");
    let mut multi = booked("multi", &days_ago(5), "1234.56", "x");
    multi["remittance_information"] = json!(["  Rad   ett ", "rad två"]);
    let mut nameless = booked("nameless", &days_ago(5), "3.00", "");
    nameless["remittance_information"] = json!([]);
    app.stub.set_transactions(vec![
        tx(
            Some("credit"),
            "BOOK",
            "CRDT",
            &days_ago(1),
            "5000.00",
            "Inbetalning",
        ),
        fee,
        foreign,
        no_ref,
        twin,
        multi,
        nameless,
        booked("zero", &days_ago(6), "0.00", "Nollbelopp"),
        tx(
            Some("rjct"),
            "RJCT",
            "DBIT",
            &days_ago(6),
            "9.00",
            "Avvisad",
        ),
    ]);
    app.connect(&admin_token(), 30).await;
    let purchases = app.purchases(&admin_token()).await;
    assert_eq!(
        purchases.len(),
        7,
        "zero and rejected skipped: {purchases:#?}"
    );
    let credit = by_merchant(&purchases, "Inbetalning");
    assert_eq!(
        (credit["direction"].as_str(), credit["amount_ore"].as_i64()),
        (Some("credit"), Some(500_000))
    );
    let fee = by_merchant(&purchases, "Årsavgift");
    assert_eq!(
        fee["direction"], "debit",
        "a debit FEE is a purchase to handle (D-179)"
    );
    let eur = by_merchant(&purchases, "Museum Example");
    assert_eq!(
        (
            &eur["amount_ore"],
            &eur["original_amount"],
            &eur["original_currency"]
        ),
        (&Value::Null, &json!("12.50"), &json!("EUR"))
    );
    assert_eq!(
        by_merchant(&purchases, "Rad ett rad två")["amount_ore"],
        123_456
    );
    assert!(
        purchases.iter().any(|p| p["merchant"] == "Kortköp"),
        "no merchant text: Kortköp"
    );
    assert_eq!(
        purchases
            .iter()
            .filter(|p| p["merchant"] == "Utan Referens")
            .count(),
        2,
        "identical rows without reference are told apart by their index"
    );
    let refs: Vec<String> = sqlx::query_scalar(
        "SELECT provider_ref FROM card_purchases WHERE merchant = 'Utan Referens'",
    )
    .fetch_all(&app.state.db)
    .await
    .unwrap();
    assert!(
        refs.iter().all(|r| r.starts_with("h:") && r.len() == 66),
        "{refs:?}"
    );

    let served = app.card_sync(&admin_token()).await.to_string();
    let stored = sqlx::query_scalar::<_, String>(
        "SELECT group_concat(quote(provider_ref) || merchant || coalesce(original_amount, ''), '|') FROM card_purchases",
    )
    .fetch_one(&app.state.db)
    .await
    .unwrap();
    let connection: String =
        sqlx::query_scalar("SELECT session_id || account_uid FROM card_connections")
            .fetch_one(&app.state.db)
            .await
            .unwrap();
    assert_eq!(connection, "sess-secret-1acc-secret-1", "kept server-side");
    for never in [
        HOLDER_NAME,
        CARD_NUMBER,
        "4711",
        MERCHANT_TOWN,
        "tid-",
        "PURCHASE",
        "FEE",
    ] {
        assert!(!served.contains(never), "served {never}");
        assert!(!stored.contains(never), "stored {never}");
    }
    for never in [
        "sess-secret-1",
        "acc-secret-1",
        "provider_ref",
        "entry_reference",
    ] {
        assert!(!served.contains(never), "served {never}");
    }
}

#[tokio::test]
async fn pending_turning_booked_keeps_the_purchase_by_reference_or_by_match() {
    let app = CardsApp::new().await;
    let expense = app
        .insert_expense(ACCOUNT_1, GROCERIES, 9900, ADMIN_ID, (4950, 4950), false)
        .await;
    app.stub.set_transactions(vec![
        pending("p1", &days_ago(3), "99.00", "Testbutik Ett"),
        pending("p2", &days_ago(4), "45.00", "Bokhandeln Exempel 123"),
        pending("p3", &days_ago(4), "45.00", "Annan Butik"),
        pending("p4", &days_ago(5), "12.00", "Försvinner"),
        pending("p5", &days_ago(5), "13.00", "Försvinner Ignorerad"),
    ]);
    let connection = app.connect(&admin_token(), 30).await;
    let before = app.purchases(&admin_token()).await;
    let id_of = |m: &str| by_merchant(&before, m)["id"].as_str().unwrap().to_owned();
    let app_ref = &app;
    let put = |id: String, body: Value| {
        let path = format!("/cards/purchases/{id}");
        async move {
            app_ref
                .call(Method::PUT, &path, &admin_token(), Some(body))
                .await
        }
    };
    assert_eq!(
        put(
            id_of("Testbutik Ett"),
            json!({"expense_id": expense, "ignored": false})
        )
        .await
        .status,
        200
    );
    assert_eq!(
        put(
            id_of("Försvinner Ignorerad"),
            json!({"expense_id": null, "ignored": true})
        )
        .await
        .status,
        200
    );

    // p1 booked under its own reference; p2 booked under a new reference; p3, p4, p5 no longer reported.
    app.stub.set_transactions(vec![
        booked("p1", &days_ago(2), "99.00", "Testbutik Ett"),
        booked("b2", &days_ago(2), "45.00", "BOKHANDELN Exempel"),
    ]);
    app.refetch(&admin_token(), &connection).await;
    let after = app.purchases(&admin_token()).await;
    let p1 = after
        .iter()
        .find(|p| p["id"] == id_of("Testbutik Ett").as_str())
        .unwrap();
    assert_eq!(
        (p1["status"].as_str(), p1["booking_date"].as_str()),
        (Some("booked"), Some(days_ago(2).as_str()))
    );
    assert_eq!(p1["expense_id"], expense.as_str(), "the link is kept");
    let p2 = after
        .iter()
        .find(|p| p["id"] == id_of("Bokhandeln Exempel 123").as_str())
        .unwrap();
    assert_eq!(
        p2["status"], "booked",
        "taken over by CP-8's secondary rule"
    );
    assert_eq!(p2["merchant"], "BOKHANDELN Exempel");
    let refs: Vec<String> = sqlx::query_scalar(
        "SELECT provider_ref FROM card_purchases WHERE deleted_at IS NULL ORDER BY provider_ref",
    )
    .fetch_all(&app.state.db)
    .await
    .unwrap();
    assert_eq!(refs, ["b2", "p1", "p5"]);
    assert!(
        after
            .iter()
            .any(|p| p["merchant"] == "Försvinner Ignorerad"),
        "an ignored pending purchase stays"
    );
    assert_eq!(
        after.len(),
        3,
        "p3 (other merchant) and p4 vanished and were deleted: {after:#?}"
    );
    let tombstones = app.card_sync(&admin_token()).await["purchases"]
        .as_array()
        .unwrap()
        .iter()
        .filter(|p| !p["deleted_at"].is_null())
        .cloned()
        .collect::<Vec<_>>();
    assert_eq!(tombstones.len(), 2);
    for t in tombstones {
        assert_eq!(
            (t["merchant"].as_str(), t["purchase_date"].as_str()),
            (Some(""), Some("1970-01-01"))
        );
        assert_eq!(
            (&t["amount_ore"], &t["expense_id"], &t["ignored"]),
            (&Value::Null, &Value::Null, &json!(false))
        );
    }
}

#[tokio::test]
async fn bank_errors_are_recorded_per_cp_14_and_cleared_by_a_success() {
    let app = CardsApp::new().await;
    let connection = app.connect(&admin_token(), 7).await;
    let first_fetch = app.connection(&admin_token()).await["last_fetch_at"].clone();
    for (answer, recorded) in [
        (eb_error(500, "ASPSP_ERROR"), "bank_unavailable"),
        (eb_error(503, ""), "bank_unavailable"),
        (eb_error(429, "ASPSP_RATE_LIMIT_EXCEEDED"), "rate_limited"),
        (eb_error(400, "WRONG_REQUEST_PARAMETERS"), "failed"),
    ] {
        app.stub.script_transactions(&[answer]);
        let reply = app.refetch(&admin_token(), &connection).await;
        assert_eq!(
            reply.json["fetched"], true,
            "a failed fetch still answers 200"
        );
        let c = app.connection(&admin_token()).await;
        assert_eq!(c["last_fetch_error"], recorded);
        assert_eq!(c["status"], "active");
        assert_eq!(
            c["last_fetch_at"], first_fetch,
            "last_fetch_at is the last successful fetch"
        );
    }
    app.refetch(&admin_token(), &connection).await;
    assert_eq!(
        app.connection(&admin_token()).await["last_fetch_error"],
        Value::Null
    );

    // An expired session at the bank expires the connection; purchases stay; refresh is refused.
    app.stub
        .set_transactions(vec![booked("k", &days_ago(1), "5.00", "Kvar")]);
    app.refetch(&admin_token(), &connection).await;
    app.stub
        .script_transactions(&[eb_error(401, "EXPIRED_SESSION")]);
    app.refetch(&admin_token(), &connection).await;
    let c = app.connection(&admin_token()).await;
    assert_eq!(
        (c["status"].as_str(), c["last_fetch_error"].as_str()),
        (Some("expired"), Some("consent_expired"))
    );
    assert_eq!(app.purchases(&admin_token()).await.len(), 1);
    let now = app.state.clock.now();
    app.clock.set(now + chrono::Duration::seconds(61));
    assert_error(
        &app.refresh(&admin_token(), &connection).await,
        409,
        "card_consent_expired",
    );
}