diff --git a/server/Cargo.lock b/server/Cargo.lock index 9072358..28a8e5d 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -473,6 +473,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctutils" version = "0.4.3" @@ -1197,8 +1218,15 @@ dependencies = [ name = "kvitt-import" version = "0.1.0" dependencies = [ + "chrono", + "clap", + "csv", "kvitt-domain", "kvitt-server", + "serde_json", + "sqlx", + "tokio", + "uuid", ] [[package]] diff --git a/server/crates/kvitt-import/Cargo.toml b/server/crates/kvitt-import/Cargo.toml index 7a28f95..4b16ba3 100644 --- a/server/crates/kvitt-import/Cargo.toml +++ b/server/crates/kvitt-import/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kvitt-import" -description = "Splitwise CSV importer for Kvitt (SPEC §10.8). Stub until milestone M9." +description = "Splitwise CSV importer for Kvitt (SPEC §10.8)." version.workspace = true edition.workspace = true rust-version.workspace = true @@ -9,6 +9,23 @@ publish.workspace = true [lints] workspace = true +[lib] +path = "src/lib.rs" + +[[bin]] +name = "kvitt-import" +path = "src/main.rs" + [dependencies] +chrono = { version = "0.4.45", default-features = false, features = ["clock", "std"] } +clap = { version = "4.6.7", features = ["derive"] } +csv = "1.4.0" kvitt-domain = { version = "0.1.0", path = "../kvitt-domain" } kvitt-server = { version = "0.1.0", path = "../kvitt-server" } +serde_json = "1.0.154" +sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "sqlite"] } +tokio = { version = "1.53.3", features = ["rt"] } +uuid = { version = "1.28.0", features = ["v7"] } + +[dev-dependencies] +tokio = { version = "1.53.3", features = ["rt", "macros"] } diff --git a/server/crates/kvitt-import/data/splitwise_categories.csv b/server/crates/kvitt-import/data/splitwise_categories.csv new file mode 100644 index 0000000..510f7ac --- /dev/null +++ b/server/crates/kvitt-import/data/splitwise_categories.csv @@ -0,0 +1,74 @@ +splitwise_name,language,default_key +Groceries,en,groceries +Food and drink - Other,en,groceries +Liquor,en,groceries +Dining out,en,restaurants +Electronics,en,home +Furniture,en,home +Household supplies,en,home +Maintenance,en,home +Mortgage,en,home +Rent,en,home +Services,en,home +Pets,en,home +Home - Other,en,home +Electricity,en,utilities +Heat/gas,en,utilities +Water,en,utilities +Trash,en,utilities +TV/Phone/Internet,en,utilities +Cleaning,en,utilities +Utilities - Other,en,utilities +Bicycle,en,transport +Bus/train,en,transport +Car,en,transport +Gas/fuel,en,transport +Parking,en,transport +Taxi,en,transport +Transportation - Other,en,transport +Hotel,en,travel +Plane,en,travel +Games,en,entertainment +Movies,en,entertainment +Music,en,entertainment +Sports,en,entertainment +Entertainment - Other,en,entertainment +Medical expenses,en,health +Clothing,en,shopping +Gifts,en,shopping +General,en,other +Childcare,en,other +Education,en,other +Insurance,en,other +Taxes,en,other +Life - Other,en,other +Other,en,other +Livsmedel,sv,groceries +Mat och dryck - Övrigt,sv,groceries +Alkohol,sv,groceries +Restaurangbesök,sv,restaurants +Elektronik,sv,home +Möbler,sv,home +Förbrukningsvaror,sv,home +Skötsel/underhåll,sv,home +Avbetalning/Amortering,sv,home +Hyra,sv,home +Hem - Övrigt,sv,home +TV/telefon/internet,sv,utilities +Buss/tåg,sv,transport +Bil,sv,transport +Bensin/bränsle,sv,transport +Parkering,sv,transport +Taxi,sv,transport +Transport - Övrigt,sv,transport +Hotell,sv,travel +Musik,sv,entertainment +Filmer,sv,entertainment +Sport,sv,entertainment +Underhållning - Övrigt,sv,entertainment +Sjukvård/medicin,sv,health +Presenter,sv,shopping +Allmänt,sv,other +Barnomsorg,sv,other +Försäkringar,sv,other +Skatter,sv,other diff --git a/server/crates/kvitt-import/src/amount.rs b/server/crates/kvitt-import/src/amount.rs new file mode 100644 index 0000000..e2ae3a0 --- /dev/null +++ b/server/crates/kvitt-import/src/amount.rs @@ -0,0 +1,156 @@ +//! Exact decimal parsing (SPEC §10.8: amounts are decimal numbers with `.` as separator, converted to öre exactly by +//! string parsing) and the `--fx` conversion `round_half_up(amount_ore × rate)`. + +/// Parses `-?digits(.d{1,2})?` (surrounding whitespace ignored) into öre. Never uses floating point. +pub fn parse_ore(text: &str) -> Result { + let trimmed = text.trim(); + let (negative, body) = match trimmed.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, trimmed), + }; + let (whole, fraction) = match body.split_once('.') { + Some((whole, fraction)) => (whole, fraction), + None => (body, ""), + }; + let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + if !digits(whole) || (body.contains('.') && !digits(fraction)) || fraction.len() > 2 { + return Err(format!( + "{trimmed:?} is not a decimal amount with at most two decimals" + )); + } + let overflow = || format!("{trimmed:?} is too large"); + let whole: i64 = whole.parse().map_err(|_| overflow())?; + let fraction: i64 = match fraction.len() { + 0 => 0, + 1 => fraction.parse::().map_err(|_| overflow())? * 10, + _ => fraction.parse().map_err(|_| overflow())?, + }; + let ore = whole + .checked_mul(100) + .and_then(|v| v.checked_add(fraction)) + .ok_or_else(overflow)?; + Ok(if negative { -ore } else { ore }) +} + +/// Formats öre as `-1234.50` (for the summary). +pub fn format_ore(ore: i64) -> String { + let sign = if ore < 0 { "-" } else { "" }; + let abs = ore.unsigned_abs(); + format!("{sign}{}.{:02}", abs / 100, abs % 100) +} + +/// A `--fx` rate: SEK per unit of the foreign currency, an exact positive decimal `num / den`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Rate { + num: i128, + den: i128, +} + +/// Most decimals a rate may have. +const RATE_MAX_DECIMALS: usize = 9; + +impl Rate { + /// Parses `digits(.digits)?`, greater than zero, at most nine decimals. + pub fn parse(text: &str) -> Result { + let trimmed = text.trim(); + let (whole, fraction) = trimmed.split_once('.').unwrap_or((trimmed, "")); + let digits = |s: &str| s.bytes().all(|b| b.is_ascii_digit()); + if whole.is_empty() + || !digits(whole) + || !digits(fraction) + || (trimmed.contains('.') && fraction.is_empty()) + || fraction.len() > RATE_MAX_DECIMALS + || whole.len() > 12 + { + return Err(format!( + "{trimmed:?} is not a positive decimal rate (SEK per unit)" + )); + } + let joined = format!("{whole}{fraction}"); + let num: i128 = joined + .parse() + .map_err(|_| format!("{trimmed:?} is not a rate"))?; + if num == 0 { + return Err(format!("{trimmed:?}: the rate must be greater than zero")); + } + let den = 10_i128.pow(u32::try_from(fraction.len()).unwrap_or(0)); + Ok(Self { num, den }) + } + + /// `round_half_up(ore × rate)` (SPEC §10.8). Negative values are converted by magnitude with the sign kept, so + /// a value and its negation convert to negations of each other. + pub fn convert(self, ore: i64) -> Result { + let magnitude = i128::from(ore.unsigned_abs()); + let scaled = (magnitude * self.num * 2 + self.den) / (2 * self.den); + let value = + i64::try_from(scaled).map_err(|_| format!("{ore} öre converts out of range"))?; + Ok(if ore < 0 { -value } else { value }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_exactly() { + assert_eq!(parse_ore("12.50"), Ok(1250)); + assert_eq!(parse_ore("12.5"), Ok(1250)); + assert_eq!(parse_ore("12"), Ok(1200)); + assert_eq!(parse_ore("0.05"), Ok(5)); + assert_eq!(parse_ore("-3.10"), Ok(-310)); + assert_eq!(parse_ore(" 7.00 "), Ok(700)); + assert_eq!(parse_ore("0.00"), Ok(0)); + // Values float parsing would get wrong. + assert_eq!(parse_ore("0.29"), Ok(29)); + assert_eq!(parse_ore("1234567.89"), Ok(123_456_789)); + } + + #[test] + fn rejects_other_forms() { + for bad in [ + "", " ", "1.234", "1,50", "abc", "1.", ".5", "+1", "--1", "1e3", "1.2.3", + ] { + assert!(parse_ore(bad).is_err(), "{bad:?}"); + } + assert!(parse_ore("99999999999999999999").is_err()); + } + + #[test] + fn formats() { + assert_eq!(format_ore(123_450), "1234.50"); + assert_eq!(format_ore(-5), "-0.05"); + assert_eq!(format_ore(0), "0.00"); + } + + #[test] + fn rate_rounds_half_up() { + let rate = Rate::parse("11.5").unwrap(); + assert_eq!(rate.convert(1000), Ok(11500)); + // 1 öre × 0.5 = 0.5 → 1 (half up); 3 × 0.5 = 1.5 → 2. + let half = Rate::parse("0.5").unwrap(); + assert_eq!(half.convert(1), Ok(1)); + assert_eq!(half.convert(3), Ok(2)); + assert_eq!(half.convert(-3), Ok(-2)); + // 0.4 → 0. + assert_eq!(Rate::parse("0.4").unwrap().convert(1), Ok(0)); + assert_eq!(Rate::parse("10").unwrap().convert(123), Ok(1230)); + } + + #[test] + fn rate_rejects_bad_values() { + for bad in [ + "", + "0", + "0.000", + "-1", + "1,5", + "abc", + "1.", + ".5", + "1.0000000001", + ] { + assert!(Rate::parse(bad).is_err(), "{bad:?}"); + } + } +} diff --git a/server/crates/kvitt-import/src/categories.rs b/server/crates/kvitt-import/src/categories.rs new file mode 100644 index 0000000..165f7bc --- /dev/null +++ b/server/crates/kvitt-import/src/categories.rs @@ -0,0 +1,157 @@ +//! The Splitwise category mapping (SPEC §10.8, D-003): `data/splitwise_categories.csv`, columns +//! `splitwise_name,language,default_key`, English and Swedish names, matched case-insensitively and exactly. It is +//! the only mapping; a name it does not hold maps to `other` and is counted as unmapped. + +use std::collections::BTreeMap; + +/// The committed table, embedded at build time from the crate's data file. +pub const TABLE_CSV: &str = include_str!("../data/splitwise_categories.csv"); + +/// The Kvitt default category keys (SPEC §14.1 `categories.default_key`). +pub const DEFAULT_KEYS: [&str; 10] = [ + "groceries", + "restaurants", + "home", + "utilities", + "transport", + "travel", + "entertainment", + "health", + "shopping", + "other", +]; + +/// The key unmapped names fall back to. +pub const FALLBACK_KEY: &str = "other"; + +/// One line of the table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + pub splitwise_name: String, + /// `en` or `sv`. + pub language: String, + pub default_key: String, +} + +/// The loaded table. +#[derive(Debug, Clone)] +pub struct CategoryTable { + pub entries: Vec, + by_name: BTreeMap, +} + +/// The case-insensitive form of a category name. +fn fold(name: &str) -> String { + name.trim().to_lowercase() +} + +impl CategoryTable { + /// Parses and checks a table: the exact header, languages `en`/`sv`, known keys, no (name, language) twice and + /// no name mapped to two keys. + pub fn parse(text: &str) -> Result { + let mut reader = csv::ReaderBuilder::new() + .has_headers(true) + .from_reader(text.as_bytes()); + let header = reader + .headers() + .map_err(|e| format!("category table: {e}"))? + .clone(); + if header.iter().collect::>() != ["splitwise_name", "language", "default_key"] { + return Err( + "category table: header must be splitwise_name,language,default_key".into(), + ); + } + let mut entries = Vec::new(); + let mut by_name: BTreeMap = BTreeMap::new(); + let mut seen = std::collections::BTreeSet::new(); + for record in reader.records() { + let record = record.map_err(|e| format!("category table: {e}"))?; + let (Some(name), Some(language), Some(key), None) = + (record.get(0), record.get(1), record.get(2), record.get(3)) + else { + return Err(format!( + "category table: line {} must have three fields", + record.position().map_or(0, |p| p.line()) + )); + }; + if name.trim().is_empty() || name.trim() != name { + return Err(format!("category table: bad name {name:?}")); + } + if language != "en" && language != "sv" { + return Err(format!("category table: bad language {language:?}")); + } + if !DEFAULT_KEYS.contains(&key) { + return Err(format!("category table: unknown default_key {key:?}")); + } + if !seen.insert((fold(name), language.to_owned())) { + return Err(format!( + "category table: {name:?} ({language}) appears twice" + )); + } + if let Some(previous) = by_name.insert(fold(name), key.to_owned()) + && previous != key + { + return Err(format!( + "category table: {name:?} maps to both {previous} and {key}" + )); + } + entries.push(Entry { + splitwise_name: name.to_owned(), + language: language.to_owned(), + default_key: key.to_owned(), + }); + } + Ok(Self { entries, by_name }) + } + + /// The committed table. + pub fn committed() -> Result { + Self::parse(TABLE_CSV) + } + + /// The default key of a Splitwise category name (case-insensitive exact match), `None` when unmapped. + pub fn lookup(&self, splitwise_name: &str) -> Option<&str> { + self.by_name.get(&fold(splitwise_name)).map(String::as_str) + } +} + +/// Whether a `Category` value marks a payment row (`Payment` / `Betalning`, SPEC §10.8). +pub fn is_payment(category: &str) -> bool { + let folded = fold(category); + folded == "payment" || folded == "betalning" +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn committed_table_loads() { + let table = CategoryTable::committed().unwrap(); + assert_eq!(table.lookup("livsmedel"), Some("groceries")); + assert_eq!(table.lookup("HEM - ÖVRIGT"), Some("home")); + assert_eq!(table.lookup(" Dining out "), Some("restaurants")); + assert_eq!(table.lookup("Taxi"), Some("transport")); + assert_eq!(table.lookup("Other"), Some("other")); + assert_eq!(table.lookup("Something new"), None); + assert_eq!(table.lookup("Payment"), None); + } + + #[test] + fn rejects_bad_tables() { + let header = "splitwise_name,language,default_key\n"; + assert!(CategoryTable::parse("name,lang,key\nA,en,home\n").is_err()); + assert!(CategoryTable::parse(&format!("{header}A,de,home\n")).is_err()); + assert!(CategoryTable::parse(&format!("{header}A,en,food\n")).is_err()); + assert!(CategoryTable::parse(&format!("{header}A,en,home\na,en,home\n")).is_err()); + assert!(CategoryTable::parse(&format!("{header}A,en,home\nA,sv,travel\n")).is_err()); + assert!(CategoryTable::parse(&format!("{header}A,en,home\nA,sv,home\n")).is_ok()); + } + + #[test] + fn payments() { + assert!(is_payment("Payment")); + assert!(is_payment("betalning")); + assert!(!is_payment("Payments")); + } +} diff --git a/server/crates/kvitt-import/src/cli.rs b/server/crates/kvitt-import/src/cli.rs new file mode 100644 index 0000000..6a7d151 --- /dev/null +++ b/server/crates/kvitt-import/src/cli.rs @@ -0,0 +1,117 @@ +//! The command line of SPEC §10.8. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use kvitt_server::audit::SqlAuditWriter; + +use crate::amount::Rate; +use crate::{EXIT_ERROR, EXIT_USAGE, ImportError, Options, UserArg, split_assignment}; + +#[derive(Debug, Parser)] +#[command( + name = "kvitt-import", + version, + about = "Import one Splitwise group export (CSV) into a Kvitt account (SPEC §10.8).", + long_about = "Import one Splitwise group export (CSV, English or Swedish header) into a Kvitt account \ +(SPEC §10.8). Run with the server stopped, or against a copy of the database. Re-running is safe: rows already \ +imported (same import key) are skipped. Every created row gets an audit entry with source `import`; nothing is \ +pushed. The import commits only when every row is valid and the account balance equals the file's totals row \ +(`Total balance` / `Totalsumma`); otherwise nothing is written.", + after_help = "Exit status: 0 imported (or dry run reconciled), 1 database or file error, 2 bad arguments or \ +users, 3 the file cannot be imported (header, invalid rows, non-SEK rows without --fx), 4 the balance does not \ +reconcile with the totals row. Nothing is written unless the status is 0 and --dry-run is absent." +)] +pub struct Cli { + /// The Kvitt SQLite database (must exist and be migrated). + #[arg(long, value_name = "PATH")] + pub db: PathBuf, + /// The Splitwise export of one group. + #[arg(long, value_name = "CSV")] + pub file: PathBuf, + /// The Kvitt account to import into; created with a 50/50 default ratio if no account has this name. + #[arg(long, value_name = "NAME")] + pub account: String, + /// Maps a member column (the full Splitwise name in the header) to a Kvitt user id or display name. Give it + /// twice, once per household user. + #[arg(long = "user", value_name = "SPLITWISE NAME=USER", required = true)] + pub users: Vec, + /// Converts rows in another currency at this many SEK per unit, e.g. EUR=11.52. Without it non-SEK rows + /// abort the import. + #[arg(long = "fx", value_name = "CUR=SEK PER UNIT")] + pub fx: Vec, + /// Print the summary (and the reconciliation) without writing anything. + #[arg(long)] + pub dry_run: bool, +} + +impl Cli { + /// The run's options, or a usage message. + pub fn options(&self) -> Result { + let users = self + .users + .iter() + .map(|u| { + split_assignment(u).map(|(splitwise_name, kvitt_user)| UserArg { + splitwise_name, + kvitt_user, + }) + }) + .collect::, _>>() + .map_err(ImportError::Usage)?; + let mut fx = BTreeMap::new(); + for text in &self.fx { + let (currency, rate) = split_assignment(text).map_err(ImportError::Usage)?; + let rate = Rate::parse(&rate).map_err(ImportError::Usage)?; + if currency == crate::plan::SEK { + return Err(ImportError::Usage("--fx for SEK makes no sense".into())); + } + if fx.insert(currency.clone(), rate).is_some() { + return Err(ImportError::Usage(format!("--fx {currency} given twice"))); + } + } + Ok(Options { + db: self.db.clone(), + file: self.file.clone(), + account: self.account.clone(), + users, + fx, + dry_run: self.dry_run, + }) + } +} + +/// Parses the command line, runs the import on a single-threaded runtime and prints the summary. +pub fn main() -> ExitCode { + let cli = Cli::parse(); + let options = match cli.options() { + Ok(options) => options, + Err(e) => { + eprintln!("{e}"); + return ExitCode::from(EXIT_USAGE); + } + }; + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(e) => { + eprintln!("cannot start the runtime: {e}"); + return ExitCode::from(EXIT_ERROR); + } + }; + let result = runtime.block_on(crate::run(&options, chrono::Utc::now(), &SqlAuditWriter)); + match result { + Ok(report) => { + println!("{report}"); + ExitCode::from(report.exit_code()) + } + Err(e) => { + eprintln!("{e}"); + ExitCode::from(e.exit_code()) + } + } +} diff --git a/server/crates/kvitt-import/src/db.rs b/server/crates/kvitt-import/src/db.rs new file mode 100644 index 0000000..f0980b0 --- /dev/null +++ b/server/crates/kvitt-import/src/db.rs @@ -0,0 +1,470 @@ +//! Writing a plan into the database (SPEC §10.8): one `BEGIN IMMEDIATE` transaction, the account found or created, +//! rows whose `import_key` exists skipped, every other row validated with the server's own push checks +//! (`kvitt_server::sync::check`), inserted with a new `server_version`, and audited through [`AuditWriter`] with +//! `source` `import`. Nothing goes through a push queue: the importer has none. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use kvitt_domain::{BalanceExpense, BalanceSettlement, compute_balances}; +use kvitt_server::api::{accounts::name_key, rows}; +use kvitt_server::audit::{AuditEvent, AuditWriter}; +use kvitt_server::db as server_db; +use kvitt_server::error::AppError; +use kvitt_server::sync::check::{self, Fail}; +use kvitt_server::sync::rows as sync_rows; +use kvitt_server::time::format_ts; +use kvitt_server::wire::{ + Allocation, AuditAction, AuditSource, EntityType, PushedExpense, PushedSettlement, Split, + SplitMode, User, +}; +use sqlx::SqliteConnection; +use uuid::Uuid; + +use crate::parse::RowProblem; +use crate::plan::{PlannedBody, PlannedExpense, PlannedRow, PlannedSettlement, PlannedSplit}; + +/// Default ratio of a created account: 50/50 (SPEC §10.8). +pub const DEFAULT_SHARE_BP: i64 = 5000; + +/// What writing produced (before the caller commits or rolls back). +#[derive(Debug, Default)] +pub struct Written { + pub account_id: String, + pub account_created: bool, + pub expenses_created: usize, + pub settlements_created: usize, + pub skipped_duplicates: usize, + /// Rows the server's checks rejected; non-empty means the caller must roll back. + pub rejected: Vec, +} + +/// The household context of a run. +pub struct Household<'a> { + pub users: &'a [User], + pub owner: Uuid, +} + +fn uuid(text: &str) -> Result { + rows::uuid(text) +} + +/// Finds the non-deleted account with this name (case-insensitive, SPEC §14.1 `name_key`), else creates it with +/// the default 50/50 ratio, created by the owner. +async fn find_or_create_account( + conn: &mut SqliteConnection, + name: &str, + household: &Household<'_>, + now_ts: &str, +) -> Result<(String, bool), AppError> { + let key = name_key(name); + let existing: Option = + sqlx::query_scalar("SELECT id FROM accounts WHERE name_key = ? AND deleted_at IS NULL") + .bind(&key) + .fetch_optional(&mut *conn) + .await?; + if let Some(id) = existing { + return Ok((id, false)); + } + let id = Uuid::now_v7().to_string(); + let version = server_db::next_server_version(conn).await?; + sqlx::query( + "INSERT INTO accounts (id, name, name_key, created_by_user_id, created_at, updated_at, archived_at, \ + deleted_at, server_version) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?)", + ) + .bind(&id) + .bind(name) + .bind(&key) + .bind(household.owner.to_string()) + .bind(now_ts) + .bind(now_ts) + .bind(version) + .execute(&mut *conn) + .await?; + for user in household.users { + sqlx::query( + "INSERT INTO account_default_shares (account_id, user_id, share_bp) VALUES (?, ?, ?)", + ) + .bind(&id) + .bind(user.id.to_string()) + .bind(DEFAULT_SHARE_BP) + .execute(&mut *conn) + .await?; + } + Ok((id, true)) +} + +/// The id of each non-deleted default category, by key. +async fn default_categories( + conn: &mut SqliteConnection, +) -> Result, AppError> { + let pairs: Vec<(String, String)> = sqlx::query_as( + "SELECT default_key, id FROM categories WHERE default_key IS NOT NULL AND deleted_at IS NULL", + ) + .fetch_all(&mut *conn) + .await?; + Ok(pairs.into_iter().collect()) +} + +async fn key_exists(conn: &mut SqliteConnection, key: &str) -> Result { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM expenses WHERE import_key = ?1) \ + OR EXISTS (SELECT 1 FROM settlements WHERE import_key = ?1)", + ) + .bind(key) + .fetch_one(&mut *conn) + .await?; + Ok(exists) +} + +fn pushed_expense( + e: &PlannedExpense, + users: &[User], + account_id: Uuid, + category_id: Uuid, + now_ts: &str, +) -> Result { + let (split_mode, splits) = match &e.split { + PlannedSplit::Exact(shares) => ( + SplitMode::Exact, + shares + .iter() + .map(|(user, share)| { + Ok(Split { + user_id: uuid(user)?, + share_ore: *share, + share_bp: None, + }) + }) + .collect::, AppError>>()?, + ), + PlannedSplit::Full(bearer) => (SplitMode::Full, full_splits(e.amount_ore, bearer, users)?), + }; + Ok(PushedExpense { + id: Uuid::now_v7(), + account_id, + category_id, + amount_ore: e.amount_ore, + description: e.description.clone(), + notes: e.notes.clone(), + expense_date: e.date.clone(), + payer_user_id: uuid(&e.payer_user_id)?, + split_mode, + splits, + created_at: now_ts.to_owned(), + updated_at: now_ts.to_owned(), + deleted_at: None, + }) +} + +/// The splits of mode `full` (SPEC §9.1: bearer 10000 bp and the whole amount, the other user 0). +fn full_splits(amount_ore: i64, bearer: &str, users: &[User]) -> Result, AppError> { + users + .iter() + .map(|u| { + let bears = u.id.to_string() == bearer; + Ok(Split { + user_id: u.id, + share_ore: if bears { amount_ore } else { 0 }, + share_bp: Some(if bears { 10_000 } else { 0 }), + }) + }) + .collect() +} + +fn pushed_settlement( + s: &PlannedSettlement, + account_id: Uuid, + now_ts: &str, +) -> Result { + Ok(PushedSettlement { + id: Uuid::now_v7(), + from_user_id: uuid(&s.from_user_id)?, + to_user_id: uuid(&s.to_user_id)?, + amount_ore: s.amount_ore, + settlement_date: s.date.clone(), + note: s.note.clone(), + allocations: vec![Allocation { + account_id, + amount_ore: s.amount_ore, + }], + created_at: now_ts.to_owned(), + updated_at: now_ts.to_owned(), + deleted_at: None, + }) +} + +async fn insert_expense( + conn: &mut SqliteConnection, + p: &PushedExpense, + import_key: &str, + owner: Uuid, + now_ts: &str, +) -> Result<(), AppError> { + let version = server_db::next_server_version(conn).await?; + sqlx::query( + "INSERT INTO expenses (id, account_id, category_id, amount_ore, description, notes, expense_date, \ + payer_user_id, split_mode, recurring_template_id, recurring_period, import_key, created_by_user_id, \ + updated_by_user_id, updated_by_device_id, created_at, updated_at, deleted_at, server_received_at, \ + server_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?, NULL, ?, ?, NULL, ?, ?)", + ) + .bind(p.id.to_string()) + .bind(p.account_id.to_string()) + .bind(p.category_id.to_string()) + .bind(p.amount_ore) + .bind(&p.description) + .bind(&p.notes) + .bind(&p.expense_date) + .bind(p.payer_user_id.to_string()) + .bind(rows::token_of(&p.split_mode)?) + .bind(import_key) + .bind(owner.to_string()) + .bind(owner.to_string()) + .bind(now_ts) + .bind(now_ts) + .bind(now_ts) + .bind(version) + .execute(&mut *conn) + .await?; + for split in &p.splits { + sqlx::query( + "INSERT INTO expense_splits (expense_id, user_id, share_ore, share_bp) VALUES (?, ?, ?, ?)", + ) + .bind(p.id.to_string()) + .bind(split.user_id.to_string()) + .bind(split.share_ore) + .bind(split.share_bp) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +async fn insert_settlement( + conn: &mut SqliteConnection, + p: &PushedSettlement, + import_key: &str, + owner: Uuid, + now_ts: &str, +) -> Result<(), AppError> { + let version = server_db::next_server_version(conn).await?; + sqlx::query( + "INSERT INTO settlements (id, from_user_id, to_user_id, amount_ore, settlement_date, note, import_key, \ + created_by_user_id, updated_by_user_id, updated_by_device_id, created_at, updated_at, deleted_at, \ + server_received_at, server_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, NULL, ?, ?)", + ) + .bind(p.id.to_string()) + .bind(p.from_user_id.to_string()) + .bind(p.to_user_id.to_string()) + .bind(p.amount_ore) + .bind(&p.settlement_date) + .bind(&p.note) + .bind(import_key) + .bind(owner.to_string()) + .bind(owner.to_string()) + .bind(now_ts) + .bind(now_ts) + .bind(now_ts) + .bind(version) + .execute(&mut *conn) + .await?; + for allocation in &p.allocations { + sqlx::query( + "INSERT INTO settlement_allocations (settlement_id, account_id, amount_ore) VALUES (?, ?, ?)", + ) + .bind(p.id.to_string()) + .bind(allocation.account_id.to_string()) + .bind(allocation.amount_ore) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +async fn audit_create( + conn: &mut SqliteConnection, + audit: &dyn AuditWriter, + entity_type: EntityType, + entity_id: Uuid, + after: serde_json::Value, + owner: Uuid, + now_ts: &str, +) -> Result<(), AppError> { + audit + .write( + conn, + AuditEvent { + entity_type, + entity_id, + action: AuditAction::Create, + source: AuditSource::Import, + actor_user_id: owner, + actor_device_id: None, + client_at: now_ts.to_owned(), + server_at: now_ts.to_owned(), + before: None, + after, + }, + ) + .await?; + Ok(()) +} + +/// Turns a server check failure into a row problem, or passes a real error through. +fn rejected(line: u64, fail: Fail) -> Result { + match fail { + Fail::Reject(r) => Ok(RowProblem { + line, + reason: format!("{}: {}", rows::token_of(&r.reason)?, r.message), + }), + Fail::Internal(e) => Err(e), + } +} + +/// Writes the planned rows on `conn` (inside the caller's `BEGIN IMMEDIATE` transaction). +pub async fn write( + conn: &mut SqliteConnection, + account_name: &str, + planned: &[PlannedRow], + household: &Household<'_>, + audit: &dyn AuditWriter, + now: DateTime, +) -> Result { + let now_ts = format_ts(now); + let (account_id, account_created) = + find_or_create_account(conn, account_name, household, &now_ts).await?; + let account_uuid = uuid(&account_id)?; + let categories = default_categories(conn).await?; + let owner = household.owner; + let mut written = Written { + account_id, + account_created, + ..Written::default() + }; + for row in planned { + if key_exists(conn, &row.import_key).await? { + written.skipped_duplicates += 1; + continue; + } + match &row.body { + PlannedBody::Expense(e) => { + let category_id = categories.get(&e.category_key).ok_or_else(|| { + AppError::internal(format!("default category {} is missing", e.category_key)) + })?; + let pushed = pushed_expense( + e, + household.users, + account_uuid, + uuid(category_id)?, + &now_ts, + )?; + let checked = match check::expense(conn, &pushed, household.users, now).await { + Ok(checked) => checked, + Err(fail) => { + written.rejected.push(rejected(row.line, fail)?); + continue; + } + }; + insert_expense(conn, &checked, &row.import_key, owner, &now_ts).await?; + let stored = rows::expense(conn, checked.id) + .await? + .ok_or_else(|| AppError::internal("imported expense vanished"))?; + let after = serde_json::to_value(&stored).map_err(AppError::internal)?; + audit_create( + conn, + audit, + EntityType::Expense, + checked.id, + after, + owner, + &now_ts, + ) + .await?; + written.expenses_created += 1; + } + PlannedBody::Settlement(s) => { + let pushed = pushed_settlement(s, account_uuid, &now_ts)?; + let checked = match check::settlement(conn, &pushed, household.users, now).await { + Ok(checked) => checked, + Err(fail) => { + written.rejected.push(rejected(row.line, fail)?); + continue; + } + }; + insert_settlement(conn, &checked, &row.import_key, owner, &now_ts).await?; + let stored = sync_rows::settlement(conn, checked.id) + .await? + .ok_or_else(|| AppError::internal("imported settlement vanished"))?; + let after = serde_json::to_value(&stored).map_err(AppError::internal)?; + audit_create( + conn, + audit, + EntityType::Settlement, + checked.id, + after, + owner, + &now_ts, + ) + .await?; + written.settlements_created += 1; + } + } + } + Ok(written) +} + +/// The account's balance per user over its non-deleted expenses and settlement allocations (SPEC §9.3), computed +/// with `kvitt_domain::compute_balances`. +pub async fn account_balance( + conn: &mut SqliteConnection, + account_id: &str, + user_ids: &[String], +) -> Result, AppError> { + let heads: Vec<(String, i64, String)> = sqlx::query_as( + "SELECT id, amount_ore, payer_user_id FROM expenses WHERE account_id = ? AND deleted_at IS NULL", + ) + .bind(account_id) + .fetch_all(&mut *conn) + .await?; + let splits: Vec<(String, String, i64)> = sqlx::query_as( + "SELECT s.expense_id, s.user_id, s.share_ore FROM expense_splits s JOIN expenses e ON e.id = s.expense_id \ + WHERE e.account_id = ? AND e.deleted_at IS NULL", + ) + .bind(account_id) + .fetch_all(&mut *conn) + .await?; + let mut shares: BTreeMap> = BTreeMap::new(); + for (expense_id, user_id, share) in splits { + shares.entry(expense_id).or_default().insert(user_id, share); + } + let expenses: Vec = heads + .into_iter() + .map(|(id, amount_ore, payer_user_id)| BalanceExpense { + account_id: account_id.to_owned(), + amount_ore, + payer_user_id, + shares: shares.remove(&id).unwrap_or_default(), + }) + .collect(); + let allocations: Vec<(String, String, i64)> = sqlx::query_as( + "SELECT s.from_user_id, s.to_user_id, a.amount_ore FROM settlement_allocations a \ + JOIN settlements s ON s.id = a.settlement_id WHERE a.account_id = ? AND s.deleted_at IS NULL", + ) + .bind(account_id) + .fetch_all(&mut *conn) + .await?; + let settlements: Vec = allocations + .into_iter() + .map(|(from_user_id, to_user_id, amount)| BalanceSettlement { + from_user_id, + to_user_id, + allocations: BTreeMap::from([(account_id.to_owned(), amount)]), + }) + .collect(); + let balances = compute_balances(user_ids, &[account_id.to_owned()], &expenses, &settlements); + Ok(balances + .per_account + .get(account_id) + .cloned() + .unwrap_or_default()) +} diff --git a/server/crates/kvitt-import/src/lib.rs b/server/crates/kvitt-import/src/lib.rs new file mode 100644 index 0000000..1090536 --- /dev/null +++ b/server/crates/kvitt-import/src/lib.rs @@ -0,0 +1,515 @@ +//! `kvitt-import`: imports Splitwise CSV exports, one file per group, into a Kvitt account (SPEC §10.8, FR-17). +//! +//! The run: read and check the file ([`parse`]), resolve the `--user` arguments against the household, build the +//! rows ([`plan`]), then in one `BEGIN IMMEDIATE` transaction find or create the account, skip rows whose +//! `import_key` exists, validate the rest with the server's own push checks, insert and audit them ([`db`]), and +//! reconcile the account balance with the file's totals row. The transaction commits only when every row is valid, +//! the balance matches and this is not a dry run; otherwise it rolls back and nothing is written. + +// No unwrap/expect outside tests (CLAUDE.md server rule); integration tests are separate crates and may use them. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +pub mod amount; +pub mod categories; +pub mod cli; +pub mod db; +pub mod parse; +pub mod plan; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use kvitt_server::api::{rows, validate}; +use kvitt_server::audit::AuditWriter; +use kvitt_server::error::AppError; +use kvitt_server::wire::{Role, User}; +use sqlx::SqliteConnection; +use uuid::Uuid; + +use crate::amount::{Rate, format_ore}; +use crate::categories::CategoryTable; +use crate::parse::{FileError, Language, RowProblem}; + +/// Exit status: success. +pub const EXIT_OK: u8 = 0; +/// Exit status: database or file system error. +pub const EXIT_ERROR: u8 = 1; +/// Exit status: bad command line (clap's own usage errors use 2 as well). +pub const EXIT_USAGE: u8 = 2; +/// Exit status: the file cannot be imported (header, invalid rows, non-SEK rows without `--fx`); nothing written. +pub const EXIT_INVALID: u8 = 3; +/// Exit status: the balance does not reconcile with the file's totals row; nothing written. +pub const EXIT_MISMATCH: u8 = 4; + +/// One `--user` argument: the Splitwise member name and the Kvitt user (an id, or a display name to resolve). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserArg { + pub splitwise_name: String, + pub kvitt_user: String, +} + +/// A whole run's inputs. +#[derive(Debug, Clone)] +pub struct Options { + pub db: PathBuf, + pub file: PathBuf, + pub account: String, + pub users: Vec, + pub fx: BTreeMap, + pub dry_run: bool, +} + +/// Why a run failed. +#[derive(Debug)] +pub enum ImportError { + /// Bad arguments or unresolvable users. + Usage(String), + /// The header is not one of the two variants or names other members. + Header(String), + /// Rows that cannot be imported, with their line numbers. + InvalidRows(Vec), + /// Non-SEK rows without a `--fx` rate: currency → rows. + MissingFx(BTreeMap), + /// Database or I/O. + Io(String), +} + +impl ImportError { + pub fn exit_code(&self) -> u8 { + match self { + Self::Usage(_) => EXIT_USAGE, + Self::Header(_) | Self::InvalidRows(_) | Self::MissingFx(_) => EXIT_INVALID, + Self::Io(_) => EXIT_ERROR, + } + } +} + +impl std::fmt::Display for ImportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Usage(m) => write!(f, "usage: {m}"), + Self::Header(m) => write!(f, "header: {m}; nothing written"), + Self::InvalidRows(problems) => { + writeln!(f, "{} invalid rows; nothing written:", problems.len())?; + for p in problems { + writeln!(f, " line {}: {}", p.line, p.reason)?; + } + Ok(()) + } + Self::MissingFx(currencies) => { + write!(f, "rows not in SEK and no --fx rate; nothing written:")?; + for (currency, count) in currencies { + write!(f, " {currency} ({count} rows)")?; + } + Ok(()) + } + Self::Io(m) => write!(f, "error: {m}"), + } + } +} + +impl From for ImportError { + fn from(e: AppError) -> Self { + Self::Io(format!("database: {} ({})", e.message, e.code)) + } +} + +impl From for ImportError { + fn from(e: sqlx::Error) -> Self { + Self::Io(format!("database: {e}")) + } +} + +/// One member's line of the reconciliation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemberBalance { + pub splitwise_name: String, + pub user_id: String, + /// The Splitwise totals row value (converted to SEK öre). + pub splitwise_ore: i64, + /// `net(u)` of the Kvitt account after the import (SPEC §9.3). + pub kvitt_ore: i64, +} + +/// The reconciliation of the account balance with the file's totals row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reconciliation { + /// Per member, in header order; empty without a totals row. + pub members: Vec, + /// Whether the file had a totals row. + pub has_totals: bool, + pub matched: bool, + /// Whether `--fx` conversion took place (rounding per value can then make an exact match impossible, so a + /// mismatch is reported but does not fail the run). + pub converted: bool, +} + +impl Reconciliation { + /// Whether the run may commit: an exact match, or a mismatch caused only by `--fx` rounding. + pub fn accepted(&self) -> bool { + self.has_totals && (self.matched || self.converted) + } +} + +/// What a run did (or, in a dry run, would do). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Report { + pub dry_run: bool, + pub committed: bool, + pub language: Language, + pub account_name: String, + pub account_id: String, + pub account_created: bool, + /// Data rows (expenses and payments); blank and totals rows excluded. + pub rows_read: usize, + pub blank_rows: usize, + pub totals_rows: usize, + pub expenses_created: usize, + pub settlements_created: usize, + pub skipped_duplicates: usize, + /// Unmapped Splitwise category → rows (imported as `other`). + pub unmapped_categories: BTreeMap, + pub non_sek_rows: usize, + pub reconciliation: Reconciliation, +} + +impl Report { + pub fn unmapped_rows(&self) -> usize { + self.unmapped_categories.values().sum() + } + + /// The process exit status for this report. + pub fn exit_code(&self) -> u8 { + if self.reconciliation.accepted() { + EXIT_OK + } else { + EXIT_MISMATCH + } + } +} + +impl std::fmt::Display for Report { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut out = String::new(); + let verb = if self.dry_run { + "would create" + } else { + "created" + }; + let _ = writeln!( + out, + "kvitt-import summary{}", + if self.dry_run { + " (dry run, nothing written)" + } else { + "" + } + ); + let _ = writeln!(out, " header variant: {}", self.language.as_str()); + let _ = writeln!( + out, + " account: {} ({}{})", + self.account_name, + self.account_id, + if self.account_created { + ", new" + } else { + ", existing" + } + ); + let _ = writeln!(out, " rows read: {}", self.rows_read); + let _ = writeln!(out, " totals rows: {}", self.totals_rows); + let _ = writeln!(out, " expenses {verb}: {}", self.expenses_created); + let _ = writeln!(out, " settlements {verb}: {}", self.settlements_created); + let _ = writeln!(out, " skipped duplicates: {}", self.skipped_duplicates); + let _ = writeln!(out, " non-SEK rows: {}", self.non_sek_rows); + let _ = writeln!(out, " unknown categories (imported as other):"); + if self.unmapped_categories.is_empty() { + let _ = writeln!(out, " none"); + } + for (name, count) in &self.unmapped_categories { + let _ = writeln!(out, " {name}: {count} rows"); + } + if self.unmapped_rows() > 0 { + let _ = writeln!( + out, + " WARNING: {} rows with an unmapped category", + self.unmapped_rows() + ); + } + let r = &self.reconciliation; + let _ = writeln!( + out, + " reconciliation (account balance vs the Splitwise totals row, SEK):" + ); + if !r.has_totals { + let _ = writeln!(out, " MISMATCH: the file has no totals row"); + } + for m in &r.members { + let _ = writeln!( + out, + " {}: splitwise {}, kvitt {}", + m.splitwise_name, + format_ore(m.splitwise_ore), + format_ore(m.kvitt_ore) + ); + } + if r.has_totals { + let verdict = match (r.matched, r.converted) { + (true, _) => "match", + (false, true) => "MISMATCH after --fx rounding (accepted)", + (false, false) => "MISMATCH", + }; + let _ = writeln!(out, " result: {verdict}"); + } + let status = if self.committed { + "committed" + } else if self.dry_run { + "rolled back (dry run)" + } else { + "rolled back, nothing written" + }; + let _ = write!(out, " transaction: {status}"); + f.write_str(&out) + } +} + +/// Parses `NAME=VALUE` (the value after the last `=`, so names may not contain `=` but nothing else is special). +pub fn split_assignment(text: &str) -> Result<(String, String), String> { + let (name, value) = text + .rsplit_once('=') + .ok_or_else(|| format!("{text:?} is not NAME=VALUE"))?; + let (name, value) = (name.trim(), value.trim()); + if name.is_empty() || value.is_empty() { + return Err(format!("{text:?} is not NAME=VALUE")); + } + Ok((name.to_owned(), value.to_owned())) +} + +/// Opens an existing database (never creates one) and checks its schema version. +pub async fn open(path: &std::path::Path) -> Result { + if !path.is_file() { + return Err(ImportError::Io(format!( + "no database at {}", + path.display() + ))); + } + let options = kvitt_server::db::pool::options(path).create_if_missing(false); + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + let version: Option = + sqlx::query_scalar("SELECT value FROM meta WHERE key = 'schema_version'") + .fetch_optional(&pool) + .await + .map_err(|e| ImportError::Io(format!("not a Kvitt database: {e}")))?; + let expected = kvitt_server::db::SCHEMA_VERSION.to_string(); + if version.as_deref() != Some(expected.as_str()) { + return Err(ImportError::Io(format!( + "schema version {version:?}, expected {expected}; run `kvitt-server migrate` first" + ))); + } + Ok(pool) +} + +/// Resolves one `--user` value: a user id, or else a display name that exactly one user has. +fn resolve_user(value: &str, users: &[User]) -> Result { + if let Ok(id) = Uuid::parse_str(value) { + return users + .iter() + .find(|u| u.id == id) + .map(|u| u.id) + .ok_or_else(|| ImportError::Usage(format!("no user with id {id}"))); + } + let matches: Vec<&User> = users.iter().filter(|u| u.display_name == value).collect(); + match matches[..] { + [user] => Ok(user.id), + [] => Err(ImportError::Usage(format!( + "no user with id or display name {value:?}" + ))), + _ => Err(ImportError::Usage(format!( + "display name {value:?} is ambiguous; give the user id" + ))), + } +} + +/// The household, the owner and the user id of each member column (header order). +async fn resolve_household( + conn: &mut SqliteConnection, + options: &Options, + members: &[String], +) -> Result<(Vec, Uuid, Vec), ImportError> { + let users = rows::complete_household(conn) + .await + .map_err(|e| ImportError::Usage(format!("household: {}", e.message)))?; + let owner = users + .iter() + .find(|u| u.role == Role::Admin) + .map(|u| u.id) + .ok_or_else(|| ImportError::Usage("the household has no admin (owner)".into()))?; + let mut by_name: BTreeMap<&str, Uuid> = BTreeMap::new(); + for arg in &options.users { + let id = resolve_user(&arg.kvitt_user, &users)?; + by_name.insert(arg.splitwise_name.as_str(), id); + } + let ids: BTreeSet = by_name.values().copied().collect(); + let household: BTreeSet = users.iter().map(|u| u.id).collect(); + if ids != household { + return Err(ImportError::Usage( + "the two --user arguments must name the two household users, one each".into(), + )); + } + let member_ids = members + .iter() + .map(|m| { + by_name + .get(m.as_str()) + .map(Uuid::to_string) + .ok_or_else(|| ImportError::Header(format!("member column {m:?} has no --user"))) + }) + .collect::, _>>()?; + Ok((users, owner, member_ids)) +} + +/// Checks the command line against the file header: two `--user` arguments with distinct names, the same set of +/// names as the header's member columns. +fn check_users(options: &Options, members: &[String]) -> Result<(), ImportError> { + let names: BTreeSet<&str> = options + .users + .iter() + .map(|u| u.splitwise_name.as_str()) + .collect(); + if options.users.len() != 2 || names.len() != 2 { + return Err(ImportError::Usage( + "give exactly two --user arguments with distinct Splitwise names".into(), + )); + } + let header: BTreeSet<&str> = members.iter().map(String::as_str).collect(); + if names != header { + return Err(ImportError::Header(format!( + "the member columns {members:?} do not match the --user names {:?}", + names.into_iter().collect::>() + ))); + } + Ok(()) +} + +/// Runs one import. `now` is the import time (`created_at`, `updated_at`, audit times; dates are checked against +/// it). Returns the report whether or not it committed (see [`Report::committed`]); errors write nothing. +pub async fn run( + options: &Options, + now: DateTime, + audit: &dyn AuditWriter, +) -> Result { + let account_name = validate::text("account", &options.account, 1, validate::ACCOUNT_NAME_MAX) + .map_err(|e| ImportError::Usage(e.message))?; + let table = CategoryTable::committed().map_err(ImportError::Io)?; + let bytes = std::fs::read(&options.file) + .map_err(|e| ImportError::Io(format!("{}: {e}", options.file.display())))?; + let file = parse::parse(&bytes).map_err(|e| match e { + FileError::Header(m) => ImportError::Header(m), + FileError::Csv(m) => ImportError::InvalidRows(vec![RowProblem { line: 0, reason: m }]), + })?; + check_users(options, &file.header.members)?; + + let pool = open(&options.db).await?; + let mut conn = pool.acquire().await?; + let (users, owner, member_ids) = + resolve_household(&mut conn, options, &file.header.members).await?; + // An existing account keeps its stored name, which is also the name in the import key, so a re-run with the + // name in another case still finds its rows. + let existing: Option = + sqlx::query_scalar("SELECT name FROM accounts WHERE name_key = ? AND deleted_at IS NULL") + .bind(kvitt_server::api::accounts::name_key(&account_name)) + .fetch_optional(&mut *conn) + .await?; + let account_name = existing.unwrap_or(account_name); + drop(conn); + + let owner_text = owner.to_string(); + let plan = plan::build( + &file, + &plan::PlanContext { + account_name: &account_name, + member_user_ids: &member_ids, + owner_user_id: &owner_text, + fx: &options.fx, + categories: &table, + }, + ); + if !plan.missing_fx.is_empty() { + return Err(ImportError::MissingFx(plan.missing_fx)); + } + if !plan.problems.is_empty() { + return Err(ImportError::InvalidRows(plan.problems)); + } + + let mut tx = kvitt_server::db::begin_immediate(&pool).await?; + let household = db::Household { + users: &users, + owner, + }; + let written = db::write(&mut tx, &account_name, &plan.rows, &household, audit, now).await?; + if !written.rejected.is_empty() { + tx.rollback().await?; + return Err(ImportError::InvalidRows(written.rejected)); + } + let balance = + db::account_balance(&mut tx, &written.account_id, &rows::user_ids(&users)).await?; + let reconciliation = match &plan.expected_totals { + Some(expected) => { + let members: Vec = file + .header + .members + .iter() + .zip(&member_ids) + .map(|(name, id)| MemberBalance { + splitwise_name: name.clone(), + user_id: id.clone(), + splitwise_ore: expected.get(id).copied().unwrap_or(0), + kvitt_ore: balance.get(id).copied().unwrap_or(0), + }) + .collect(); + let matched = members.iter().all(|m| m.splitwise_ore == m.kvitt_ore); + Reconciliation { + members, + has_totals: true, + matched, + converted: plan.converted, + } + } + None => Reconciliation { + members: Vec::new(), + has_totals: false, + matched: false, + converted: plan.converted, + }, + }; + let commit = !options.dry_run && reconciliation.accepted(); + if commit { + tx.commit().await?; + } else { + tx.rollback().await?; + } + pool.close().await; + Ok(Report { + dry_run: options.dry_run, + committed: commit, + language: file.header.language, + account_name, + account_id: written.account_id, + account_created: written.account_created, + rows_read: file.rows.len(), + blank_rows: file.blank_rows, + totals_rows: file.totals.len(), + expenses_created: written.expenses_created, + settlements_created: written.settlements_created, + skipped_duplicates: written.skipped_duplicates, + unmapped_categories: plan.unmapped, + non_sek_rows: plan.non_sek_rows, + reconciliation, + }) +} diff --git a/server/crates/kvitt-import/src/main.rs b/server/crates/kvitt-import/src/main.rs index 1629148..64400af 100644 --- a/server/crates/kvitt-import/src/main.rs +++ b/server/crates/kvitt-import/src/main.rs @@ -1,20 +1,10 @@ -//! `kvitt-import`: the Splitwise CSV importer (SPEC §10.8). M9 owns this crate; in M2a it prints its usage and -//! exits 2. +//! `kvitt-import`: the Splitwise CSV importer (SPEC §10.8). The work is in the library (`kvitt_import`). // No unwrap/expect outside tests (CLAUDE.md server rule); integration tests are separate crates and may use them. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] use std::process::ExitCode; -// The importer reuses the server crate for database access and audit writing and the domain crate for the money -// rules (SPEC §6.3); referenced here so the dependency edges exist from the start. -use kvitt_domain as _; -use kvitt_server as _; - -const USAGE: &str = - "usage: kvitt-import (Splitwise CSV importer, SPEC §10.8; not implemented in M2a)"; - fn main() -> ExitCode { - eprintln!("{USAGE}"); - ExitCode::from(2) + kvitt_import::cli::main() } diff --git a/server/crates/kvitt-import/src/parse.rs b/server/crates/kvitt-import/src/parse.rs new file mode 100644 index 0000000..0344f38 --- /dev/null +++ b/server/crates/kvitt-import/src/parse.rs @@ -0,0 +1,288 @@ +//! Reading a Splitwise export (SPEC §10.8): CSV quoting, the English or Swedish header with the member columns +//! named by full Splitwise name, and row classification (data rows, the trailing totals row, blank rows). + +use std::collections::BTreeMap; + +/// The fixed leading columns of the English export. +pub const ENGLISH_COLUMNS: [&str; 5] = ["Date", "Description", "Category", "Cost", "Currency"]; +/// The fixed leading columns of the Swedish export. +pub const SWEDISH_COLUMNS: [&str; 5] = ["Datum", "Beskrivning", "Kategori", "Kostnad", "Valuta"]; +/// `Description` of the trailing totals row, per language. +pub const TOTAL_DESCRIPTIONS: [&str; 2] = ["Total balance", "Totalsumma"]; +/// Member columns per file: the household has exactly two users. +pub const MEMBER_COLUMNS: usize = 2; + +/// The export's UI language. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Language { + English, + Swedish, +} + +impl Language { + pub fn as_str(self) -> &'static str { + match self { + Self::English => "English", + Self::Swedish => "Swedish", + } + } +} + +/// The parsed header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Header { + pub language: Language, + /// Member names in header order. + pub members: Vec, +} + +/// One data row (an expense or a payment), its fields as in the file (after CSV unquoting). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataRow { + /// 1-based line number in the file. + pub line: u64, + pub date: String, + pub description: String, + pub category: String, + pub cost: String, + pub currency: String, + /// Member nets in header order. + pub nets: Vec, + /// 0-based index of this row among identical rows earlier in the file (SPEC §10.8 ``). + pub occurrence: usize, +} + +/// A totals row: the member nets of the whole group, per currency. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TotalRow { + pub line: u64, + pub currency: String, + /// Member nets in header order. + pub nets: Vec, +} + +/// A row that cannot be read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RowProblem { + pub line: u64, + pub reason: String, +} + +/// A whole parsed file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedFile { + pub header: Header, + pub rows: Vec, + pub totals: Vec, + pub blank_rows: usize, + pub problems: Vec, +} + +/// Why a file cannot be read at all. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileError { + /// Wrong or missing header (SPEC §10.8: fails the import, writing nothing). + Header(String), + /// Not CSV at all (bad UTF-8, broken quoting). + Csv(String), +} + +impl std::fmt::Display for FileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Header(m) | Self::Csv(m) => f.write_str(m), + } + } +} + +fn expected_headers() -> String { + format!( + "expected \"{}\" (English) or \"{}\" (Swedish), then exactly {MEMBER_COLUMNS} member columns named by full \ + Splitwise name", + ENGLISH_COLUMNS.join(","), + SWEDISH_COLUMNS.join(",") + ) +} + +/// Checks the header line. +pub fn parse_header(fields: &[String]) -> Result { + let fixed: Vec<&str> = fields.iter().take(5).map(String::as_str).collect(); + let language = if fixed == ENGLISH_COLUMNS { + Language::English + } else if fixed == SWEDISH_COLUMNS { + Language::Swedish + } else { + return Err(FileError::Header(format!( + "unrecognised header; {}", + expected_headers() + ))); + }; + let members: Vec = fields.iter().skip(5).map(|m| m.trim().to_owned()).collect(); + if members.len() != MEMBER_COLUMNS { + return Err(FileError::Header(format!( + "the header has {} member columns; {}", + members.len(), + expected_headers() + ))); + } + if members.iter().any(String::is_empty) { + return Err(FileError::Header("a member column has no name".into())); + } + if members[0] == members[1] { + return Err(FileError::Header( + "the two member columns have the same name".into(), + )); + } + Ok(Header { language, members }) +} + +/// The 1-based line number of the record csv reports at byte `offset`. csv reports the offset where it started +/// reading, before the empty lines it skips, so those are stepped over first. +fn line_at(bytes: &[u8], offset: u64) -> u64 { + let mut end = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(bytes.len()); + while end < bytes.len() && (bytes[end] == b'\n' || bytes[end] == b'\r') { + end += 1; + } + let newlines = bytes[..end].iter().filter(|b| **b == b'\n').count(); + u64::try_from(newlines) + .unwrap_or(u64::MAX) + .saturating_add(1) +} + +/// Parses a whole export. Rows that do not have the header's column count are problems, not errors, so every bad +/// row is reported at once. +pub fn parse(bytes: &[u8]) -> Result { + let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes); + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(true) + .from_reader(bytes); + let mut records = reader.records(); + let header_fields: Vec = match records.next() { + Some(Ok(record)) => record.iter().map(str::to_owned).collect(), + Some(Err(e)) => return Err(FileError::Csv(format!("cannot read the header: {e}"))), + None => { + return Err(FileError::Header(format!( + "the file is empty; {}", + expected_headers() + ))); + } + }; + let header = parse_header(&header_fields)?; + let width = 5 + header.members.len(); + let mut rows = Vec::new(); + let mut totals = Vec::new(); + let mut problems = Vec::new(); + let mut blank_rows = 0; + let mut occurrences: BTreeMap, usize> = BTreeMap::new(); + for record in records { + let record = record.map_err(|e| FileError::Csv(format!("CSV error: {e}")))?; + // csv's own line counter does not count the empty lines it skips. + let line = record.position().map_or(0, |p| line_at(bytes, p.byte())); + let fields: Vec = record.iter().map(str::to_owned).collect(); + if fields.iter().all(|f| f.trim().is_empty()) { + blank_rows += 1; + continue; + } + if fields.len() != width { + problems.push(RowProblem { + line, + reason: format!("{} fields, the header has {width}", fields.len()), + }); + continue; + } + let nets = fields[5..].to_vec(); + if TOTAL_DESCRIPTIONS.contains(&fields[1].trim()) && fields[3].trim().is_empty() { + totals.push(TotalRow { + line, + currency: fields[4].trim().to_owned(), + nets, + }); + continue; + } + let count = occurrences.entry(fields.clone()).or_insert(0); + let occurrence = *count; + *count += 1; + rows.push(DataRow { + line, + date: fields[0].clone(), + description: fields[1].clone(), + category: fields[2].clone(), + cost: fields[3].clone(), + currency: fields[4].clone(), + nets, + occurrence, + }); + } + Ok(ParsedFile { + header, + rows, + totals, + blank_rows, + problems, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fields(line: &str) -> Vec { + line.split(',').map(str::to_owned).collect() + } + + #[test] + fn headers() { + let en = parse_header(&fields("Date,Description,Category,Cost,Currency,A B,C D")).unwrap(); + assert_eq!(en.language, Language::English); + assert_eq!(en.members, ["A B", "C D"]); + let sv = + parse_header(&fields("Datum,Beskrivning,Kategori,Kostnad,Valuta,C D,A B")).unwrap(); + assert_eq!(sv.language, Language::Swedish); + assert_eq!(sv.members, ["C D", "A B"]); + for bad in [ + "Date,Description,Category,Cost", + "Date,Description,Category,Cost,Currency,A B", + "Date,Description,Category,Cost,Currency,A B,C D,E F", + "Datum,Description,Category,Cost,Currency,A B,C D", + "Date,Description,Category,Cost,Valuta,A B,C D", + "Date,Description,Category,Cost,Currency,A B,A B", + "Date,Description,Category,Cost,Currency,A B,", + ] { + assert!( + matches!(parse_header(&fields(bad)), Err(FileError::Header(_))), + "{bad}" + ); + } + } + + #[test] + fn rows_and_occurrences() { + let text = "\u{feff}Date,Description,Category,Cost,Currency,A B,C D\n\n\ + 2025-01-01,\"Milk, bread\",Groceries,10.00,SEK,5.00,-5.00\n\ + 2025-01-01,\"Milk, bread\",Groceries,10.00,SEK,5.00,-5.00\n\ + ,,,,,,\n\ + 2025-01-02,Short,Groceries,10.00,SEK\n\ + \n\ + 2025-01-03,Total balance, , ,SEK,5.00,-5.00\n"; + let parsed = parse(text.as_bytes()).unwrap(); + assert_eq!(parsed.rows.len(), 2); + assert_eq!(parsed.rows[0].description, "Milk, bread"); + assert_eq!(parsed.rows[0].occurrence, 0); + assert_eq!(parsed.rows[1].occurrence, 1); + assert_eq!(parsed.rows[0].line, 3); + assert_eq!(parsed.totals.len(), 1); + assert_eq!(parsed.totals[0].nets, ["5.00", "-5.00"]); + assert_eq!(parsed.problems.len(), 1); + assert_eq!(parsed.problems[0].line, 6); + assert!(parsed.blank_rows >= 1); + } + + #[test] + fn empty_and_broken_files() { + assert!(matches!(parse(b""), Err(FileError::Header(_)))); + assert!(matches!(parse(b"a,b\n"), Err(FileError::Header(_)))); + } +} diff --git a/server/crates/kvitt-import/src/plan.rs b/server/crates/kvitt-import/src/plan.rs new file mode 100644 index 0000000..83b3ac5 --- /dev/null +++ b/server/crates/kvitt-import/src/plan.rs @@ -0,0 +1,407 @@ +//! From parsed rows to the rows Kvitt will store (SPEC §10.8): split derivation from the member nets, payments as +//! settlements, `--fx` conversion, the import key, description truncation and the category mapping. Pure: no I/O. + +use std::collections::BTreeMap; + +use kvitt_server::auth::sha256_hex; + +use crate::amount::{Rate, parse_ore}; +use crate::categories::{CategoryTable, FALLBACK_KEY, is_payment}; +use crate::parse::{DataRow, ParsedFile, RowProblem}; + +/// The home currency. +pub const SEK: &str = "SEK"; +/// Expense description limit in code points (SPEC §9.6); longer ones are truncated, the full text goes to notes. +pub const DESCRIPTION_MAX: usize = 100; +/// Settlement note limit in code points (SPEC §9.6). +pub const NOTE_MAX: usize = 200; + +/// How an imported expense is split. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlannedSplit { + /// Mode `exact`: user id → share in öre. + Exact(BTreeMap), + /// Mode `full` on this user (both nets 0: the owner). + Full(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedExpense { + pub date: String, + pub description: String, + pub notes: String, + pub category_key: String, + pub amount_ore: i64, + pub payer_user_id: String, + pub split: PlannedSplit, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedSettlement { + pub date: String, + pub note: String, + pub from_user_id: String, + pub to_user_id: String, + pub amount_ore: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlannedBody { + Expense(PlannedExpense), + Settlement(PlannedSettlement), +} + +/// One row to create unless its key already exists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedRow { + pub line: u64, + pub import_key: String, + pub body: PlannedBody, +} + +/// What a file turns into. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Plan { + pub rows: Vec, + pub problems: Vec, + /// Unmapped category name (as written in the file, trimmed) → row count. + pub unmapped: BTreeMap, + /// Data rows whose currency is not SEK. + pub non_sek_rows: usize, + /// Non-SEK currencies without a `--fx` rate → row count (data and totals rows). Non-empty aborts the import. + pub missing_fx: BTreeMap, + /// The Splitwise totals per user id (summed over the totals rows, converted to SEK); `None` without a totals row. + pub expected_totals: Option>, + /// Whether any value was converted with `--fx`. + pub converted: bool, +} + +/// Inputs the plan needs besides the file. +pub struct PlanContext<'a> { + /// The account name as stored (trimmed), part of the import key. + pub account_name: &'a str, + /// User id per member column, in header order. + pub member_user_ids: &'a [String], + pub owner_user_id: &'a str, + pub fx: &'a BTreeMap, + pub categories: &'a CategoryTable, +} + +/// `import_key` (SPEC §10.8): lowercase hex SHA-256 of +/// `splitwise|v1||||||||`. +pub fn import_key(account_name: &str, row: &DataRow) -> String { + let mut parts: Vec<&str> = vec![ + "splitwise", + "v1", + account_name, + &row.date, + &row.description, + &row.category, + &row.cost, + &row.currency, + ]; + parts.extend(row.nets.iter().map(String::as_str)); + let occurrence = row.occurrence.to_string(); + parts.push(&occurrence); + sha256_hex(&parts.join("|")) +} + +/// The first `max` code points of `text`. +fn truncate(text: &str, max: usize) -> String { + text.chars().take(max).collect() +} + +/// A converter for one currency: identity for SEK, the `--fx` rate otherwise, `None` when no rate is given. +fn converter(currency: &str, fx: &BTreeMap) -> Option> { + if currency == SEK { + Some(None) + } else { + fx.get(currency).map(|rate| Some(*rate)) + } +} + +fn convert(value: i64, rate: Option) -> Result { + match rate { + None => Ok(value), + Some(rate) => rate.convert(value), + } +} + +/// Builds the plan for a parsed file. +pub fn build(file: &ParsedFile, ctx: &PlanContext<'_>) -> Plan { + let mut plan = Plan { + problems: file.problems.clone(), + ..Plan::default() + }; + for row in &file.rows { + let currency = row.currency.trim(); + if currency != SEK { + plan.non_sek_rows += 1; + } + let Some(rate) = converter(currency, ctx.fx) else { + *plan.missing_fx.entry(currency.to_owned()).or_insert(0) += 1; + continue; + }; + if rate.is_some() { + plan.converted = true; + } + match plan_row(row, rate, ctx, &mut plan.unmapped) { + Ok(body) => plan.rows.push(PlannedRow { + line: row.line, + import_key: import_key(ctx.account_name, row), + body, + }), + Err(reason) => plan.problems.push(RowProblem { + line: row.line, + reason, + }), + } + } + plan.expected_totals = totals(file, ctx, &mut plan); + plan.problems.sort_by_key(|p| p.line); + plan +} + +/// Sums the totals rows per user (converted to SEK). +fn totals( + file: &ParsedFile, + ctx: &PlanContext<'_>, + plan: &mut Plan, +) -> Option> { + if file.totals.is_empty() { + return None; + } + let mut sums: BTreeMap = + ctx.member_user_ids.iter().map(|u| (u.clone(), 0)).collect(); + let mut seen = std::collections::BTreeSet::new(); + for total in &file.totals { + if !seen.insert(total.currency.clone()) { + plan.problems.push(RowProblem { + line: total.line, + reason: format!("a second totals row for {}", total.currency), + }); + continue; + } + let Some(rate) = converter(&total.currency, ctx.fx) else { + *plan.missing_fx.entry(total.currency.clone()).or_insert(0) += 1; + continue; + }; + for (user, net) in ctx.member_user_ids.iter().zip(&total.nets) { + match parse_ore(net).and_then(|v| convert(v, rate)) { + Ok(value) => { + if let Some(sum) = sums.get_mut(user) { + *sum += value; + } + } + Err(reason) => plan.problems.push(RowProblem { + line: total.line, + reason: format!("totals row: {reason}"), + }), + } + } + } + Some(sums) +} + +fn plan_row( + row: &DataRow, + rate: Option, + ctx: &PlanContext<'_>, + unmapped: &mut BTreeMap, +) -> Result { + let cost = parse_ore(&row.cost).map_err(|e| format!("cost: {e}"))?; + let nets = row + .nets + .iter() + .map(|n| parse_ore(n).map_err(|e| format!("member net: {e}"))) + .collect::, String>>()?; + let [net_a, net_b] = nets[..] else { + return Err("expected two member nets".into()); + }; + if net_a.checked_add(net_b) != Some(0) { + return Err("the member nets do not sum to 0".into()); + } + let (user_a, user_b) = match ctx.member_user_ids { + [a, b] => (a.clone(), b.clone()), + _ => return Err("expected two members".into()), + }; + // The member with positive net and that net, if any. + let positive = if net_a > 0 { + Some((user_a, user_b, net_a)) + } else if net_b > 0 { + Some((user_b, user_a, net_b)) + } else { + None + }; + let description = row.description.trim(); + + if is_payment(&row.category) { + let Some((from, to, net)) = positive else { + return Err("a payment with both nets 0".into()); + }; + return Ok(PlannedBody::Settlement(PlannedSettlement { + date: row.date.trim().to_owned(), + note: truncate(description, NOTE_MAX), + from_user_id: from, + to_user_id: to, + amount_ore: convert(net, rate)?, + })); + } + + let category = row.category.trim(); + let category_key = match ctx.categories.lookup(category) { + Some(key) => key.to_owned(), + None => { + *unmapped.entry(category.to_owned()).or_insert(0) += 1; + FALLBACK_KEY.to_owned() + } + }; + if description.is_empty() { + return Err("empty description".into()); + } + let (short, notes) = if description.chars().count() > DESCRIPTION_MAX { + ( + truncate(description, DESCRIPTION_MAX), + description.to_owned(), + ) + } else { + (description.to_owned(), String::new()) + }; + let amount = convert(cost, rate)?; + let (payer, split) = match positive { + Some((payer, other, net)) => { + let share_other = convert(net, rate)?; + let share_payer = amount - share_other; + if share_payer < 0 { + return Err( + "the payer's share would be negative (net larger than the cost)".into(), + ); + } + let shares = BTreeMap::from([(payer.clone(), share_payer), (other, share_other)]); + (payer, PlannedSplit::Exact(shares)) + } + None => ( + ctx.owner_user_id.to_owned(), + PlannedSplit::Full(ctx.owner_user_id.to_owned()), + ), + }; + Ok(PlannedBody::Expense(PlannedExpense { + date: row.date.trim().to_owned(), + description: short, + notes, + category_key, + amount_ore: amount, + payer_user_id: payer, + split, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(category: &str, cost: &str, nets: [&str; 2]) -> DataRow { + DataRow { + line: 2, + date: "2025-03-01".into(), + description: "Thing".into(), + category: category.into(), + cost: cost.into(), + currency: "SEK".into(), + nets: nets.iter().map(|n| (*n).to_owned()).collect(), + occurrence: 0, + } + } + + fn plan_one(r: DataRow, fx: &BTreeMap) -> Result { + let table = CategoryTable::committed().unwrap(); + let members = ["ua".to_owned(), "ub".to_owned()]; + let ctx = PlanContext { + account_name: "Acc", + member_user_ids: &members, + owner_user_id: "ub", + fx, + categories: &table, + }; + let rate = converter(r.currency.trim(), fx).unwrap(); + plan_row(&r, rate, &ctx, &mut BTreeMap::new()) + } + + #[test] + fn exact_split_from_nets() { + let none = BTreeMap::new(); + let PlannedBody::Expense(e) = + plan_one(row("Groceries", "100.00", ["-30.00", "30.00"]), &none).unwrap() + else { + panic!() + }; + assert_eq!(e.payer_user_id, "ub"); + assert_eq!( + e.split, + PlannedSplit::Exact(BTreeMap::from([("ub".into(), 7000), ("ua".into(), 3000)])) + ); + assert_eq!(e.category_key, "groceries"); + } + + #[test] + fn zero_nets_are_full_on_owner() { + let PlannedBody::Expense(e) = + plan_one(row("Rent", "10.00", ["0.00", "0.00"]), &BTreeMap::new()).unwrap() + else { + panic!() + }; + assert_eq!(e.payer_user_id, "ub"); + assert_eq!(e.split, PlannedSplit::Full("ub".into())); + } + + #[test] + fn invalid_rows() { + let none = BTreeMap::new(); + assert!(plan_one(row("Rent", "10.00", ["5.00", "-4.00"]), &none).is_err()); + assert!(plan_one(row("Rent", "10.00", ["15.00", "-15.00"]), &none).is_err()); + assert!(plan_one(row("Rent", "1.234", ["0.00", "0.00"]), &none).is_err()); + assert!(plan_one(row("Payment", "10.00", ["0.00", "0.00"]), &none).is_err()); + } + + #[test] + fn payment_from_positive_member() { + let PlannedBody::Settlement(s) = plan_one( + row("Betalning", "25.00", ["25.00", "-25.00"]), + &BTreeMap::new(), + ) + .unwrap() else { + panic!() + }; + assert_eq!( + (s.from_user_id.as_str(), s.to_user_id.as_str(), s.amount_ore), + ("ua", "ub", 2500) + ); + } + + #[test] + fn fx_recomputes_payer_share() { + let fx = BTreeMap::from([("EUR".to_owned(), Rate::parse("0.5").unwrap())]); + let mut r = row("Groceries", "0.03", ["0.01", "-0.01"]); + r.currency = "EUR".into(); + let PlannedBody::Expense(e) = plan_one(r, &fx).unwrap() else { + panic!() + }; + // C' = rhu(1.5) = 2, other' = rhu(0.5) = 1, payer = 2 − 1. + assert_eq!(e.amount_ore, 2); + assert_eq!( + e.split, + PlannedSplit::Exact(BTreeMap::from([("ua".into(), 1), ("ub".into(), 1)])) + ); + } + + #[test] + fn import_key_formula() { + let mut r = row("Groceries", "10.00", ["5.00", "-5.00"]); + r.occurrence = 1; + assert_eq!( + import_key("Acc", &r), + sha256_hex("splitwise|v1|Acc|2025-03-01|Thing|Groceries|10.00|SEK|5.00|-5.00|1") + ); + assert_eq!(import_key("Acc", &r).len(), 64); + } +} diff --git a/server/crates/kvitt-import/tests/fixtures/bad_header.csv b/server/crates/kvitt-import/tests/fixtures/bad_header.csv new file mode 100644 index 0000000..fd775d8 --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/bad_header.csv @@ -0,0 +1,2 @@ +Date,Description,Category,Amount,Currency,Alex Example,Robin Sample +2025-05-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 diff --git a/server/crates/kvitt-import/tests/fixtures/english.csv b/server/crates/kvitt-import/tests/fixtures/english.csv new file mode 100644 index 0000000..dc9ab4c --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/english.csv @@ -0,0 +1,15 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample + +2025-01-05,Weekly shop,Groceries,300.00,SEK,150.00,-150.00 +2025-01-06,"Paint, brushes and tape",Household supplies,120.50,SEK,-60.25,60.25 +2025-01-07,Cinema,Movies,200.00,SEK,0.00,0.00 +2025-01-08,Bus card,Bus/train,100.00,SEK,-100.00,100.00 +2025-01-09,Mystery box,Widgets,50.00,SEK,25.00,-25.00 +2025-01-10,Coffee,Dining out,40.00,SEK,20.00,-20.00 +2025-01-10,Coffee,Dining out,40.00,SEK,20.00,-20.00 +2025-01-11,An extremely long synthetic description written only to exercise the truncation rule of the importer at one hundred code points,General,10.00,SEK,5.00,-5.00 +2025-01-12,Robin paid Alex,Payment,100.00,SEK,-100.00,100.00 +2025-01-13,"Rent, January",Rent,1000.00,SEK,500.00,-500.00 + +2025-01-14,Total balance, , ,SEK,459.75,-459.75 + diff --git a/server/crates/kvitt-import/tests/fixtures/invalid_date.csv b/server/crates/kvitt-import/tests/fixtures/invalid_date.csv new file mode 100644 index 0000000..b19e9fb --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/invalid_date.csv @@ -0,0 +1,7 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample + +2025-04-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 +2025-13-01,Bad date,Groceries,10.00,SEK,5.00,-5.00 + +2025-04-03,Total balance, , ,SEK,10.00,-10.00 + diff --git a/server/crates/kvitt-import/tests/fixtures/invalid_nets.csv b/server/crates/kvitt-import/tests/fixtures/invalid_nets.csv new file mode 100644 index 0000000..3210b64 --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/invalid_nets.csv @@ -0,0 +1,7 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample + +2025-04-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 +2025-04-02,Broken row,Groceries,10.00,SEK,5.00,-4.00 + +2025-04-03,Total balance, , ,SEK,10.00,-9.00 + diff --git a/server/crates/kvitt-import/tests/fixtures/mismatch.csv b/server/crates/kvitt-import/tests/fixtures/mismatch.csv new file mode 100644 index 0000000..25313ef --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/mismatch.csv @@ -0,0 +1,6 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample + +2025-05-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 + +2025-05-02,Total balance, , ,SEK,7.00,-7.00 + diff --git a/server/crates/kvitt-import/tests/fixtures/no_totals.csv b/server/crates/kvitt-import/tests/fixtures/no_totals.csv new file mode 100644 index 0000000..26b961f --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/no_totals.csv @@ -0,0 +1,2 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample +2025-05-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 diff --git a/server/crates/kvitt-import/tests/fixtures/non_sek.csv b/server/crates/kvitt-import/tests/fixtures/non_sek.csv new file mode 100644 index 0000000..0728393 --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/non_sek.csv @@ -0,0 +1,8 @@ +Date,Description,Category,Cost,Currency,Alex Example,Robin Sample + +2025-03-01,Market,Groceries,100.00,SEK,50.00,-50.00 +2025-03-02,Hotel abroad,Hotel,10.00,EUR,5.00,-5.00 + +2025-03-03,Total balance, , ,SEK,50.00,-50.00 +2025-03-03,Total balance, , ,EUR,5.00,-5.00 + diff --git a/server/crates/kvitt-import/tests/fixtures/swedish.csv b/server/crates/kvitt-import/tests/fixtures/swedish.csv new file mode 100644 index 0000000..8fb8c50 --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/swedish.csv @@ -0,0 +1,10 @@ +Datum,Beskrivning,Kategori,Kostnad,Valuta,Robin Sample,Alex Example + +2025-02-01,Matkasse,Livsmedel,200.00,SEK,-100.00,100.00 +2025-02-02,"Lampa, golv",Möbler,80.00,SEK,40.00,-40.00 +2025-02-03,Okänd sak,Påhittad kategori,10.00,SEK,5.00,-5.00 +2025-02-04,Betalning till Alex,Betalning,55.00,SEK,55.00,-55.00 +2025-02-05,Biljett,Buss/tåg,30.00,SEK,-15.00,15.00 + +2025-02-06,Totalsumma, , ,SEK,-15.00,15.00 + diff --git a/server/crates/kvitt-import/tests/fixtures/wrong_member.csv b/server/crates/kvitt-import/tests/fixtures/wrong_member.csv new file mode 100644 index 0000000..1b1bb34 --- /dev/null +++ b/server/crates/kvitt-import/tests/fixtures/wrong_member.csv @@ -0,0 +1,4 @@ +Date,Description,Category,Cost,Currency,Alex Example,Sam Other +2025-05-01,Fine row,Groceries,10.00,SEK,5.00,-5.00 + +2025-05-02,Total balance, , ,SEK,5.00,-5.00 diff --git a/server/crates/kvitt-import/tests/import.rs b/server/crates/kvitt-import/tests/import.rs new file mode 100644 index 0000000..73feced --- /dev/null +++ b/server/crates/kvitt-import/tests/import.rs @@ -0,0 +1,804 @@ +//! Integration tests of `kvitt-import` (SPEC §10.8, FR-17 AC-17.1 to AC-17.6) over synthetic fixtures only, against +//! a temporary database built with the server's migrations. Member names, descriptions and amounts are made up. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use chrono::{DateTime, Utc}; +use kvitt_import::amount::Rate; +use kvitt_import::categories::CategoryTable; +use kvitt_import::{ImportError, Options, Report, UserArg}; +use kvitt_server::audit::SqlAuditWriter; +use kvitt_server::auth::sha256_hex; +use kvitt_server::db; +use kvitt_server::time::parse_ts; +use sqlx::SqlitePool; +use uuid::Uuid; + +const OWNER: &str = "0192c000-0000-7000-8000-000000000001"; +const PARTNER: &str = "0192c000-0000-7000-8000-000000000002"; +const OWNER_NAME: &str = "Alex Example"; +const PARTNER_NAME: &str = "Robin Sample"; +const NOW: &str = "2026-10-17T12:00:00.000Z"; + +fn now() -> DateTime { + parse_ts(NOW).unwrap() +} + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name) +} + +/// A migrated database with the two synthetic users, removed on drop. +struct TestDb { + dir: PathBuf, + path: PathBuf, +} + +impl Drop for TestDb { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +impl TestDb { + async fn new() -> Self { + let dir = std::env::temp_dir().join(format!("kvitt-import-{}", Uuid::now_v7())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("kvitt.sqlite"); + let pool = db::connect(&path).await.unwrap(); + db::migrate(&pool).await.unwrap(); + for (id, role, name, version) in [ + (OWNER, "admin", "Alex", 11), + (PARTNER, "member", "Robin", 12), + ] { + sqlx::query( + "INSERT INTO users (id, role, display_name, picture_version, created_at, updated_at, server_version) \ + VALUES (?, ?, ?, NULL, '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z', ?)", + ) + .bind(id) + .bind(role) + .bind(name) + .bind(version) + .execute(&pool) + .await + .unwrap(); + } + sqlx::query("UPDATE sync_sequence SET value = 12") + .execute(&pool) + .await + .unwrap(); + pool.close().await; + Self { dir, path } + } + + async fn pool(&self) -> SqlitePool { + db::connect(&self.path).await.unwrap() + } + + async fn scalar(&self, sql: &str) -> i64 { + let pool = self.pool().await; + let value: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(sql.to_owned())) + .fetch_one(&pool) + .await + .unwrap(); + pool.close().await; + value + } + + /// Row counts of every table the importer may write, plus the sequence value. + async fn snapshot(&self) -> Vec { + let mut out = Vec::new(); + for table in [ + "accounts", + "account_default_shares", + "expenses", + "expense_splits", + "settlements", + "settlement_allocations", + "audit_log", + "sync_mutations", + ] { + out.push(self.scalar(&format!("SELECT COUNT(*) FROM {table}")).await); + } + out.push(self.scalar("SELECT value FROM sync_sequence").await); + out + } + + fn options(&self, file: &str, account: &str) -> Options { + Options { + db: self.path.clone(), + file: fixture(file), + account: account.to_owned(), + users: vec![ + UserArg { + splitwise_name: OWNER_NAME.into(), + kvitt_user: OWNER.into(), + }, + UserArg { + splitwise_name: PARTNER_NAME.into(), + kvitt_user: PARTNER.into(), + }, + ], + fx: BTreeMap::new(), + dry_run: false, + } + } + + async fn import(&self, options: &Options) -> Result { + kvitt_import::run(options, now(), &SqlAuditWriter).await + } +} + +/// `(payer, split_mode, amount, owner share, partner share, owner bp, description, notes, category key, key)` of +/// every expense, by expense date and id. +type ExpenseLine = ( + String, + String, + i64, + i64, + i64, + Option, + String, + String, + String, + String, +); + +/// `(entity_type, action, source, actor_user_id, actor_device_id, client_at, server_at, changes)`. +type AuditLine = ( + String, + String, + String, + String, + Option, + String, + String, + String, +); + +async fn expenses(db: &TestDb) -> Vec { + let pool = db.pool().await; + let rows: Vec = sqlx::query_as( + "SELECT e.payer_user_id, e.split_mode, e.amount_ore, so.share_ore, sp.share_ore, so.share_bp, \ + e.description, e.notes, c.default_key, e.import_key FROM expenses e \ + JOIN expense_splits so ON so.expense_id = e.id AND so.user_id = ?1 \ + JOIN expense_splits sp ON sp.expense_id = e.id AND sp.user_id = ?2 \ + JOIN categories c ON c.id = e.category_id ORDER BY e.expense_date, e.id", + ) + .bind(OWNER) + .bind(PARTNER) + .fetch_all(&pool) + .await + .unwrap(); + pool.close().await; + rows +} + +#[tokio::test] +async fn english_export_imports_expenses_settlements_and_audit() { + let db = TestDb::new().await; + let report = db + .import(&db.options("english.csv", "Shared")) + .await + .unwrap(); + assert!(report.committed); + assert!(report.account_created); + assert_eq!(report.rows_read, 10); + assert_eq!(report.totals_rows, 1); + assert_eq!(report.expenses_created, 9); + assert_eq!(report.settlements_created, 1); + assert_eq!(report.skipped_duplicates, 0); + assert_eq!(report.non_sek_rows, 0); + assert_eq!( + report.unmapped_categories, + BTreeMap::from([("Widgets".to_owned(), 1)]) + ); + assert_eq!(report.unmapped_rows(), 1); + assert!(report.reconciliation.matched); + assert_eq!(report.exit_code(), 0); + let summary = report.to_string(); + assert!(summary.contains("Widgets: 1 rows"), "{summary}"); + assert!( + summary.contains("WARNING: 1 rows with an unmapped category"), + "{summary}" + ); + assert!(summary.contains("result: match"), "{summary}"); + + // The account: created by the owner with the default 50/50 ratio. + assert_eq!( + db.scalar("SELECT COUNT(*) FROM accounts WHERE name = 'Shared' AND created_by_user_id = '0192c000-0000-7000-8000-000000000001'") + .await, + 1 + ); + assert_eq!( + db.scalar("SELECT COUNT(*) FROM account_default_shares WHERE share_bp = 5000") + .await, + 2 + ); + + let rows = expenses(&db).await; + assert_eq!(rows.len(), 9); + // Owner paid, exact split from the nets. + assert_eq!( + ( + &rows[0].0[..], + &rows[0].1[..], + rows[0].2, + rows[0].3, + rows[0].4, + rows[0].5 + ), + (OWNER, "exact", 30000, 15000, 15000, None) + ); + assert_eq!(rows[0].8, "groceries"); + // Quoted description with a comma; partner paid. + assert_eq!(rows[1].6, "Paint, brushes and tape"); + assert_eq!( + (&rows[1].0[..], rows[1].3, rows[1].4), + (PARTNER, 6025, 6025) + ); + assert_eq!(rows[1].8, "home"); + // Both nets 0: full on the owner. + assert_eq!( + ( + &rows[2].0[..], + &rows[2].1[..], + rows[2].3, + rows[2].4, + rows[2].5 + ), + (OWNER, "full", 20000, 0, Some(10_000)) + ); + assert_eq!(rows[2].8, "entertainment"); + // Partner paid all of it for the owner: payer share 0. + assert_eq!((&rows[3].0[..], rows[3].3, rows[3].4), (PARTNER, 10000, 0)); + assert_eq!(rows[3].8, "transport"); + // Unknown category to other. + assert_eq!(rows[4].8, "other"); + // Two identical lines both imported, with different keys. + assert_eq!(rows[5].6, "Coffee"); + assert_eq!(rows[6].6, "Coffee"); + assert_ne!(rows[5].9, rows[6].9); + // Truncated description: 100 code points, the full text in notes. + assert_eq!(rows[7].6.chars().count(), 100); + assert!(rows[7].7.starts_with(&rows[7].6)); + assert_eq!(rows[7].7.chars().count(), 127); + assert_eq!(rows[8].6, "Rent, January"); + assert!( + rows.iter() + .all(|r| r.7.is_empty() || r.6.chars().count() == 100) + ); + + // The import key formula of SPEC §10.8, for the first row and for the second of the two identical lines. + let key = |line: &str, n: usize| { + let fields: Vec<&str> = line.split(',').collect(); + sha256_hex(&format!("splitwise|v1|Shared|{}|{n}", fields.join("|"))) + }; + assert_eq!( + rows[0].9, + key( + "2025-01-05,Weekly shop,Groceries,300.00,SEK,150.00,-150.00", + 0 + ) + ); + assert_eq!( + rows[6].9, + key("2025-01-10,Coffee,Dining out,40.00,SEK,20.00,-20.00", 1) + ); + + // The payment: a settlement from the positive-net member, one allocation to the account. + let pool = db.pool().await; + let settlement: (String, String, i64, String, String, i64, i64) = sqlx::query_as( + "SELECT s.from_user_id, s.to_user_id, s.amount_ore, s.note, s.settlement_date, \ + (SELECT COUNT(*) FROM settlement_allocations a WHERE a.settlement_id = s.id), \ + (SELECT SUM(a.amount_ore) FROM settlement_allocations a JOIN accounts ac ON ac.id = a.account_id \ + WHERE a.settlement_id = s.id AND ac.name = 'Shared') FROM settlements s", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + settlement, + ( + PARTNER.to_owned(), + OWNER.to_owned(), + 10000, + "Robin paid Alex".to_owned(), + "2025-01-12".to_owned(), + 1, + 10000 + ) + ); + + // Created rows: owner as creator and updater, no device, import time, server_received_at set. + for table in ["expenses", "settlements"] { + let odd: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM {table} WHERE created_by_user_id <> ?1 OR updated_by_user_id <> ?1 \ + OR updated_by_device_id IS NOT NULL OR created_at <> ?2 OR updated_at <> ?2 OR deleted_at IS NOT NULL \ + OR import_key IS NULL OR length(import_key) <> 64" + ))) + .bind(OWNER) + .bind(NOW) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(odd, 0, "{table}"); + } + + // One audit create per row, source import, actor the owner, no device, both times the import time, with + // the create diff of SPEC §14.4.7. + let audits: Vec = sqlx::query_as( + "SELECT entity_type, action, source, actor_user_id, actor_device_id, client_at, server_at, changes \ + FROM audit_log ORDER BY server_version", + ) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(audits.len(), 10); + assert_eq!(audits.iter().filter(|a| a.0 == "expense").count(), 9); + assert_eq!(audits.iter().filter(|a| a.0 == "settlement").count(), 1); + for a in &audits { + assert_eq!((&a.1[..], &a.2[..], &a.3[..]), ("create", "import", OWNER)); + assert_eq!(a.4, None); + assert_eq!((&a.5[..], &a.6[..]), (NOW, NOW)); + let changes: serde_json::Value = serde_json::from_str(&a.7).unwrap(); + assert!( + changes + .as_array() + .unwrap() + .iter() + .all(|c| c["old"].is_null()) + ); + } + let audited: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_log a WHERE a.entity_id IN (SELECT id FROM expenses UNION SELECT id FROM settlements)", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(audited, 10); + + // New server_versions from the global sequence: account + 9 expenses + 1 settlement + 10 audit entries. + let max_version: i64 = sqlx::query_scalar("SELECT value FROM sync_sequence") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(max_version, 12 + 1 + 9 + 1 + 10); + // No pushes: the importer has no push queue and touches no sync state. + let mutations: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM sync_mutations") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(mutations, 0); + pool.close().await; +} + +#[tokio::test] +async fn swedish_export_with_reversed_member_columns() { + let db = TestDb::new().await; + let report = db + .import(&db.options("swedish.csv", "Vardagen")) + .await + .unwrap(); + assert!(report.committed); + assert_eq!(report.language, kvitt_import::parse::Language::Swedish); + assert_eq!(report.rows_read, 5); + assert_eq!(report.expenses_created, 4); + assert_eq!(report.settlements_created, 1); + assert_eq!( + report.unmapped_categories, + BTreeMap::from([("Påhittad kategori".to_owned(), 1)]) + ); + assert!(report.reconciliation.matched); + // Members are matched by name: the second column is the owner. + assert_eq!( + report.reconciliation.members[0].splitwise_name, + PARTNER_NAME + ); + assert_eq!(report.reconciliation.members[0].kvitt_ore, -1500); + assert_eq!(report.reconciliation.members[1].kvitt_ore, 1500); + + let rows = expenses(&db).await; + assert_eq!( + (&rows[0].0[..], rows[0].3, rows[0].4), + (OWNER, 10000, 10000) + ); + assert_eq!(rows[0].8, "groceries"); + assert_eq!(rows[1].6, "Lampa, golv"); + assert_eq!( + (&rows[1].0[..], rows[1].3, rows[1].4), + (PARTNER, 4000, 4000) + ); + assert_eq!(rows[1].8, "home"); + assert_eq!(rows[2].8, "other"); + assert_eq!(rows[3].8, "transport"); + let (from, to, amount): (String, String, i64) = + sqlx::query_as("SELECT from_user_id, to_user_id, amount_ore FROM settlements") + .fetch_one(&db.pool().await) + .await + .unwrap(); + assert_eq!((&from[..], &to[..], amount), (PARTNER, OWNER, 5500)); +} + +#[tokio::test] +async fn rerun_creates_nothing_and_existing_account_is_reused() { + let db = TestDb::new().await; + let first = db + .import(&db.options("english.csv", "Shared")) + .await + .unwrap(); + assert!(first.committed); + let before = db.snapshot().await; + // Same file, account name in another case: the existing account, every row skipped. + let second = db + .import(&db.options("english.csv", "SHARED")) + .await + .unwrap(); + assert!(second.committed); + assert_eq!(second.account_name, "Shared"); + assert!(!second.account_created); + assert_eq!(second.account_id, first.account_id); + assert_eq!(second.expenses_created, 0); + assert_eq!(second.settlements_created, 0); + assert_eq!(second.skipped_duplicates, 10); + assert!(second.reconciliation.matched); + assert_eq!(db.snapshot().await, before); + + // A second file goes into a second account; the account name is part of the key. + let other = db + .import(&db.options("swedish.csv", "Other group")) + .await + .unwrap(); + assert!(other.account_created); + assert_eq!(other.expenses_created, 4); +} + +#[tokio::test] +async fn existing_account_matched_case_insensitively() { + let db = TestDb::new().await; + let first = db + .import(&db.options("swedish.csv", "Vardag")) + .await + .unwrap(); + let second = db + .import(&db.options("english.csv", " vardag ")) + .await + .unwrap(); + assert!(!second.account_created); + assert_eq!(second.account_id, first.account_id); + // The balance now holds both files, so it no longer equals the second file's totals: rolled back. + assert!(!second.reconciliation.matched); + assert!(!second.committed); + assert_eq!(second.exit_code(), kvitt_import::EXIT_MISMATCH); + assert_eq!(db.scalar("SELECT COUNT(*) FROM expenses").await, 4); +} + +#[tokio::test] +async fn dry_run_writes_nothing_and_reports_the_same_counts() { + let db = TestDb::new().await; + let before = db.snapshot().await; + let mut options = db.options("english.csv", "Shared"); + options.dry_run = true; + let report = db.import(&options).await.unwrap(); + assert!(report.dry_run); + assert!(!report.committed); + assert_eq!(report.expenses_created, 9); + assert_eq!(report.settlements_created, 1); + assert_eq!(report.unmapped_rows(), 1); + assert!(report.reconciliation.matched); + assert_eq!(report.exit_code(), 0); + assert!(report.to_string().contains("dry run, nothing written")); + assert_eq!(db.snapshot().await, before); + + // After a real import, a dry run reports every row as a duplicate. + options.dry_run = false; + db.import(&options).await.unwrap(); + options.dry_run = true; + let again = db.import(&options).await.unwrap(); + assert_eq!(again.skipped_duplicates, 10); + assert_eq!(again.expenses_created, 0); +} + +#[tokio::test] +async fn invalid_row_aborts_with_nothing_written() { + let db = TestDb::new().await; + let before = db.snapshot().await; + match db.import(&db.options("invalid_nets.csv", "Shared")).await { + Err(ImportError::InvalidRows(problems)) => { + assert_eq!(problems.len(), 1); + assert_eq!(problems[0].line, 4); + assert!( + problems[0].reason.contains("sum to 0"), + "{}", + problems[0].reason + ); + } + other => panic!("{other:?}"), + } + // A row the server's checks reject (an impossible date) also aborts the whole file. + match db.import(&db.options("invalid_date.csv", "Shared")).await { + Err(e @ ImportError::InvalidRows(_)) => { + assert_eq!(e.exit_code(), kvitt_import::EXIT_INVALID); + let ImportError::InvalidRows(problems) = e else { + unreachable!() + }; + assert_eq!(problems.len(), 1); + assert_eq!(problems[0].line, 4); + assert!( + problems[0].reason.starts_with("validation_failed"), + "{}", + problems[0].reason + ); + } + other => panic!("{other:?}"), + } + assert_eq!(db.snapshot().await, before); +} + +#[tokio::test] +async fn header_failures_write_nothing() { + let db = TestDb::new().await; + let before = db.snapshot().await; + for file in ["bad_header.csv", "wrong_member.csv"] { + match db.import(&db.options(file, "Shared")).await { + Err(e @ ImportError::Header(_)) => { + assert_eq!(e.exit_code(), kvitt_import::EXIT_INVALID) + } + other => panic!("{file}: {other:?}"), + } + } + assert_eq!(db.snapshot().await, before); +} + +#[tokio::test] +async fn non_sek_aborts_without_fx_and_converts_with_it() { + let db = TestDb::new().await; + let before = db.snapshot().await; + let mut options = db.options("non_sek.csv", "Trip"); + match db.import(&options).await { + Err(ImportError::MissingFx(currencies)) => { + // One data row and one totals row in EUR. + assert_eq!(currencies, BTreeMap::from([("EUR".to_owned(), 2)])); + } + other => panic!("{other:?}"), + } + assert_eq!(db.snapshot().await, before); + + options + .fx + .insert("EUR".into(), Rate::parse("11.5").unwrap()); + let report = db.import(&options).await.unwrap(); + assert!(report.committed); + assert_eq!(report.non_sek_rows, 1); + assert_eq!(report.expenses_created, 2); + assert!(report.reconciliation.matched); + let rows = expenses(&db).await; + // 10.00 EUR × 11.5 = 115.00 SEK; the partner's share 5.00 EUR = 57.50 SEK; the payer's share is the rest. + assert_eq!((rows[1].2, rows[1].3, rows[1].4), (11500, 5750, 5750)); +} + +#[tokio::test] +async fn reconciliation_mismatch_or_missing_totals_rolls_back() { + let db = TestDb::new().await; + let before = db.snapshot().await; + let report = db + .import(&db.options("mismatch.csv", "Shared")) + .await + .unwrap(); + assert!(!report.reconciliation.matched); + assert!(!report.committed); + assert_eq!(report.exit_code(), kvitt_import::EXIT_MISMATCH); + let summary = report.to_string(); + assert!(summary.contains("splitwise 7.00, kvitt 5.00"), "{summary}"); + assert!(summary.contains("result: MISMATCH"), "{summary}"); + assert_eq!(db.snapshot().await, before); + + let report = db + .import(&db.options("no_totals.csv", "Shared")) + .await + .unwrap(); + assert!(!report.reconciliation.has_totals); + assert!(!report.committed); + assert_eq!(report.exit_code(), kvitt_import::EXIT_MISMATCH); + assert_eq!(db.snapshot().await, before); +} + +#[tokio::test] +async fn users_resolve_by_display_name_and_must_be_the_household() { + let db = TestDb::new().await; + let mut options = db.options("english.csv", "Shared"); + options.users[0].kvitt_user = "Alex".into(); + options.users[1].kvitt_user = "Robin".into(); + let report = db.import(&options).await.unwrap(); + assert!(report.committed); + assert_eq!(report.reconciliation.members[0].user_id, OWNER); + + let mut both_same = db.options("english.csv", "Other"); + both_same.users[1].kvitt_user = OWNER.into(); + assert!(matches!( + db.import(&both_same).await, + Err(ImportError::Usage(_)) + )); + let mut unknown = db.options("english.csv", "Other"); + unknown.users[1].kvitt_user = "Nobody".into(); + assert!(matches!( + db.import(&unknown).await, + Err(ImportError::Usage(_)) + )); + let mut one = db.options("english.csv", "Other"); + one.users.pop(); + assert!(matches!(db.import(&one).await, Err(ImportError::Usage(_)))); +} + +#[tokio::test] +async fn missing_database_is_not_created() { + let db = TestDb::new().await; + let mut options = db.options("english.csv", "Shared"); + options.db = db.dir.join("absent.sqlite"); + assert!(matches!(db.import(&options).await, Err(ImportError::Io(_)))); + assert!(!options.db.exists()); +} + +/// Every category name of the SPEC §10.8 table is in the committed data file exactly once, with its key, and the +/// file holds nothing else. +#[test] +fn category_table_matches_the_spec() { + let spec = + std::fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../SPEC.md")) + .unwrap(); + let section = spec.split("### 10.8 Splitwise import").nth(1).unwrap(); + let section = section.split("\n### ").next().unwrap(); + let mut expected: Vec<(String, String, String)> = Vec::new(); + for line in section.lines() { + let cells: Vec<&str> = line + .trim() + .trim_matches('|') + .split('|') + .map(str::trim) + .collect(); + if cells.len() != 3 || !cells[2].starts_with('`') || cells[0].starts_with("any other") { + continue; + } + let key = cells[2].trim_matches('`').to_owned(); + for (column, language) in [(cells[0], "en"), (cells[1], "sv")] { + for name in column.split(", ") { + expected.push((name.to_owned(), language.to_owned(), key.clone())); + } + } + } + assert!( + expected.len() > 60, + "parsed {} names from the spec", + expected.len() + ); + let table = CategoryTable::committed().unwrap(); + let mut actual: Vec<(String, String, String)> = table + .entries + .iter() + .map(|e| { + ( + e.splitwise_name.clone(), + e.language.clone(), + e.default_key.clone(), + ) + }) + .collect(); + expected.sort(); + actual.sort(); + assert_eq!(actual, expected); +} + +fn binary() -> Command { + Command::new(env!("CARGO_BIN_EXE_kvitt-import")) +} + +fn cli_args(db: &TestDb, file: &str, account: &str) -> Vec { + vec![ + "--db".into(), + db.path.to_string_lossy().into_owned(), + "--file".into(), + fixture(file).to_string_lossy().into_owned(), + "--account".into(), + account.into(), + "--user".into(), + format!("{OWNER_NAME}={OWNER}"), + "--user".into(), + format!("{PARTNER_NAME}=Robin"), + ] +} + +#[test] +fn cli_help_and_usage_errors() { + let help = binary().arg("--help").output().unwrap(); + assert!(help.status.success()); + let text = String::from_utf8_lossy(&help.stdout); + for option in [ + "--db", + "--file", + "--account", + "--user", + "--fx", + "--dry-run", + "Exit status", + ] { + assert!(text.contains(option), "{option}: {text}"); + } + let missing = binary().args(["--db", "x"]).output().unwrap(); + assert_eq!(missing.status.code(), Some(2)); + let bad_fx = binary() + .args([ + "--db", + "x", + "--file", + "y", + "--account", + "A", + "--user", + "a=b", + "--user", + "c=d", + "--fx", + "EUR", + ]) + .output() + .unwrap(); + assert_eq!(bad_fx.status.code(), Some(2)); +} + +#[tokio::test] +async fn cli_end_to_end_exit_codes() { + let db = TestDb::new().await; + let mut dry = cli_args(&db, "swedish.csv", "Vardag"); + dry.push("--dry-run".into()); + let out = binary().args(&dry).output().unwrap(); + assert_eq!( + out.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert!(String::from_utf8_lossy(&out.stdout).contains("dry run")); + assert_eq!(db.scalar("SELECT COUNT(*) FROM expenses").await, 0); + + let out = binary() + .args(cli_args(&db, "swedish.csv", "Vardag")) + .output() + .unwrap(); + assert_eq!( + out.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("expenses created: 4"), "{stdout}"); + assert!(stdout.contains("result: match"), "{stdout}"); + assert_eq!(db.scalar("SELECT COUNT(*) FROM expenses").await, 4); + assert_eq!( + db.scalar("SELECT COUNT(*) FROM audit_log WHERE source = 'import'") + .await, + 5 + ); + + let out = binary() + .args(cli_args(&db, "invalid_nets.csv", "Other")) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(3)); + assert!(String::from_utf8_lossy(&out.stderr).contains("line 4")); + let out = binary() + .args(cli_args(&db, "mismatch.csv", "Other")) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(4)); + let out = binary() + .args(cli_args(&db, "bad_header.csv", "Other")) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(3)); + assert_eq!(db.scalar("SELECT COUNT(*) FROM accounts").await, 1); +}