diff --git a/README.md b/README.md index 26a0bad..321a551 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Faerro KB can upload a consistent SQLite snapshot to a Nextcloud or other WebDAV Alternatively, set `BACKUP_WEBDAV_URL`, `BACKUP_WEBDAV_USERNAME`, and `BACKUP_WEBDAV_PASSWORD` in `.env`. A non-empty `BACKUP_WEBDAV_URL` makes those settings environment-managed and read-only in the UI. When configured in the UI, credentials are stored on the server in `./data/webdav-backup.json` with owner-only file permissions; include the data directory in your own local backups if you need to preserve that configuration. -The API records pending backup work in SQLite and retries it at startup or after a later transcript save; a failed upload does not undo the saved transcript. The settings dialog shows failures and offers a retry. WebDAV `PUT` replaces the complete database file: WebDAV does not provide safe incremental SQLite page syncing. Configure Nextcloud file versioning separately if you want server-side historical versions. This feature backs up the database only, not audio files, and does not replace a backup of the full `./data` directory. The app has no login layer, so keep it behind your VPN/firewall; anyone who can reach it can alter backup settings, as well as access the notes API. +The API records pending backup work in SQLite and retries it at startup or after a later transcript save; a failed upload does not undo the saved transcript. The settings dialog shows failures and offers a retry. Rotation keeps the current `faerro-kb.sqlite3` latest file plus timestamped archives: the newest snapshot for each of the last 7 UTC dates, 4 ISO weeks, 12 calendar months, and 5 years. Overlapping periods share an archive. Only files matching Faerro KB's timestamped archive naming pattern in the configured folder are eligible for deletion; other files are left alone. WebDAV `PUT` replaces the complete database file: WebDAV does not provide safe incremental SQLite page syncing. Configure Nextcloud file versioning separately if you want server-side historical versions. This feature backs up the database only, not audio files, and does not replace a backup of the full `./data` directory. The app has no login layer, so keep it behind your VPN/firewall; anyone who can reach it can alter backup settings, as well as access the notes API. ## Development diff --git a/app/main.py b/app/main.py index 1cdf874..6300610 100644 --- a/app/main.py +++ b/app/main.py @@ -8,9 +8,10 @@ import subprocess import tempfile import time import uuid -from datetime import datetime, timezone +import xml.etree.ElementTree as ET +from datetime import datetime, timedelta, timezone from pathlib import Path -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit import httpx from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, UploadFile @@ -192,6 +193,79 @@ def webdav_file_url(folder_url: str) -> str: return folder_url.rstrip("/") + "/faerro-kb.sqlite3" +def retained_backup_archives(names: list[str], now: datetime) -> set[str]: + archives = [] + for name in names: + match = re.fullmatch(r"faerro-kb-(\d{8}T\d{6}Z)-r\d+\.sqlite3", name) + if not match: + continue + timestamp = datetime.strptime(match.group(1), "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) + if timestamp <= now: + archives.append((name, timestamp)) + + daily: dict[object, tuple[str, datetime]] = {} + weekly: dict[object, tuple[str, datetime]] = {} + monthly: dict[object, tuple[str, datetime]] = {} + yearly: dict[object, tuple[str, datetime]] = {} + today = now.date() + current_week = today - timedelta(days=today.weekday()) + current_month = now.year * 12 + now.month + + for name, timestamp in archives: + date = timestamp.date() + age_days = (today - date).days + week = date - timedelta(days=date.weekday()) + age_weeks = (current_week - week).days // 7 + age_months = current_month - (timestamp.year * 12 + timestamp.month) + age_years = now.year - timestamp.year + for buckets, key, age, limit in ( + (daily, date, age_days, 7), + (weekly, (week.isocalendar().year, week.isocalendar().week), age_weeks, 4), + (monthly, (timestamp.year, timestamp.month), age_months, 12), + (yearly, timestamp.year, age_years, 5), + ): + if 0 <= age < limit and (key not in buckets or timestamp > buckets[key][1]): + buckets[key] = (name, timestamp) + + return { + name + for buckets in (daily, weekly, monthly, yearly) + for name, _timestamp in buckets.values() + } + + +async def rotate_remote_backups(client: httpx.AsyncClient, folder_url: str) -> None: + request_body = b""" + """ + response = await client.request( + "PROPFIND", + folder_url, + headers={"Depth": "1", "Content-Type": "application/xml"}, + content=request_body, + ) + response.raise_for_status() + root = ET.fromstring(response.content) + folder_path = urlsplit(folder_url).path.rstrip("/") + remote_archives = [] + for item in root.findall("{DAV:}response"): + href = item.findtext("{DAV:}href") + if not href: + continue + item_url = httpx.URL(folder_url).join(href) + item_path = item_url.path + if item_path.rsplit("/", 1)[0].rstrip("/") != folder_path: + continue + name = unquote(item_path.rsplit("/", 1)[-1]) + if re.fullmatch(r"faerro-kb-\d{8}T\d{6}Z-r\d+\.sqlite3", name): + remote_archives.append((name, str(item_url))) + + retained = retained_backup_archives([name for name, _url in remote_archives], datetime.now(timezone.utc)) + for name, item_url in remote_archives: + if name not in retained: + response = await client.delete(item_url) + response.raise_for_status() + + async def run_backup() -> None: async with BACKUP_LOCK: while True: @@ -217,9 +291,19 @@ async def run_backup() -> None: (revision,), ) auth = (settings["username"], settings["password"]) if settings["username"] else None + archive_name = ( + f"faerro-kb-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" + f"-r{revision:010d}.sqlite3" + ) async with httpx.AsyncClient(timeout=120, auth=auth) as client: + response = await client.put( + f"{settings['url'].rstrip('/')}/{archive_name}", + content=file_chunks(snapshot_path), + ) + response.raise_for_status() response = await client.put(target_url, content=file_chunks(snapshot_path)) response.raise_for_status() + await rotate_remote_backups(client, settings["url"]) with connect_db() as connection: connection.execute( "UPDATE backup_state SET completed_revision = ?, last_error = NULL WHERE id = 1", @@ -378,7 +462,7 @@ def get_backup_status() -> dict[str, object]: @app.put("/api/backup/config") -def configure_backup(payload: dict[str, str]) -> dict[str, object]: +async def configure_backup(payload: dict[str, str]) -> dict[str, object]: if BACKUP_WEBDAV_URL: raise HTTPException(status_code=409, detail="WebDAV backup is managed by environment variables") url = payload.get("url", "").strip() @@ -411,11 +495,18 @@ async def test_backup_connection() -> dict[str, str]: test_url = f"{folder_url}/faerro-kb-test-{uuid.uuid4().hex}" auth = (settings["username"], settings["password"]) if settings["username"] else None async with httpx.AsyncClient(timeout=30, auth=auth) as client: + response = await client.request( + "PROPFIND", + folder_url, + headers={"Depth": "1", "Content-Type": "application/xml"}, + content=b'', + ) + response.raise_for_status() response = await client.put(test_url, content=b"Faerro KB WebDAV test") response.raise_for_status() response = await client.delete(test_url) response.raise_for_status() - return {"status": "ok", "message": "WebDAV write and delete test passed."} + return {"status": "ok", "message": "WebDAV listing, write, and delete test passed."} except httpx.HTTPStatusError as error: raise HTTPException( status_code=502, @@ -426,7 +517,7 @@ async def test_backup_connection() -> dict[str, str]: @app.post("/api/backup/retry", status_code=202) -def retry_backup() -> dict[str, object]: +async def retry_backup() -> dict[str, object]: if not get_backup_settings()["url"]: raise HTTPException(status_code=400, detail="Configure a WebDAV folder first") schedule_backup() diff --git a/frontend/index.html b/frontend/index.html index 9abda77..045d7dd 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -26,7 +26,7 @@

Database backup

-

A consistent SQLite snapshot is uploaded after each saved transcript. WebDAV receives the full database file each time.

+

A consistent SQLite snapshot is uploaded after each saved transcript. Keeps latest, 7 daily, 4 weekly, 12 monthly, and 5 yearly snapshots. WebDAV receives the full database file each time.