diff --git a/README.md b/README.md index a0ffade..26a0bad 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,12 @@ The transcript remains in SQLite after the audio files are deleted. A future loc Docker persists SQLite metadata in `./data/voice-kb.sqlite3` and audio awaiting transcription in `./data/audio/`. After a non-empty transcript is saved, its audio files are removed. Back up the whole `./data` directory while the service is stopped or use a SQLite-aware backup procedure. Model files should also be stored locally and backed up separately if needed. +Faerro KB can upload a consistent SQLite snapshot to a Nextcloud or other WebDAV folder whenever a transcript is saved. Open **Backup settings** in the app and enter the folder's WebDAV URL, username, and an app password. The destination file is `faerro-kb.sqlite3`; the folder must already exist. For Nextcloud, the URL commonly starts with `https:///remote.php/dav/files//`. Use HTTPS and an app password rather than your account password. + +Alternatively, set `BACKUP_WEBDAV_URL`, `BACKUP_WEBDAV_USERNAME`, and `BACKUP_WEBDAV_PASSWORD` in `.env`. A non-empty `BACKUP_WEBDAV_URL` makes those settings environment-managed and read-only in the UI. When configured in the UI, credentials are stored on the server in `./data/webdav-backup.json` with owner-only file permissions; include the data directory in your own local backups if you need to preserve that configuration. + +The API records pending backup work in SQLite and retries it at startup or after a later transcript save; a failed upload does not undo the saved transcript. The settings dialog shows failures and offers a retry. WebDAV `PUT` replaces the complete database file: WebDAV does not provide safe incremental SQLite page syncing. Configure Nextcloud file versioning separately if you want server-side historical versions. This feature backs up the database only, not audio files, and does not replace a backup of the full `./data` directory. The app has no login layer, so keep it behind your VPN/firewall; anyone who can reach it can alter backup settings, as well as access the notes API. + ## Development ```sh diff --git a/app/main.py b/app/main.py index 932db23..1cdf874 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,5 @@ import asyncio +import json import logging import os import re @@ -9,6 +10,7 @@ import time import uuid from datetime import datetime, timezone from pathlib import Path +from urllib.parse import urlsplit import httpx from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, UploadFile @@ -18,10 +20,16 @@ from fastapi.staticfiles import StaticFiles DATA_DIR = Path(os.environ.get("DATA_DIR", "./data")) AUDIO_DIR = DATA_DIR / "audio" DB_PATH = DATA_DIR / "faerro-kb.sqlite3" +BACKUP_SETTINGS_PATH = DATA_DIR / "webdav-backup.json" WHISPER_SERVER_URL = os.environ.get("WHISPER_SERVER_URL", "").strip() +BACKUP_WEBDAV_URL = os.environ.get("BACKUP_WEBDAV_URL", "").strip() +BACKUP_WEBDAV_USERNAME = os.environ.get("BACKUP_WEBDAV_USERNAME", "") +BACKUP_WEBDAV_PASSWORD = os.environ.get("BACKUP_WEBDAV_PASSWORD", "") FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend" WHISPER_TEST_AUDIO_PATH = Path(__file__).resolve().parent.parent / "audio test.wav" TRANSCRIPTION_TASKS: set[asyncio.Task[None]] = set() +BACKUP_TASK: asyncio.Task[None] | None = None +BACKUP_LOCK = asyncio.Lock() logger = logging.getLogger(__name__) logging.getLogger("httpx").setLevel(logging.WARNING) @@ -53,6 +61,15 @@ def initialize_db() -> None: """CREATE VIRTUAL TABLE IF NOT EXISTS note_search USING fts5(note_id UNINDEXED, transcript)""" ) + connection.execute( + """CREATE TABLE IF NOT EXISTS backup_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL DEFAULT 0, + completed_revision INTEGER NOT NULL DEFAULT 0, + last_error TEXT + )""" + ) + connection.execute("INSERT OR IGNORE INTO backup_state (id) VALUES (1)") columns = {row[1] for row in connection.execute("PRAGMA table_info(notes)")} if "original_audio_path" not in columns: connection.execute("ALTER TABLE notes ADD COLUMN original_audio_path TEXT") @@ -86,6 +103,145 @@ def update_transcript(note_id: str, transcript: str, status: str) -> None: "INSERT INTO note_search (note_id, transcript) VALUES (?, ?)", (note_id, transcript.strip()), ) + if result.rowcount and status != "transcription_failed": + connection.execute("UPDATE backup_state SET revision = revision + 1 WHERE id = 1") + if result.rowcount and status != "transcription_failed": + schedule_backup() + + +def get_backup_settings() -> dict[str, str]: + if BACKUP_WEBDAV_URL: + return { + "url": BACKUP_WEBDAV_URL, + "username": BACKUP_WEBDAV_USERNAME, + "password": BACKUP_WEBDAV_PASSWORD, + } + try: + settings = json.loads(BACKUP_SETTINGS_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + settings = {} + return { + "url": settings.get("url", ""), + "username": settings.get("username", ""), + "password": settings.get("password", ""), + } + + +def save_backup_settings(settings: dict[str, str]) -> None: + temporary_path = BACKUP_SETTINGS_PATH.with_suffix(".tmp") + temporary_path.write_text(json.dumps(settings), encoding="utf-8") + temporary_path.chmod(0o600) + temporary_path.replace(BACKUP_SETTINGS_PATH) + + +def backup_state() -> dict[str, object]: + with connect_db() as connection: + row = connection.execute( + "SELECT revision, completed_revision, last_error FROM backup_state WHERE id = 1" + ).fetchone() + settings = get_backup_settings() + return { + "enabled": bool(settings["url"]), + "managed_by_environment": bool(BACKUP_WEBDAV_URL), + "url": settings["url"], + "username": settings["username"], + "password_set": bool(settings["password"]), + "revision": row["revision"], + "completed_revision": row["completed_revision"], + "pending": row["revision"] > row["completed_revision"], + "last_error": row["last_error"], + } + + +def schedule_backup() -> None: + global BACKUP_TASK + if BACKUP_TASK and not BACKUP_TASK.done(): + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + BACKUP_TASK = loop.create_task(run_backup()) + + +async def file_chunks(path: Path): + with path.open("rb") as backup_file: + while chunk := await asyncio.to_thread(backup_file.read, 1024 * 1024): + yield chunk + + +async def create_database_snapshot(path: Path) -> None: + def copy_database() -> None: + source = sqlite3.connect(DB_PATH) + destination = sqlite3.connect(path) + try: + source.backup(destination) + finally: + destination.close() + source.close() + + await asyncio.to_thread(copy_database) + + +def webdav_file_url(folder_url: str) -> str: + parsed = urlsplit(folder_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.query or parsed.fragment: + raise ValueError("Enter a valid WebDAV folder URL") + if parsed.username or parsed.password: + raise ValueError("Put WebDAV credentials in the username and password fields") + return folder_url.rstrip("/") + "/faerro-kb.sqlite3" + + +async def run_backup() -> None: + async with BACKUP_LOCK: + while True: + settings = get_backup_settings() + if not settings["url"]: + return + with connect_db() as connection: + state = connection.execute( + "SELECT revision, completed_revision FROM backup_state WHERE id = 1" + ).fetchone() + if state["revision"] <= state["completed_revision"]: + return + revision = state["revision"] + snapshot_path = None + try: + target_url = webdav_file_url(settings["url"]) + with tempfile.NamedTemporaryFile(suffix=".sqlite3", dir=DATA_DIR, delete=False) as snapshot: + snapshot_path = Path(snapshot.name) + await create_database_snapshot(snapshot_path) + with sqlite3.connect(snapshot_path) as snapshot_connection: + snapshot_connection.execute( + "UPDATE backup_state SET completed_revision = ?, last_error = NULL WHERE id = 1", + (revision,), + ) + auth = (settings["username"], settings["password"]) if settings["username"] else None + async with httpx.AsyncClient(timeout=120, auth=auth) as client: + response = await client.put(target_url, content=file_chunks(snapshot_path)) + response.raise_for_status() + with connect_db() as connection: + connection.execute( + "UPDATE backup_state SET completed_revision = ?, last_error = NULL WHERE id = 1", + (revision,), + ) + except httpx.HTTPStatusError as error: + error_message = f"WebDAV returned HTTP {error.response.status_code}" + with connect_db() as connection: + connection.execute("UPDATE backup_state SET last_error = ? WHERE id = 1", (error_message,)) + logger.warning("WebDAV database backup failed: HTTP %s", error.response.status_code) + return + except Exception as error: + error_message = "WebDAV backup failed; check the folder URL and credentials" + if isinstance(error, ValueError): + error_message = str(error) + with connect_db() as connection: + connection.execute("UPDATE backup_state SET last_error = ? WHERE id = 1", (error_message,)) + logger.warning("WebDAV database backup failed: %s", type(error).__name__) + return + finally: + if snapshot_path: + snapshot_path.unlink(missing_ok=True) def remove_transcribed_audio(note_id: str) -> None: @@ -175,6 +331,12 @@ async def test_whisper_connection() -> bool: @app.on_event("startup") async def resume_pending_transcriptions() -> None: + if BACKUP_WEBDAV_URL: + with connect_db() as connection: + connection.execute( + "UPDATE backup_state SET revision = 1 WHERE id = 1 AND revision = 0 AND completed_revision = 0" + ) + schedule_backup() with connect_db() as connection: connection.execute( "UPDATE notes SET status = 'awaiting_transcription' WHERE status = 'transcribing'" @@ -210,6 +372,67 @@ def health() -> dict[str, str]: return {"status": "ok"} +@app.get("/api/backup") +def get_backup_status() -> dict[str, object]: + return backup_state() + + +@app.put("/api/backup/config") +def configure_backup(payload: dict[str, str]) -> dict[str, object]: + if BACKUP_WEBDAV_URL: + raise HTTPException(status_code=409, detail="WebDAV backup is managed by environment variables") + url = payload.get("url", "").strip() + username = payload.get("username", "").strip() + current = get_backup_settings() + password = "" if payload.get("clear_password") == "true" else current["password"] + if payload.get("password"): + password = payload["password"] + if url: + try: + webdav_file_url(url) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + save_backup_settings({"url": url, "username": username, "password": password}) + if url: + with connect_db() as connection: + connection.execute("UPDATE backup_state SET revision = revision + 1 WHERE id = 1") + schedule_backup() + return backup_state() + + +@app.post("/api/backup/test") +async def test_backup_connection() -> dict[str, str]: + settings = get_backup_settings() + if not settings["url"]: + raise HTTPException(status_code=400, detail="Configure a WebDAV folder first") + try: + folder_url = settings["url"].rstrip("/") + webdav_file_url(folder_url) + test_url = f"{folder_url}/faerro-kb-test-{uuid.uuid4().hex}" + auth = (settings["username"], settings["password"]) if settings["username"] else None + async with httpx.AsyncClient(timeout=30, auth=auth) as client: + response = await client.put(test_url, content=b"Faerro KB WebDAV test") + response.raise_for_status() + response = await client.delete(test_url) + response.raise_for_status() + return {"status": "ok", "message": "WebDAV write and delete test passed."} + except httpx.HTTPStatusError as error: + raise HTTPException( + status_code=502, + detail=f"WebDAV returned HTTP {error.response.status_code} during the test", + ) from error + except Exception as error: + raise HTTPException(status_code=502, detail="Could not connect to the WebDAV folder") from error + + +@app.post("/api/backup/retry", status_code=202) +def retry_backup() -> dict[str, object]: + if not get_backup_settings()["url"]: + raise HTTPException(status_code=400, detail="Configure a WebDAV folder first") + schedule_backup() + return backup_state() + + @app.post("/api/notes", status_code=201) async def create_note( background_tasks: BackgroundTasks, @@ -367,7 +590,7 @@ def delete_note(note_id: str) -> dict[str, str]: @app.put("/api/notes/{note_id}/transcript") -def save_transcript(note_id: str, payload: dict[str, str]) -> dict[str, str]: +async def save_transcript(note_id: str, payload: dict[str, str]) -> dict[str, str]: transcript = payload.get("transcript", "").strip() with connect_db() as connection: note = connection.execute("SELECT status FROM notes WHERE id = ?", (note_id,)).fetchone() diff --git a/docker-compose.yml b/docker-compose.yml index dcd8997..bc200aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,9 @@ services: environment: DATA_DIR: /data WHISPER_SERVER_URL: ${WHISPER_SERVER_URL:-} + BACKUP_WEBDAV_URL: ${BACKUP_WEBDAV_URL:-} + BACKUP_WEBDAV_USERNAME: ${BACKUP_WEBDAV_USERNAME:-} + BACKUP_WEBDAV_PASSWORD: ${BACKUP_WEBDAV_PASSWORD:-} volumes: - ./data:/data restart: unless-stopped \ No newline at end of file diff --git a/frontend/app.js b/frontend/app.js index c2213f8..613b700 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -7,6 +7,8 @@ const elements = Object.fromEntries([ 'record-button', 'record-title', 'record-help', 'record-time', 'record-meter-fill', 'audio-level', 'audio-level-fill', 'capture-message', 'audio-file-input', 'queue-list', 'queue-empty', 'queue-count', 'sync-button', 'archive-list', 'archive-empty', 'network-state', 'storage-status', 'voice-mode', 'write-mode', 'voice-panel', 'write-panel', 'written-note', 'draft-status', 'save-written-note', + 'settings-button', 'settings-dialog', 'backup-form', 'backup-fields', 'backup-url', 'backup-username', 'backup-password', + 'clear-backup-password', 'close-settings', 'test-backup', 'backup-status', 'retry-backup', ].map((id) => [id, document.getElementById(id)])); let databasePromise; @@ -154,6 +156,94 @@ function updateNetwork() { } } +async function refreshBackupStatus() { + try { + const response = await fetch('/api/backup'); + if (!response.ok) throw new Error('Backup status unavailable'); + const backup = await response.json(); + elements['backup-fields'].disabled = backup.managed_by_environment; + const managedNotice = backup.managed_by_environment ? 'Settings are managed by environment variables. ' : ''; + if (backup.last_error) { + elements['backup-status'].textContent = `${managedNotice}Backup issue: ${backup.last_error}`; + } else if (backup.pending && backup.enabled) { + elements['backup-status'].textContent = `${managedNotice}A database snapshot is waiting to upload.`; + } else if (backup.pending) { + elements['backup-status'].textContent = 'Backup is waiting; configure a WebDAV folder to upload it.'; + } else if (backup.managed_by_environment) { + elements['backup-status'].textContent = `${managedNotice}Latest database snapshot is current.`; + } else if (backup.enabled) { + elements['backup-status'].textContent = 'WebDAV backup is enabled. Latest database snapshot is current.'; + } else { + elements['backup-status'].textContent = 'WebDAV backup is not configured.'; + } + elements['retry-backup'].hidden = !backup.enabled || !backup.pending; + if (!backup.managed_by_environment) { + elements['backup-url'].value = backup.url; + elements['backup-username'].value = backup.username; + } + return backup; + } catch { + elements['backup-status'].textContent = 'Could not load backup status.'; + return null; + } +} + +async function saveBackupConfiguration() { + const response = await fetch('/api/backup/config', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: elements['backup-url'].value.trim(), + username: elements['backup-username'].value.trim(), + password: elements['backup-password'].value, + clear_password: String(elements['clear-backup-password'].checked), + }), + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || 'Could not save backup settings.'); + elements['backup-password'].value = ''; + elements['clear-backup-password'].checked = false; + await refreshBackupStatus(); +} + +elements['settings-button'].addEventListener('click', () => { + elements['settings-dialog'].showModal(); + refreshBackupStatus(); +}); +elements['close-settings'].addEventListener('click', () => elements['settings-dialog'].close()); +elements['backup-form'].addEventListener('submit', async (event) => { + event.preventDefault(); + elements['backup-status'].textContent = 'Saving settings…'; + try { + await saveBackupConfiguration(); + } catch (error) { + elements['backup-status'].textContent = error.message; + } +}); +elements['test-backup'].addEventListener('click', async () => { + elements['backup-status'].textContent = 'Testing WebDAV…'; + try { + await saveBackupConfiguration(); + const response = await fetch('/api/backup/test', { method: 'POST' }); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || 'WebDAV test failed.'); + elements['backup-status'].textContent = result.message; + } catch (error) { + elements['backup-status'].textContent = error.message; + } +}); +elements['retry-backup'].addEventListener('click', async () => { + elements['backup-status'].textContent = 'Queuing backup retry…'; + try { + const response = await fetch('/api/backup/retry', { method: 'POST' }); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || 'Could not retry backup.'); + await refreshBackupStatus(); + } catch (error) { + elements['backup-status'].textContent = error.message; + } +}); + function formatDuration(milliseconds) { const seconds = Math.floor(milliseconds / 1000); return `${String(Math.floor(seconds / 60)).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`; diff --git a/frontend/index.html b/frontend/index.html index 11ec3b1..9abda77 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -14,9 +14,37 @@
KBFaerro KB - Checking +
+ + Checking +
+ +
+
+

Database backup

+ +
+

A consistent SQLite snapshot is uploaded after each saved transcript. WebDAV receives the full database file each time.

+
+ + + + + + + +
+ + +
+
+

Loading backup status…

+ +
+ +
diff --git a/frontend/styles.css b/frontend/styles.css index 3796afc..c754075 100644 --- a/frontend/styles.css +++ b/frontend/styles.css @@ -16,6 +16,7 @@ * { box-sizing: border-box; } body { margin: 0; background: var(--paper); color: var(--ink); font-family: var(--sans); } .topbar { height: 68px; padding: 0 max(24px, calc((100vw - 1040px) / 2)); border-bottom: 1px solid var(--line); display: flex; align-items: center; justify-content: space-between; } +.topbar-actions { display: flex; align-items: center; gap: 18px; } .brand { display: flex; align-items: center; gap: 10px; text-decoration: none; color: var(--ink); font-size: 15px; font-weight: 600; } .brand b { color: var(--moss); } .brand-mark { width: 28px; height: 28px; flex: 0 0 auto; border-radius: 50%; background: var(--moss-dark); color: #fff; display: grid; place-items: center; font: 500 15px var(--serif); } @@ -68,6 +69,23 @@ h1 em { color: var(--moss); font-weight: 500; } h2 { margin: 8px 0 0; font: 500 25px/1.1 var(--serif); } #queue-count { display: inline-grid; place-items: center; width: 23px; height: 23px; margin-left: 5px; border-radius: 50%; background: #e6e7df; color: var(--moss-dark); font: 11px var(--mono); vertical-align: 3px; } .icon-button { position: absolute; right: 0; width: 38px; height: 38px; border: 1px solid var(--line); background: transparent; color: var(--moss-dark); font-size: 22px; cursor: pointer; } +.settings-button { position: static; flex: 0 0 auto; font-size: 18px; } +.settings-dialog { width: min(520px, calc(100% - 32px)); max-height: min(720px, calc(100% - 32px)); border: 1px solid var(--line); padding: 24px; color: var(--ink); background: var(--paper); } +.settings-dialog::backdrop { background: rgb(32 39 36 / 45%); } +.dialog-heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; } +.dialog-heading h2 { margin: 0; } +.dialog-close { position: static; flex: 0 0 auto; font-size: 24px; } +.settings-copy { margin: 16px 0; color: #606b63; font-size: 12px; line-height: 1.5; } +#backup-fields { display: grid; gap: 8px; margin: 0; border: 0; padding: 0; } +#backup-fields:disabled { opacity: .62; } +#backup-fields > label { margin-top: 6px; font: 10px var(--mono); color: var(--muted); } +#backup-fields > input { width: 100%; min-width: 0; height: 40px; border: 1px solid var(--line); padding: 0 10px; color: var(--ink); background: #fffefa; font: 12px var(--sans); } +#backup-fields > input:focus { outline: 2px solid var(--moss); outline-offset: 1px; } +.clear-password { display: flex; align-items: center; gap: 8px; } +.settings-actions { display: flex; flex-wrap: wrap; gap: 9px; margin-top: 8px; } +.secondary-button { min-height: 40px; border: 1px solid var(--line); padding: 0 13px; color: var(--moss-dark); background: transparent; font: 11px var(--mono); cursor: pointer; } +.backup-status { min-height: 20px; margin: 16px 0 0; color: var(--muted); font-size: 11px; line-height: 1.5; overflow-wrap: anywhere; } +.retry-backup { margin-top: 8px; } .icon-button:disabled { opacity: .4; cursor: wait; } .queue-list, .archive-list { list-style: none; margin: 16px 0 0; padding: 0; } .queue-item, .archive-item { display: flex; align-items: flex-start; gap: 15px; padding: 14px 0; border-top: 1px solid #e3e4dd; animation: reveal .35s ease both; } @@ -94,6 +112,7 @@ footer { max-width: 1040px; margin: auto; padding: 18px 24px calc(18px + env(saf @keyframes transcription-progress { from { transform: translateX(-110%); } to { transform: translateX(300%); } } @media (max-width: 600px) { .topbar { height: 58px; padding: 0 18px; } + .topbar-actions { gap: 10px; } main { padding: 0 18px; } .capture { padding: 38px 0 30px; background-position: right 8px top 25px; } .capture-mode { min-width: 100px; } @@ -108,5 +127,6 @@ footer { max-width: 1040px; margin: auto; padding: 18px 24px calc(18px + env(saf h2 { font-size: 23px; } .archive-date { width: 60px; font-size: 9px; } footer { padding-left: 18px; padding-right: 18px; font-size: 8px; } + .settings-dialog { padding: 18px; } } @media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation-duration: .01ms !important; transition-duration: .01ms !important; } } \ No newline at end of file diff --git a/frontend/sw.js b/frontend/sw.js index 1b5dd41..f50d34b 100644 --- a/frontend/sw.js +++ b/frontend/sw.js @@ -1,4 +1,4 @@ -const CACHE_NAME = 'faerro-kb-shell-v3'; +const CACHE_NAME = 'faerro-kb-shell-v4'; const APP_SHELL = ['/', '/styles.css', '/app.js', '/manifest.webmanifest', '/favicon.svg']; self.addEventListener('install', (event) => {