Add backup for bictoin-pulse db

configuration.nix

@@ -119,5 +119,11 @@
     defaultSopsFile = ./secrets/tobserver.yaml;
     defaultSopsFormat = "yaml";
     age.keyFile = "/var/lib/sops-nix/key.txt";
+
+    secrets."backblaze/tobcloud-backup-env" = {
+      owner = "root";
+      group = "root";
+      mode = "0400";
+    };
   };
 }

docs/bitcoin-pulse-postgresql.md

@@ -19,9 +19,9 @@
   change after testing backups, startup, and application compatibility.
 
 The named volume belongs to the rootless Podman account and survives container
-removal, service recreation, and NixOS generation changes. Backup and restore
+removal, service recreation, and NixOS generation changes. Backup portability
-portability is handled separately by BIT-23; do not copy Podman's private
+is provided by the logical Restic backup described below; do not copy Podman's
-storage directly.
+private storage directly.
 
 ## Human-managed secrets
 
@@ -41,8 +41,76 @@ Nix expression, command argument, shell history, log, or pull-request text. The
 agent change intentionally declares only the secret paths; it does not modify
 or decrypt `secrets/tobserver.yaml`.
 
-`sops-nix` materializes each credential as a restricted runtime file. Before
+## Logical off-server backups
-container startup, the service stages private copies owned by PostgreSQL's uid
+
+`modules/bitcoin-pulse.nix` declares the daily
+`restic-backups-bitcoin-pulse.service` and its systemd timer. The job:
+
+1. Requires the PostgreSQL container service to be available.
+2. Creates a consistent custom-format `pg_dump` through the loopback database
+   port using the administrator password as a systemd credential.
+3. Uploads the dump to the separate Restic repository
+   `b2:tobcloud-backup:bitcoin-pulse`.
+4. Removes the local staging dump whether the upload succeeds or fails.
+5. Retains 7 daily, 4 weekly, and 12 monthly snapshots.
+
+The timer starts the job daily between 03:00 and 03:30 Europe/Berlin time. It is
+persistent, so systemd starts a missed job after the next boot. A failed dump,
+upload, or prune operation makes the unit fail and leaves details in the system
+journal. A run that exceeds six hours is stopped and marked failed so it cannot
+block future timer runs indefinitely. Alert delivery and automated restore
+testing are intentionally outside this deployment scope.
+
+### Backup credentials
+
+The existing `tobcloud-backup` bucket and its Backblaze application key are
+shared with the Nextcloud backup. Before deploying, confirm that the key is not
+restricted only to the `nextcloud/` prefix and can also access
+`bitcoin-pulse/`. For the native B2 backend, it needs `listBuckets`,
+`listFiles`, `readFiles`, `writeFiles`, and `deleteFiles` because Restic must
+find the bucket and retention must remove expired repository files.
+
+Generate a separate, strong Restic repository password and retain it in the
+approved password manager. Losing this password makes every backup in the
+repository unusable.
+
+Use `sops secrets/tobserver.yaml` from an authorized workstation to add:
+
+```yaml
+restic:
+  bitcoin-pulse-password: <new Restic repository password>
+backblaze:
+  tobcloud-backup-env: |
+    B2_ACCOUNT_ID=<shared application key ID>
+    B2_ACCOUNT_KEY=<shared application key>
+```
+
+Rename the existing `backblaze/nextcloud-env` SOPS entry to
+`backblaze/tobcloud-backup-env`; do not keep a second plaintext copy of the same
+credential. Commit only the encrypted result. The NixOS configuration will not
+activate successfully until the renamed Backblaze key and the new Restic
+password exist.
+
+### Initial backup verification
+
+After adding the credentials and deploying the reviewed NixOS generation, run
+the first backup manually rather than waiting for the timer:
+
+```console
+sudo systemctl start restic-backups-bitcoin-pulse.service
+sudo systemctl status restic-backups-bitcoin-pulse.service
+sudo journalctl -u restic-backups-bitcoin-pulse.service -b --no-pager
+sudo restic-bitcoin-pulse snapshots
+systemctl list-timers restic-backups-bitcoin-pulse.timer
+```
+
+The service must exit successfully, `snapshots` must show the new off-server
+snapshot, and `/var/backup/bitcoin-pulse` must not retain a dump after the job.
+The journal must not contain any credential value.
+
+For PostgreSQL, `sops-nix` materializes each credential as a restricted runtime
+file. Before container startup, the service stages private copies owned by
+PostgreSQL's uid
 inside the rootless user namespace. The copies are mounted read-only and are
 removed from the host runtime directory after PostgreSQL reports healthy and
 again when the unit stops. The application and migration SOPS files remain

modules/bitcoin-pulse.nix

@@ -1,6 +1,10 @@
 { bitcoin-pulse, config, lib, pkgs, ... }:
 
 let
+  backupDirectory = "/var/backup/bitcoin-pulse";
+  backupDump = "${backupDirectory}/bitcoin-pulse.dump";
+  backupRepository = "b2:tobcloud-backup:bitcoin-pulse";
+  backupServiceName = "restic-backups-bitcoin-pulse";
   containerName = "bitcoin-pulse-postgres";
   databaseUrl = "jdbc:postgresql://127.0.0.1:5432/bitcoin_pulse?tcpKeepAlive=true";
   migrationServiceName = "bitcoin-pulse-migrate";
@@ -10,7 +14,9 @@ let
   secretFiles = {
     admin = config.sops.secrets."bitcoin-pulse/postgres/admin-password".path;
     app = config.sops.secrets."bitcoin-pulse/postgres/app-password".path;
+    backblaze = config.sops.secrets."backblaze/tobcloud-backup-env".path;
     migration = config.sops.secrets."bitcoin-pulse/postgres/migration-password".path;
+    restic = config.sops.secrets."restic/bitcoin-pulse-password".path;
   };
 
   prepareSecrets = pkgs.writeShellApplication {
@@ -106,6 +112,67 @@ in
     };
   };
 
+  services.restic.backups.bitcoin-pulse = {
+    initialize = true;
+    repository = backupRepository;
+
+    passwordFile = secretFiles.restic;
+    environmentFile = secretFiles.backblaze;
+
+    paths = [ backupDump ];
+
+    backupPrepareCommand = ''
+      set -euo pipefail
+
+      install -d -m 0700 -o root -g root ${lib.escapeShellArg backupDirectory}
+
+      dump_path=${lib.escapeShellArg backupDump}
+      temporary_dump="$dump_path.tmp"
+      pgpass_file="$RUNTIME_DIRECTORY/postgresql.pgpass"
+
+      rm -f "$dump_path" "$temporary_dump" "$pgpass_file"
+
+      password="$(cat "$CREDENTIALS_DIRECTORY/postgres-password")"
+      escaped_password="''${password//\\/\\\\}"
+      escaped_password="''${escaped_password//:/\\:}"
+      printf '127.0.0.1:5432:bitcoin_pulse:bitcoin_pulse_admin:%s\n' \
+        "$escaped_password" > "$pgpass_file"
+      chmod 0600 "$pgpass_file"
+
+      trap 'rm -f "$temporary_dump" "$pgpass_file"' EXIT
+
+      PGPASSFILE="$pgpass_file" ${pkgs.postgresql_18}/bin/pg_dump \
+        --host=127.0.0.1 \
+        --port=5432 \
+        --username=bitcoin_pulse_admin \
+        --dbname=bitcoin_pulse \
+        --format=custom \
+        --no-password \
+        --file="$temporary_dump"
+
+      mv "$temporary_dump" "$dump_path"
+    '';
+
+    backupCleanupCommand = ''
+      rm -f \
+        ${lib.escapeShellArg backupDump} \
+        ${lib.escapeShellArg "${backupDump}.tmp"} \
+        "$RUNTIME_DIRECTORY/postgresql.pgpass"
+    '';
+
+    timerConfig = {
+      OnCalendar = "*-*-* 03:00:00";
+      Persistent = true;
+      RandomizedDelaySec = "30m";
+    };
+
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 12"
+    ];
+  };
+
   users.groups = {
     bitcoin-pulse = { };
     bitcoin-pulse-migration = { };
@@ -151,6 +218,12 @@ in
       group = "bitcoin-pulse-migration";
       mode = "0440";
     };
+
+    "restic/bitcoin-pulse-password" = {
+      owner = "root";
+      group = "root";
+      mode = "0400";
+    };
   };
 
   virtualisation.oci-containers = {
@@ -263,6 +336,19 @@ in
       requires = [ "${migrationServiceName}.service" ];
       after = [ "${migrationServiceName}.service" ];
     };
+
+    ${backupServiceName} = {
+      requires = [ "${serviceName}.service" ];
+      after = [ "${serviceName}.service" ];
+
+      serviceConfig = {
+        LoadCredential = [
+          "postgres-password:${secretFiles.admin}"
+        ];
+        TimeoutStartSec = "6h";
+        UMask = "0077";
+      };
+    };
   };
 
   services.nginx = {
@@ -297,5 +383,9 @@ in
     };
   };
 
+  systemd.tmpfiles.rules = [
+    "d ${backupDirectory} 0700 root root - -"
+  ];
+
   environment.systemPackages = [ pkgs.postgresql_18 ];
 }

modules/nextcloud.nix

@@ -22,12 +22,6 @@
     mode = "0400";
   };
 
-  sops.secrets."backblaze/nextcloud-env" = {
-    owner = "root";
-    group = "root";
-    mode = "0400";
-  };
-
   services = {
 
     nextcloud = {
@@ -99,7 +93,7 @@
       repository = "b2:tobcloud-backup:nextcloud";
 
       passwordFile = config.sops.secrets."restic/nextcloud-password".path;
-      environmentFile = config.sops.secrets."backblaze/nextcloud-env".path;
+      environmentFile = config.sops.secrets."backblaze/tobcloud-backup-env".path;
 
       paths = [
         "/nextcloud-data"

secrets/tobserver.yaml

@@ -5,8 +5,9 @@ nextcloud:
     admin-pass: ENC[AES256_GCM,data:3xufFHvcm5g5+jn+B4qxRljuWDqDXo1mLu4IlSuax7Cz+3ZoOuNXoCXoiaY=,iv:WaWeu/a45Vuz+HvO7m0dUlCGCexk4IX/ijURmb/7jPU=,tag:ytRjS89AmHiGOhKymViTrg==,type:str]
 restic:
     nextcloud-password: ENC[AES256_GCM,data:fbH++0KWr9V4QR90t4kmBdhwyF+dbtBdf9RFarv2E9MyrH7QMO4yxdppaeE=,iv:VnS2936xTO2sTl51lqXl8/3MBeGldARc4/ZFZYL/hG4=,tag:QGp6OFqEmt6iP11lG6GwmA==,type:str]
+    bitcoin-pulse-password: ENC[AES256_GCM,data:OAm3stWlJIcOEa/Gp5zkNq+z/gejyw5okWR0UAEKZXa6UXT/XAXGIYcnzOtbmBZK0G3Lt9OpxOyEACjL8xp+Xw==,iv:JddNNeJN5aRzvizHl/iSi3vjqVHrzrax1IPE7Fmh8bU=,tag:ckYd4yqaYjHROB3zlubleQ==,type:str]
 backblaze:
-    nextcloud-env: ENC[AES256_GCM,data:fZsW4+R0nDTqhx6pKQCOKULrheseS1j6JOBvHVCuOPaVqkkZv8/TsgtDJdWnKXTNS9/RBTqhLFPIaUSUrfH/bh6rIc4iDe0BehPzHCxh/0CBwirmu2iS,iv:6ggfA8R+zpj+kxcfaWlG4OH1Iteex6zGIin8+TUemGw=,tag:VQW3hbrbgNBQXzmeSrMaAw==,type:str]
+    tobcloud-backup-env: ENC[AES256_GCM,data:Lo2Xhjz/Wc1wPJljp7/2Vp2pSNAqGW/oOKvxc7s7q4LwJoGbOv1B2B5YqeJAgLR2LzB7cMwxpyK02uoUf7muFofYJmUC0A1CWuWe0pAsSkez7E13aOGh,iv:LEfrirUCbq4BQP3u9keo3nb6uI+geaTjjhSfhCSnRM4=,tag:A2xd+I+Mam8eXdu1r3sRpA==,type:str]
 hermes-env: ENC[AES256_GCM,data:WcN3pP4BIrMoKfehixAHZFTI2rk0js2mwi0m4+NsfgqamDH8IqHpbMJGhIg4zUmJBUlyz+4ZhjtNmvlxtj/i7UxIkPNq4qg95eK59gEM3npW9euRoX+BziKvsM6fFMRbWt60GaKigiI05z83M9I9/A8qbZC/PEWPfj6a5wlqHSVSe2qJjg5tGpthVhApaJ9ZmuTzhyHjs9zRvlkC9SFp9WsdyWWxNfChIKj8MySeAGs1YlDAGFVzsjX3+JlPVlCwPw==,iv:euK36Gk50/NQfjkspCP7tv3/a/gknzrDM21GUqVhEBY=,tag:jPF67Qa6e1VrY7byXKQw0w==,type:str]
 hermes:
     auth.json: ENC[AES256_GCM,data:sIp98d9dYqSe5vjTIaGQTgN7zl2XGRtT8CLMmqpf79DKQ2KDbbSdYyzhbbajccwH+S4T7vFYqs41hoNdf5a5zsrNBq68nQYzmT2Ja+hUCvxN0KsIF8xk/yjC56btlHBBH7ApzsDq8IQg31j1+hwXZSmLRNB4KB9fC1gWTyPedJDfVpTSmv0y37m6MauqgkJDZc1JWcYnswqW7qSqcNAAm0CwBycwEOvcxKRF9Xqg+KP6fYc+2MUagbzqWuv7JO14ZaoSrub8ob0bCHg6z21fEVkYCD4EyqTUmyOQlbmyfA6h2kjamD81sFAC/87crQ22bnTy6pUydAs85X2Ea7GGx9lirnzLzneJ5LDcFuPzxi2Aryp1/BY4bOPa8cD2kWAOH/zmqicwmATtMJPn8XVyfDLrOG+0Z6ADCFrUaKKfqJ5gzNoeF7gy3L4qYvHnvHLbpnWTWQETmMP8Y3eHTWfzqaH3m+u3yuH32Uh4NqEL7WmxROPyQhxmOjxWm4zSdvby4qsGA0ILZ39zKtzo7XTMRvKZScPql2zCmubvrFmV1pofEbqP3BN5b4kIYVa1JabpR3/saxO3+IqyoD8HsRaCsmvCSu3i4sMmxkfqLCsa8RmtpMMRgNQCie9VSDXqciAyHg6zMW3c+tZPX20gIEdgsAgis5Y2Rdbxnooh5HiU1r05p0pci1+X9sTsWs+0tyjnFwV4bYyPq/DwoPBJmiw4IyjfSgZOaiCJddWg4QnO5hVz/eWI5/g53/mlgF7AN6DRuobiepShvyZuDy7m32uBuYLOjgtvIfi+seAp+kzaPgIctjxxwRdGRs24ajuAHW69OIbwyk/zdANrK0K/5NDPHQ9vw6xUed+TCZnBZvKmyCtSNxgsnEk2cjiMxENawXTNpWbdfXAelRJP8f+hBBDJOo7FFkVLWS2OzE6m3ZHMyJf+dnQoZXLynwLTGq5aZKeP2YOVtQtjUlRnsNjKjzw8A/gAiwbu6xtIAummC9OYC1L0aX3Kh/cSg3eZw8z8Aaw7jposWXDhJaOvYYWp6RzMOtwpKU3Om27xGrsTQNBfweyX8/gucOEo70zE+XQXhH0VXwbdrGZVF1qU1bd5bx9P/GaHidkESkQrpM28L3ahJSJSCfwV+9Qd7toCotn14QPs1g4Je0GF4dd+KhyU0EKdlGW7s4+ozFfJS8PfmPVQOuXi0ul0cPPuID8CtqFjtaanWSuGRL/16CRPK71cf4/kY+JRIw/hTuQW/9lYXWW93pq2v/WLxG5505KWu+Lrmdx8MpuLKfyXPP39GkvMT1jT/zWfh9ZsmLs0emIEEoK1IZfSnFsiqcxmg6zpUBffATyBDellCFYvnDFkfpXyLYWi5uhr4hgcn5QyrXk0M98mggdEFrJtzDdm68/cNvs0pta7A6WHV5OtkjneIzsj7p4EJcEkeST6iuv/43vscL6lGI/nbCby7U+yXZdngf1nGkkJ5tpkIS/dWdnA00HIVLk9c8iPhp1HneeUB9GIKgaHRglGu7UVocqiAeRnZQoKp9giRNcKT6yPbLqVAYqjH+00C34F3DGXLaISS+wxHDGqGfxCeAj8PCC54e9XuYEbkes3BJSDsCbfeP0MAzgZfyU5RZKPyykKzseR9/5We3MZfyoXjO2FzgpKPYfUq2gHclMYvwgN6inXZ9NtFi+NZCLxs14k4fneaUI6Wgysq8GFRQ3inLjE3gWCCtt1RbyCmfFZ/8KLo8O16j0rIHvz83LLtN+N2NjibHE2f8j2WkrLd79htPGu9i61ETD19hGhx63f0EtZFgE/KUPz6EGGOzEyzwpm4VXIBNt0S53FxToVl0dNF1f7DD/RekNhBN5jkrRnkGf5wpgIceGlr7pjQUSrhArWhxAPAlsFlqE5elPdon2KfJFltSboeIDFtt7z6YEkhXL6eW3C6UM/p5WLL3Fo3s7mtyxaAq7+Yq2XJnrAvUhRuTgSEX4SgXLpIj4/xUxk5iWZcxwvjC+O7GaQ2V2JgMibIg8vhcCD/InYB+iydMegRaSCUPcbBsRUsUlwhnLlEmP9sNkzjV3qMHoemBDcKOpOudbnOUOTd9W2SYBduXSXAohWTgyCmkPTd4txMJ/bA4aXddTxkzRhkXLW8Q1lve2yhn4gS1+1xtRUjk9PgpvVKNrNXiI/nOH3sJX76Dbcrkh9f06PkB8Kt04pFVACR2JVmubcE04HQpzBmMy369iqDYQk+t9kxaJ32kTiOApV8YdbnW5xIdA9nrhZccgzP6PjY5G2uljLVlJWOB5qhk5OHiXGLy6QemW4RmM0Pe4TuF4onK61FugUan7D7X7g79Slcg8VNgOie1aaMgMJwu3RxjMKBmhbY4u5y8hx9PNPs+b8gmgn3IHfqf1GmfzOAu6VtUUxyZa+bZ+d+7C7PeotHgKz2OL/MPOykxknNmTblqzuLiwtVg1UN1SKmwQNH1Bydxxc7qCb7jTH/NAHB9D9ABrZ5WFJqZd/Ls87XJWe6VlF/8tYTuBanXFrWxd8ogh4kDEhSr/FIB1Y8Ub6qHMQJoPttmmHSIZEDSZ4QzShDocMtAPBjkMOx/UB+qbzoJq9yihszJN13+fgcERngSTXTozX+Z9+rxOVPP0PPCfiylH2d0kUS8AUejxzpVzgkycZ8bW4mkEWf6WiF0YDtEfmQBwHoGs0H6ddtamHOXZDyoi4ddxoif5CXVVbU7xbqX0T+mP/DRtQULgnwR3I/QC3q5F7B8d/zvpxHf5Mk+WCID+9zIZrRFOk3XO63OnKbndXHdCCjAqIDTPpp9yb8PGZkuTKU+jC98pgKj4kA/+FWMdCdIAKoLfepXwicHN9SoXNMDo6XZimmEzZPavQAizQF02UXDQTnsPclo7ulLrnn+J81LABcjawgkS5CGE5xioz3Z9akFY6rQ+LDlDN/88zuBIunOwconcTTstM/hNe6tC+6eleTBc2frpbKh43cB25sK6qIxTYgEP9hXTpq5PWtTMp3Jox6pgAMIP1d/Rg3Pf7MzdlIEp1d/XKPawIuvLuy6sBxJYi3k/rpDbfDKydSiKQN8Jja6uZNNrzGV6KMd5q6aXDVOyEOVlahILIYVFW8a9YvYggzL5TaAg9bOpxflqcqGg/roSHvy/e6j8Hib+R6+LLZMtqNTtT0YaQBt78i0SUo7lIuzNpAUPpHBbPJleBIUs5x6b0GIvgU7bp56UA9ZunX4ySxAO8wwhhKSP4iNpjrHCdaoJk4pxl/4ook8u9ve6Pwt9LDJeet1GMn1nHAQRXtrhx2qFrn+yrT1R48QWXi4O/eWvAKYfFp9NvMLPFcdM3qmQ6LkwbqzcZZnmck0jPmhYqaPaRxWepkdQorBO5t32lNaguYc0X5PUtqfm6e1+u1LE438tIP/9xEpDjqnBDIYO1Yw86V/SZs5THKOx88t5Mx3u9oiFFYk9+ULxgwRbCbSekdnlAdzyjueujP2dh9iGGTTNcm6ZzeVqIJCErGky2+Ffp4nl+6fbHJ9u0zjKZNtCP/AHskS7PTnI4adXDoQcJUh4pdtvkz6DFQhk0z+Af2qLu2PtWR7aVwE221PSVSdITOoj2PS4lEMuN91gkUGPf0oMl46kF9fRLMyuWqlcqJaom+Un4MrHnETf0udDicSB5hXwAHhNHPxOf4VfUUkkg3psPOZA4Jz/MIPM+at5hNAvf4qvz0TT4wl5y7MtqdDNnr1Qyo9trx+OfwYUwNZpcfEDeNtrt67EYM9L5hYY7+AUA6RKG2SnBArH6UCUl4zkgYPxecQ3aInXyvJAMAK6bfgo6IZoLxz98skByESBOkFwamqriKARt8CTC+apf9E00KnVhIHBaQx1cq5ANXU6ItydyIsT28LIcG+Wlyu1B6xONwhZTRJE+kowdRs6Znl26GjAjqh/lNAcbvGRmpmec7UfEXJyGoDhtbUQ1/QZKcXpYLwr3ABoldNKHwO1hlePWdZQ/9/p4ATkcGO3JqHwfrj2V5bSCU/79z66TS8XtRVmkic81WUjGGXiU4Z3cY1V8OFpYovwQL/opdtVjahSSuMVBAl7JXuJcmB8cqlxPTuAXT26NfbqzvDlc7ehf9nUnZOZvK30/QEh72lpeFCgPVS1Rgvt/K4oIexGPBnBkU5Wqgw==,iv:2laQqhM+pxSbClSOJBX7P9K87Iaxb23dhDqNIgy6oMM=,tag:iO4IWEZ2PJliahaVZtELzA==,type:str]
@@ -37,7 +38,7 @@ sops:
             cVE5VG56cFAzT1dsRFN4cCtnVEJJdU0KSBWm4blGhnVlOJ3k7lh8aJVOEH8NfFOh
             EwU/4Up6zWYBoe2+GKiQf8imnVLSSxMR7dlNd06ttuFD4B5baqFRcA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-07-28T11:23:40Z"
+    lastmodified: "2026-07-30T05:44:20Z"
-    mac: ENC[AES256_GCM,data:ahLy4sHoyGf4mwrK/90KoQQEnn+wNWxX80xHX+pj1SyonsgVe7Vul/dtXCti52zMRoiCF8NRx1eeQUeGnysFf4vyYRpPL/DuzGRC4+s6A3jb5Wz5wnKZeGdDJRFVcBxpBF1FYA2vQRuBe/fOz1XGzn5o4LjQfo2Hk0K21hidjJs=,iv:CDHXFYdqUDBXSMLrYx/azsMHPBtkU621zHC70jpUL8c=,tag:ntXkNKzomCXzCZq5uKKUeQ==,type:str]
+    mac: ENC[AES256_GCM,data:zpu938KpSjicnCJicrxSwV4sbdxRW02HnD7m1GXVSRH+j2Kuv3eKfB7JF4lBx339mnnOIasBZbcIgv5k/F7wX8EXGc7008AmM7Q5LwbBxPQSbIf3amGePoxL/pNw2i+h2MhWCNRadjdd1TJpGPFrFfTqI3FO0MLOacxAdxwvMF0=,iv:l5PLV2fOvukJFwPz5XitnGPICsIYC5JB9MxL1Ak+2q8=,tag:7cAhK9+R1xNIyEH9/6pCdA==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1