diff --git a/docs/bitcoin-pulse-postgresql.md b/docs/bitcoin-pulse-postgresql.md index 14189e5..b1946c3 100644 --- a/docs/bitcoin-pulse-postgresql.md +++ b/docs/bitcoin-pulse-postgresql.md @@ -13,8 +13,8 @@ - Host port 5432 is published only as `127.0.0.1:5432` and is not opened in the NixOS firewall. - Podman reports the service ready only after `pg_isready` succeeds. -- PostgreSQL receives `SIGINT` for smart shutdown, with 60 seconds for Podman - and 90 seconds for the enclosing systemd service. +- PostgreSQL receives `SIGINT` for a clean fast shutdown, with 60 seconds for + Podman and 90 seconds for the enclosing systemd service. - Automatic image updates are disabled. Update the digest only in a reviewed change after testing backups, startup, and application compatibility. @@ -46,7 +46,7 @@ container startup, the service stages private copies owned by PostgreSQL's uid inside the rootless user namespace. The copies are mounted read-only and are removed from the host runtime directory after PostgreSQL reports healthy and again when the unit stops. The application and migration SOPS files remain -available only through their dedicated host groups for the later BIT-28 units. +available only through the dedicated groups of their respective host services. ## Empty-database initialization and privileges @@ -67,6 +67,31 @@ objects, roles, or databases and must never run migrations. Initialization scripts run only for an empty volume. Changing a SOPS value does not change a role password in an existing PostgreSQL cluster. +## Migration-gated backend startup + +`bitcoin-pulse.service` requires `bitcoin-pulse-migrate.service`, and the +migration service requires the health-gated PostgreSQL container service. The +resulting startup order is: + +```text +PostgreSQL healthy -> bounded readiness check -> Migratus -> backend +``` + +The migration service waits up to 60 seconds for `pg_isready`, then runs the +same packaged backend version's `bitcoin-pulse-migrate migrate` command as the +unprivileged `bitcoin-pulse-migration` host user. It receives only the migration +role and its SOPS password-file path. The backend separately receives the +restricted `bitcoin_pulse_app` role and application password-file path. + +The migration unit is a one-shot service without `RemainAfterExit`, so every +new start transaction that pulls it in runs Migratus again. Migratus records +completed versions in `schema_migrations`; already-applied migrations therefore +make repeated starts harmless. A readiness timeout, invalid credential, or +migration error fails the one-shot unit. Because the backend is ordered after +and requires that unit, the backend process is not started after such a failure. +Inspect `journalctl -u bitcoin-pulse-migrate.service` for the bounded readiness +message, Migratus error, and systemd exit status. + ## Deployment and verification A human must review and merge the change, provision the encrypted keys above, @@ -97,9 +122,9 @@ SELECT has_schema_privilege( 'bitcoin_pulse_app', 'public', 'CREATE'); ``` -The expected results are `true`, `false`, and `false`. After BIT-28 runs the -foundation migration, also verify the application can perform only the DML -allowed by the migrated schema and that only the migration role can apply DDL. +The expected results are `true`, `false`, and `false`. Verify the application +can perform only the DML allowed by the migrated schema and that only the +migration role can apply DDL. To test recreation and persistence, create a temporary probe table as the administrator or migration role, restart the unit, confirm the row remains, and @@ -120,9 +145,41 @@ DROP TABLE bit29_recreation_probe; ``` A normal `systemctl stop` must complete within 90 seconds. Review the journal to -confirm a smart PostgreSQL shutdown and no forced kill. Start the unit again and -confirm it becomes healthy. Perform these production checks during a reviewed -maintenance window. +confirm a clean fast PostgreSQL shutdown and no forced kill. Start the unit again +and confirm it becomes healthy. Perform these production checks during a +reviewed maintenance window. + +After deploying the migration gate, verify the one-shot and backend: + +```console +systemctl is-active podman-bitcoin-pulse-postgres.service +systemctl is-active bitcoin-pulse.service +systemctl show bitcoin-pulse-migrate.service \ + --property=Result --property=ExecMainStatus +journalctl -u bitcoin-pulse-migrate.service -b --no-pager +ss -ltn '( sport = :3000 )' +``` + +A successful migration unit becomes inactive after it exits; `Result=success` +and `ExecMainStatus=0` are expected. The backend must be active and port 3000 +must be bound only to `127.0.0.1`. Restart `bitcoin-pulse.service` once and +confirm the migration unit runs successfully again without reapplying completed +migrations. + +Successful and repeated paths may be verified during deployment. Exercise the +database-unavailable and invalid-migration-credential paths only on a disposable +NixOS test host: both must make `bitcoin-pulse-migrate.service` fail, leave +`bitcoin-pulse.service` stopped, finish within the configured timeout, and emit +an actionable journal message. Never inject invalid credentials into production. + +## Rollback expectations + +A NixOS generation rollback changes the application package and service +configuration but does not reverse migrations in PostgreSQL. Do not use +`bitcoin-pulse-migrate rollback` as an automatic deployment action. Every schema +change must remain compatible with the previous application generation. Use an +expand/migrate/contract sequence for breaking changes, and perform any reviewed +manual rollback only after assessing data loss and application compatibility. ## Credential rotation for an existing database diff --git a/modules/bitcoin-pulse.nix b/modules/bitcoin-pulse.nix index 5dbe51d..0936541 100644 --- a/modules/bitcoin-pulse.nix +++ b/modules/bitcoin-pulse.nix @@ -2,6 +2,8 @@ let containerName = "bitcoin-pulse-postgres"; + databaseUrl = "jdbc:postgresql://127.0.0.1:5432/bitcoin_pulse?tcpKeepAlive=true"; + migrationServiceName = "bitcoin-pulse-migrate"; serviceName = "podman-${containerName}"; runtimeDirectory = "/run/${containerName}"; @@ -50,18 +52,58 @@ let ${lib.escapeShellArg "${runtimeDirectory}/migration-password"} ''; }; + + waitForPostgresql = pkgs.writeShellApplication { + name = "bitcoin-pulse-wait-for-postgresql"; + runtimeInputs = [ + pkgs.coreutils + pkgs.postgresql_18 + ]; + text = '' + deadline=$((SECONDS + 60)) + echo "Waiting up to 60 seconds for PostgreSQL on 127.0.0.1:5432" + + while (( SECONDS < deadline )); do + if pg_isready \ + --host=127.0.0.1 \ + --port=5432 \ + --username=bitcoin_pulse_migration \ + --dbname=bitcoin_pulse \ + --timeout=2 \ + --quiet; then + echo "PostgreSQL is accepting connections" + exit 0 + fi + + sleep 2 + done + + echo "PostgreSQL did not become ready on 127.0.0.1:5432 within 60 seconds" >&2 + pg_isready \ + --host=127.0.0.1 \ + --port=5432 \ + --username=bitcoin_pulse_migration \ + --dbname=bitcoin_pulse \ + --timeout=2 || true + exit 1 + ''; + }; in { imports = [ bitcoin-pulse.nixosModules.default ]; services.bitcoin-pulse = { - # BIT-28 will enable the production instance after PostgreSQL readiness - # and migrations have been integrated with these credentials. - enable = false; + enable = true; bindAddress = "127.0.0.1"; port = 3000; mempool.baseUrl = "http://127.0.0.1:8999"; + + database = { + url = databaseUrl; + username = "bitcoin_pulse_app"; + passwordFile = secretFiles.app; + }; }; users.groups = { @@ -70,20 +112,28 @@ in bitcoin-pulse-postgres.gid = 400; }; - users.users.bitcoin-pulse-postgres = { - description = "Rootless PostgreSQL container account for Bitcoin Pulse"; - isSystemUser = true; - uid = 400; - group = "bitcoin-pulse-postgres"; - extraGroups = [ - "bitcoin-pulse" - "bitcoin-pulse-migration" - ]; - home = "/var/lib/bitcoin-pulse-postgres"; - homeMode = "0700"; - createHome = true; - linger = true; - autoSubUidGidRange = true; + users.users = { + bitcoin-pulse-migration = { + description = "Bitcoin Pulse database migration account"; + isSystemUser = true; + group = "bitcoin-pulse-migration"; + }; + + bitcoin-pulse-postgres = { + description = "Rootless PostgreSQL container account for Bitcoin Pulse"; + isSystemUser = true; + uid = 400; + group = "bitcoin-pulse-postgres"; + extraGroups = [ + "bitcoin-pulse" + "bitcoin-pulse-migration" + ]; + home = "/var/lib/bitcoin-pulse-postgres"; + homeMode = "0700"; + createHome = true; + linger = true; + autoSubUidGidRange = true; + }; }; sops.secrets = { @@ -151,14 +201,68 @@ in }; }; - systemd.services.${serviceName}.serviceConfig = { - ExecStartPre = lib.mkAfter [ "${prepareSecrets}/bin/bitcoin-pulse-postgresql-prepare-secrets" ]; - ExecStartPost = [ "${cleanupSecrets}/bin/bitcoin-pulse-postgresql-cleanup-secrets" ]; - ExecStopPost = lib.mkAfter [ "${cleanupSecrets}/bin/bitcoin-pulse-postgresql-cleanup-secrets" ]; - RuntimeDirectoryMode = "0700"; - RestartSec = "5s"; - TimeoutStopSec = lib.mkForce "90s"; - UMask = "0077"; + systemd.services = { + ${serviceName}.serviceConfig = { + ExecStartPre = lib.mkAfter [ "${prepareSecrets}/bin/bitcoin-pulse-postgresql-prepare-secrets" ]; + ExecStartPost = [ "${cleanupSecrets}/bin/bitcoin-pulse-postgresql-cleanup-secrets" ]; + ExecStopPost = lib.mkAfter [ "${cleanupSecrets}/bin/bitcoin-pulse-postgresql-cleanup-secrets" ]; + RuntimeDirectoryMode = "0700"; + RestartSec = "5s"; + TimeoutStopSec = lib.mkForce "90s"; + UMask = "0077"; + }; + + ${migrationServiceName} = { + description = "Migrate the Bitcoin Pulse PostgreSQL database"; + requires = [ "${serviceName}.service" ]; + after = [ "${serviceName}.service" ]; + before = [ "bitcoin-pulse.service" ]; + + environment = { + MIGRATUS_DATABASE_URL = databaseUrl; + MIGRATUS_USERNAME = "bitcoin_pulse_migration"; + MIGRATUS_PASSWORD_FILE = secretFiles.migration; + }; + + serviceConfig = { + Type = "oneshot"; + ExecStartPre = [ "${waitForPostgresql}/bin/bitcoin-pulse-wait-for-postgresql" ]; + ExecStart = "${config.services.bitcoin-pulse.package}/bin/bitcoin-pulse-migrate migrate"; + User = "bitcoin-pulse-migration"; + Group = "bitcoin-pulse-migration"; + Restart = "no"; + TimeoutStartSec = "5min"; + + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + LockPersonality = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + UMask = "0077"; + }; + }; + + bitcoin-pulse = { + requires = [ "${migrationServiceName}.service" ]; + after = [ "${migrationServiceName}.service" ]; + }; }; environment.systemPackages = [ pkgs.postgresql_18 ];