feature: the router makes scheduled backups to a local folder, copied to the cloud with rclone, or to a Proxmox Backup Server, and vpn-router-backup restores them.

2026-10-25 20:18:00 +01:00
1 parent 7d1f639508
commit fe45272f7c
@@ -63,6 +63,9 @@ MANAGER_PORT   = 7080
 #: The password of the metrics service account, which setup makes and keeps (root only).
 METRICS_SECRET = pathlib.Path('/etc/vpn-router/metrics.secret')

+#: The schedule of the backup timer, written from backup.schedule.
+BACKUP_SCHEDULE = pathlib.Path('/etc/systemd/system/vpn-router-backup.timer.d/schedule.conf')
+
 #: The configuration of Caddy, the HTTPS proxy, which the package's drop-in for caddy.service names;
 #: Caddy runs as the user and group CADDY_GROUP, which the caddy package creates.
 CADDYFILE   = pathlib.Path('/etc/caddy/vpn-router.Caddyfile')
@@ -566,6 +569,27 @@ def manage_metrics(ctx):
    run(['systemctl', 'restart', unit], check=False)


+def manage_backup(cfg):
+    """Run the backup timer on backup.schedule while backups are enabled, and stop it when not.
+
+    The schedule is a drop-in of the timer, BACKUP_SCHEDULE, which replaces the OnCalendar of the shipped unit.
+    """
+    unit = 'vpn-router-backup.timer'
+    if not cfg['backup']['enabled']:
+        run(['systemctl', 'disable', '--now', unit], check=False)
+        if BACKUP_SCHEDULE.exists():
+            BACKUP_SCHEDULE.unlink()
+            run(['systemctl', 'daemon-reload'], check=False)
+        return
+    text = f'[Timer]\nOnCalendar=\nOnCalendar={cfg["backup"]["schedule"]}\n'
+    if not BACKUP_SCHEDULE.exists() or BACKUP_SCHEDULE.read_text(encoding='utf-8') != text:
+        BACKUP_SCHEDULE.parent.mkdir(parents=True, exist_ok=True)
+        BACKUP_SCHEDULE.write_text(text, encoding='utf-8')
+        run(['systemctl', 'daemon-reload'], check=False)
+        run(['systemctl', 'restart', unit], check=False)
+    run(['systemctl', 'enable', '--now', unit], check=False)
+
+
 def run_platform_setup(cfg, ctx):
    """Let the platform module apply its own state, if it defines any."""
    module = vpnrouter.platform_module(cfg)
@@ -734,6 +758,8 @@ def main():
    refresh_web(cfg, ctx)
    progress('running', 'metrics service')
    manage_metrics(ctx)
+    progress('running', 'backup timer')
+    manage_backup(cfg)
    if problem:
        progress('failed', failed_step, problem)
        print(f'vpn-router: error: {problem}', file=sys.stderr)
@@ -66,6 +66,11 @@ SCHEMA = {
                   'otlp_endpoint': '', 'otlp_username': '', 'otlp_password': '',
                   'keep_details': '24 hours', 'keep_hourly': '1 month', 'keep_daily': '3 months',
                   'keep_monthly': '1 year', 'keep_yearly': '10 years'},
+    'backup':     {'enabled': False, 'schedule': 'daily', 'target': 'local', 'path': '/var/backups/vpn-router',
+                   'keep_hourly': 24, 'keep_daily': 7, 'keep_weekly': 4, 'keep_monthly': 12, 'keep_yearly': 0,
+                   'metrics': False, 'rclone_remote': '', 'rclone_mode': 'sync',
+                   'pbs_repository': '', 'pbs_password': '', 'pbs_fingerprint': '', 'pbs_namespace': '',
+                   'pbs_keyfile': ''},
 }

 #: site_to_site maps a connection name to its settings: option -> default, the type of the default being the required type.
@@ -83,6 +88,30 @@ METRICS_KEEP = ('keep_details', 'keep_hourly', 'keep_daily', 'keep_monthly', 'ke
 #: A retention period as PostgreSQL reads an interval: a count and a unit.
 METRICS_INTERVAL = re.compile(r'^[1-9]\d{0,4} (hour|day|month|year)s?$')

+#: How often backups run, as systemd OnCalendar shorthands, from the most to the least often.
+BACKUP_SCHEDULES = ('hourly', 'daily', 'weekly')
+
+#: Where backups go: an archive in backup.path, or a Proxmox Backup Server.
+BACKUP_TARGETS = ('local', 'pbs')
+
+#: How rclone follows backup.path in the cloud: sync mirrors it, copy deletes nothing remotely.
+RCLONE_MODES = ('sync', 'copy')
+
+#: The retention settings of local backups, from the hourly to the yearly ones.
+BACKUP_KEEP = ('keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly', 'keep_yearly')
+
+#: A remote of rclone.conf and an optional path: remote:path.
+RCLONE_REMOTE = re.compile(r'^[\w.][\w. -]*:\S*$')
+
+#: A PBS repository: [[auth-id@]server[:port]:]datastore.
+PBS_REPOSITORY = re.compile(r'^(\S+:)?[A-Za-z0-9_][A-Za-z0-9._-]*$')
+
+#: The SHA-256 fingerprint of a server certificate, as PBS shows it.
+PBS_FINGERPRINT = re.compile(r'^([0-9A-Fa-f]{2}:){31}[0-9A-Fa-f]{2}$')
+
+#: A PBS namespace: up to seven names joined by /.
+PBS_NAMESPACE = re.compile(r'^[A-Za-z0-9_][A-Za-z0-9._-]*(/[A-Za-z0-9_][A-Za-z0-9._-]*){0,6}$')
+
 #: The port of Let's Encrypt's HTTP challenge, which Caddy answers while HTTPS is on.
 ACME_HTTP_PORT = 80

@@ -412,6 +441,46 @@ def validate(cfg):
            raise ConfigError(
                f'metrics.{option}: {keep!r} is not a count and a unit, such as "24 hours" or "3 months"')

+    schedule = _get(cfg, 'backup', 'schedule')
+    if schedule not in BACKUP_SCHEDULES:
+        raise ConfigError(
+            f'backup.schedule: {schedule!r} is not one of {", ".join(BACKUP_SCHEDULES)}')
+    target = _get(cfg, 'backup', 'target')
+    if target not in BACKUP_TARGETS:
+        raise ConfigError(f'backup.target: {target!r} is not one of {", ".join(BACKUP_TARGETS)}')
+    for option in ('path', 'pbs_keyfile'):
+        value = _get(cfg, 'backup', option)
+        if value and not value.startswith('/'):
+            raise ConfigError(f'backup.{option}: {value!r} is not an absolute path')
+    if not _get(cfg, 'backup', 'path'):
+        raise ConfigError('backup.path is required')
+    for option in BACKUP_KEEP:
+        if _get(cfg, 'backup', option) < 0:
+            raise ConfigError(f'backup.{option} must not be negative')
+    if target == 'local' and not any(_get(cfg, 'backup', option) for option in backup_keeps(schedule)):
+        raise ConfigError(
+            f'backup: with the {schedule} schedule, at least one of {", ".join(backup_keeps(schedule))} '
+            'is above 0')
+    remote = _get(cfg, 'backup', 'rclone_remote')
+    if remote and not RCLONE_REMOTE.match(remote):
+        raise ConfigError(f'backup.rclone_remote: {remote!r} is not a remote and a path, such as azure:backups/router')
+    rclone_mode = _get(cfg, 'backup', 'rclone_mode')
+    if rclone_mode not in RCLONE_MODES:
+        raise ConfigError(f'backup.rclone_mode: {rclone_mode!r} is not one of {", ".join(RCLONE_MODES)}')
+    if target == 'pbs':
+        repository = _get(cfg, 'backup', 'pbs_repository')
+        if not repository or not _get(cfg, 'backup', 'pbs_password'):
+            raise ConfigError('backup.pbs_repository and backup.pbs_password are required with the pbs target')
+        if not PBS_REPOSITORY.match(repository):
+            raise ConfigError(
+                f'backup.pbs_repository: {repository!r} is not of the form user@realm!token@host:datastore')
+    fingerprint = _get(cfg, 'backup', 'pbs_fingerprint')
+    if fingerprint and not PBS_FINGERPRINT.match(fingerprint):
+        raise ConfigError(f'backup.pbs_fingerprint: {fingerprint!r} is not 32 hexadecimal pairs joined by colons')
+    namespace = _get(cfg, 'backup', 'pbs_namespace')
+    if namespace and not PBS_NAMESPACE.match(namespace):
+        raise ConfigError(f'backup.pbs_namespace: {namespace!r} is not up to seven names joined by /')
+
    docker_mode = _get(cfg, 'docker', 'mode')
    if docker_mode not in DOCKER_MODES:
        raise ConfigError(
@@ -438,6 +507,11 @@ def validate(cfg):
            'a range counting as two')


+def backup_keeps(schedule):
+    """The retention settings of local backups that apply to the schedule: none finer than it."""
+    return BACKUP_KEEP[BACKUP_SCHEDULES.index(schedule):]
+
+
 def resolve_psk(connection, name):
    """Return the pre-shared key of the site_to_site connection: psk_file wins, else psk; '' when none is configured."""
    psk_file = connection['psk_file'].strip()
@@ -0,0 +1,412 @@
+"""The backups of the router, for vpn-router-backup and the web service.
+
+A backup is a consistent copy of the state that setup cannot make again, staged in STAGING: the
+trees of TREES, the files of site_to_site psk_file outside them under files/, the directory as
+slapcat writes it (ldap/vpn-router.ldif), with backup.metrics the database as pg_dump writes it
+(postgresql/vpn-router.dump), and manifest.json. The trees and the directory are copied while
+the configuration lock and the users lock are held, so the configuration, the PKI and the users
+match; pg_dump reads a snapshot of its own.
+
+backup.target local writes the staging directory as a tar archive compressed with zstd,
+ARCHIVE, to backup.path, removes the archives of this host that the retention settings of the
+schedule no longer keep, and with backup.rclone_remote syncs or copies the archives of this host
+to the remote of RCLONE_CONFIG. backup.target pbs sends the staging directory to a Proxmox Backup
+Server as the archive router.pxar of the snapshot host/<host>/<time>; the server keeps them.
+
+A run holds RUN_LOCK and records itself in STATE: result running, ok, warning (the backup is
+made, the cloud copy failed) or failed, with the times, the target, the archive or snapshot,
+its size, the error and the cloud copy. restore() puts a backup back and runs setup.
+"""
+
+import contextlib
+import datetime
+import fcntl
+import json
+import os
+import pathlib
+import re
+import shutil
+import socket
+import subprocess
+import tempfile
+import time
+
+import vpnrouter
+import vpnrouter_auth
+import vpnrouter_ldap
+import vpnrouter_users
+
+STATE = pathlib.Path('/var/lib/vpn-router/backup.json')
+WORK_DIR = pathlib.Path('/var/lib/vpn-router/backup')
+STAGING = WORK_DIR / 'staging'
+RESTORE_DIR = WORK_DIR / 'restore'
+#: What a restore replaces is kept here, in a directory named after the time of the restore.
+BEFORE_RESTORE = WORK_DIR / 'before-restore'
+RUN_LOCK = pathlib.Path('/run/vpn-router/backup.lock')
+RCLONE_CONFIG = pathlib.Path('/etc/vpn-router/rclone.conf')
+PBS_CLIENT = pathlib.Path('/usr/bin/proxmox-backup-client')
+UNIT = 'vpn-router-backup.service'
+TIMER = 'vpn-router-backup.timer'
+
+#: The state a backup copies: directories and files, each where it is on the system.
+TREES = ('/etc/vpn-router', '/etc/wireguard', '/etc/ufw', '/etc/default/ufw', '/etc/vpn-router-manager')
+
+#: The name of a local archive: the host and the UTC time of the backup.
+ARCHIVE = re.compile(r'^vpn-router-(?P<host>[A-Za-z0-9][A-Za-z0-9.-]*)-(?P<stamp>\d{8}T\d{6}Z)\.tar\.zst$')
+STAMP = '%Y%m%dT%H%M%SZ'
+
+LDIF = 'ldap/vpn-router.ldif'
+DUMP = 'postgresql/vpn-router.dump'
+PBS_ARCHIVE = 'router.pxar'
+
+#: The period of each retention setting, as strftime formats of local time; weeks are ISO weeks.
+PERIODS = {'keep_hourly': '%Y-%m-%d %H', 'keep_daily': '%Y-%m-%d', 'keep_weekly': '%G-%V', 'keep_monthly': '%Y-%m',
+           'keep_yearly': '%Y'}
+
+#: The units a restore stops while it replaces the directory and the database, and the ones it restarts after setup.
+RESTORE_STOPS = ('vpn-router-web.service', 'vpn-router-metrics.service', 'slapd.service')
+RESTORE_RESTARTS = ('wg-quick@wg0.service', 'openvpn-server@vpn-router.service', 'strongswan.service',
+                    'vpn-router-manager.service')
+
+
+class Failure(Exception):
+    """A backup or a restore that cannot be done."""
+
+
+def now():
+    """The local time in ISO 8601, to the second, with its offset."""
+    return datetime.datetime.now().astimezone().isoformat(timespec='seconds')
+
+
+def host():
+    """The name of this host, which names its archives and its PBS backup group."""
+    return socket.gethostname().split('.')[0]
+
+
+def command(args, **kwargs):
+    """Run a command; Failure with its standard error when it fails."""
+    try:
+        result = subprocess.run(args, capture_output=True, text=True, **kwargs)
+    except OSError as exc:
+        raise Failure(f'{args[0]}: {exc.strerror}')
+    if result.returncode != 0:
+        raise Failure(f'{" ".join(args[:2])} failed: {result.stderr.strip() or result.stdout.strip()}')
+    return result.stdout
+
+
+def write_state(state):
+    """Write STATE in one step."""
+    STATE.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, name = tempfile.mkstemp(dir=STATE.parent, prefix='.backup.json.')
+    with os.fdopen(descriptor, 'w', encoding='utf-8') as handle:
+        handle.write(json.dumps(state, indent=2) + '\n')
+    os.chmod(name, 0o600)
+    os.replace(name, STATE)
+
+
+def running():
+    """Whether a run holds RUN_LOCK."""
+    try:
+        with open(RUN_LOCK, encoding='utf-8') as handle:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except FileNotFoundError:
+        return False
+    except BlockingIOError:
+        return True
+    return False
+
+
+def state():
+    """The state of the last run, {} when there was none; a run that ended without recording it is interrupted."""
+    try:
+        recorded = json.loads(STATE.read_text(encoding='utf-8'))
+    except (OSError, ValueError):
+        return {}
+    if recorded.get('result') == 'running' and not running():
+        recorded['result'] = 'interrupted'
+    return recorded
+
+
+@contextlib.contextmanager
+def run_lock():
+    """Hold RUN_LOCK; Failure when another run holds it."""
+    RUN_LOCK.parent.mkdir(parents=True, exist_ok=True)
+    with open(RUN_LOCK, 'a', encoding='utf-8') as handle:
+        try:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise Failure('another backup or restore is running')
+        yield
+
+
+def archives(path):
+    """The archives of this host in path, newest first, as [{name, time, size}]; time is the UTC time of the backup."""
+    found = []
+    try:
+        entries = list(os.scandir(path))
+    except OSError:
+        return found
+    for entry in entries:
+        match = ARCHIVE.match(entry.name)
+        if match and match['host'] == host() and entry.is_file():
+            when = datetime.datetime.strptime(match['stamp'], STAMP).replace(tzinfo=datetime.timezone.utc)
+            found.append({'name': entry.name, 'time': when.isoformat(), 'size': entry.stat().st_size})
+    return sorted(found, key=lambda item: item['time'], reverse=True)
+
+
+def kept(found, keeps):
+    """The names of the archives the retention settings keep, by the rules of Proxmox Backup Server's prune.
+
+    found is newest first. The settings apply in the order of PERIODS: each keeps the newest archive of each of its
+    last N periods that have archives and no archive a former setting kept, and the older archives of those periods
+    are removed, so no later setting keeps them.
+    """
+    marks = {}
+    for option, period in PERIODS.items():
+        count = keeps.get(option, 0)
+        if not count:
+            continue
+        periods = {moment(item).strftime(period) for item in found if marks.get(item['name']) == 'keep'}
+        included = set()
+        for item in found:
+            if item['name'] in marks:
+                continue
+            key = moment(item).strftime(period)
+            if key in periods:
+                continue
+            if key not in included:
+                if len(included) >= count:
+                    break
+                included.add(key)
+                marks[item['name']] = 'keep'
+            else:
+                marks[item['name']] = 'remove'
+    return {name for name, mark in marks.items() if mark == 'keep'}
+
+
+def moment(item):
+    """The local time of an archive of archives()."""
+    return datetime.datetime.fromisoformat(item['time']).astimezone()
+
+
+def psk_files(cfg):
+    """The psk_file of each site_to_site connection outside the trees of TREES."""
+    files = []
+    for connection in cfg['site_to_site'].values():
+        name = connection['psk_file'].strip()
+        if name and not any(name == tree or name.startswith(tree + '/') for tree in TREES) and os.path.isfile(name):
+            files.append(name)
+    return files
+
+
+def database_args(cfg):
+    """The command prefix and the database argument of pg_dump and pg_restore: postgres and vpn-router, or web.database."""
+    configured = cfg['web']['database'].strip()
+    if configured:
+        return [], [f'--dbname={configured}']
+    return ['runuser', '-u', 'postgres', '--'], [f'--dbname={vpnrouter_auth.DATABASE}']
+
+
+def stage(cfg):
+    """Copy what a backup holds to STAGING, emptied first, and return the manifest."""
+    shutil.rmtree(STAGING, ignore_errors=True)
+    STAGING.mkdir(parents=True, mode=0o700)
+    trees = [tree for tree in TREES if os.path.exists(tree)]
+    files = psk_files(cfg)
+    vpnrouter_users.LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with vpnrouter.config_lock(), open(vpnrouter_users.LOCK_FILE, 'w') as users:
+        fcntl.flock(users, fcntl.LOCK_EX)
+        command(['cp', '-a', '--parents', *trees, str(STAGING)])
+        if files:
+            (STAGING / 'files').mkdir()
+            command(['cp', '-a', '--parents', *files, str(STAGING / 'files')])
+        (STAGING / LDIF).parent.mkdir()
+        command(['slapcat', '-b', vpnrouter_ldap.BASE, '-l', str(STAGING / LDIF)])
+    items = trees + [f'files{name}' for name in files] + [LDIF]
+    if cfg['backup']['metrics']:
+        (STAGING / DUMP).parent.mkdir()
+        prefix, database = database_args(cfg)
+        with open(STAGING / DUMP, 'wb') as dump:
+            result = subprocess.run([*prefix, 'pg_dump', '-Fc', *database], stdout=dump, stderr=subprocess.PIPE)
+        if result.returncode != 0:
+            raise Failure(f'pg_dump failed: {result.stderr.decode(errors="replace").strip()}')
+        items.append(DUMP)
+    version = subprocess.run(['dpkg-query', '-W', '-f=${Version}', 'vpn-router'], capture_output=True, text=True)
+    manifest = {'host': host(), 'time': now(), 'version': version.stdout.strip() or None,
+                'schedule': cfg['backup']['schedule'], 'items': items}
+    (STAGING / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
+    return manifest
+
+
+def pbs_environment(cfg):
+    """The environment of proxmox-backup-client: the repository, the token secret and the fingerprint of backup."""
+    if not PBS_CLIENT.exists():
+        raise Failure(f'{PBS_CLIENT} does not exist; install proxmox-backup-client-static')
+    backup = cfg['backup']
+    environment = dict(os.environ, PBS_REPOSITORY=backup['pbs_repository'], PBS_PASSWORD=backup['pbs_password'])
+    if backup['pbs_fingerprint']:
+        environment['PBS_FINGERPRINT'] = backup['pbs_fingerprint']
+    return environment
+
+
+def pbs_options(cfg):
+    """The namespace and key file options of proxmox-backup-client from backup."""
+    options = []
+    if cfg['backup']['pbs_namespace']:
+        options += ['--ns', cfg['backup']['pbs_namespace']]
+    if cfg['backup']['pbs_keyfile']:
+        options += ['--keyfile', cfg['backup']['pbs_keyfile']]
+    return options
+
+
+def cloud(cfg):
+    """Sync or copy the archives of this host in backup.path to backup.rclone_remote; Failure when it fails."""
+    backup = cfg['backup']
+    rclone = shutil.which('rclone')
+    if not rclone:
+        raise Failure('rclone is not installed')
+    if not RCLONE_CONFIG.is_file():
+        raise Failure(f'{RCLONE_CONFIG} does not exist; make the remote with rclone config --config {RCLONE_CONFIG}')
+    command([rclone, backup['rclone_mode'], backup['path'], backup['rclone_remote'], '--config', str(RCLONE_CONFIG),
+             '--include', f'vpn-router-{host()}-*.tar.zst'])
+
+
+def run():
+    """Make a backup to backup.target as the configuration says, recording it in STATE; the recorded state."""
+    cfg = vpnrouter.load_config()
+    backup = cfg['backup']
+    started = time.time()
+    record = {'result': 'running', 'started': now(), 'finished': None, 'target': backup['target'], 'archive': None,
+              'snapshot': None, 'size': None, 'removed': [], 'error': None, 'cloud': None}
+    with run_lock():
+        write_state(record)
+        try:
+            stage(cfg)
+            if backup['target'] == 'pbs':
+                epoch = int(started)
+                command([str(PBS_CLIENT), 'backup', f'{PBS_ARCHIVE}:{STAGING}', '--backup-type', 'host',
+                         '--backup-id', host(), '--backup-time', str(epoch), *pbs_options(cfg)],
+                        env=pbs_environment(cfg))
+                stamp = datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+                record['snapshot'] = f'host/{host()}/{stamp}'
+                record['size'] = sum(path.stat().st_size for path in STAGING.rglob('*') if path.is_file())
+            else:
+                path = pathlib.Path(backup['path'])
+                path.mkdir(parents=True, mode=0o700, exist_ok=True)
+                name = f'vpn-router-{host()}-{time.strftime(STAMP, time.gmtime(started))}.tar.zst'
+                partial = path / f'{name}.partial'
+                command(['tar', '--zstd', '-cf', str(partial), '-C', str(STAGING), '.'])
+                partial.chmod(0o600)
+                partial.rename(path / name)
+                record['archive'] = name
+                record['size'] = (path / name).stat().st_size
+                found = archives(path)
+                keep = kept(found, {option: backup[option] for option in vpnrouter.backup_keeps(backup['schedule'])})
+                for item in found:
+                    if item['name'] not in keep:
+                        (path / item['name']).unlink()
+                        record['removed'].append(item['name'])
+                if backup['rclone_remote']:
+                    record['cloud'] = {'remote': backup['rclone_remote'], 'mode': backup['rclone_mode'],
+                                       'result': 'ok', 'error': None}
+                    try:
+                        cloud(cfg)
+                    except Failure as exc:
+                        record['cloud'].update(result='failed', error=str(exc))
+            record['result'] = 'warning' if record['cloud'] and record['cloud']['result'] != 'ok' else 'ok'
+        except (Failure, OSError) as exc:
+            record['result'] = 'failed'
+            record['error'] = str(exc)
+        finally:
+            shutil.rmtree(STAGING, ignore_errors=True)
+            record['finished'] = now()
+            write_state(record)
+    return record
+
+
+def snapshots(cfg):
+    """The PBS snapshots of this host, as proxmox-backup-client lists them."""
+    namespace = cfg['backup']['pbs_namespace']
+    return command([str(PBS_CLIENT), 'snapshot', 'list', f'host/{host()}', *(['--ns', namespace] if namespace else [])],
+                   env=pbs_environment(cfg))
+
+
+def extract(cfg, source, pbs):
+    """Unpack a local archive, or with pbs fetch a snapshot, to RESTORE_DIR, emptied first; the manifest."""
+    shutil.rmtree(RESTORE_DIR, ignore_errors=True)
+    RESTORE_DIR.mkdir(parents=True, mode=0o700)
+    if pbs:
+        command([str(PBS_CLIENT), 'restore', source, PBS_ARCHIVE, str(RESTORE_DIR), *pbs_options(cfg)],
+                env=pbs_environment(cfg))
+    else:
+        command(['tar', '--zstd', '-xf', source, '-C', str(RESTORE_DIR)])
+    try:
+        return json.loads((RESTORE_DIR / 'manifest.json').read_text(encoding='utf-8'))
+    except (OSError, ValueError):
+        raise Failure(f'{source} is not a backup of vpn-router-backup: it has no manifest.json')
+
+
+def put_back(relative, target, saved):
+    """Move target to saved, keeping its path, and copy the staged relative path of RESTORE_DIR in its place."""
+    target = pathlib.Path(target)
+    if target.exists() or target.is_symlink():
+        keep = saved / target.relative_to('/')
+        keep.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(target, keep)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    command(['cp', '-a', str(RESTORE_DIR / relative), str(target)])
+
+
+def restore(source, pbs, confirm):
+    """Put the backup back: a local archive, or with pbs a snapshot; confirm(manifest) is asked before any change.
+
+    What is replaced is kept in BEFORE_RESTORE. The services of the directory and the database are stopped while they
+    are replaced, and setup applies the configuration afterwards. Returns the directory of what was replaced, or None
+    when confirm declined.
+    """
+    cfg = vpnrouter.load_config()
+    with run_lock():
+        try:
+            manifest = extract(cfg, source, pbs)
+            if not confirm(manifest):
+                return None
+            saved = BEFORE_RESTORE / time.strftime(STAMP, time.gmtime())
+            saved.mkdir(parents=True, mode=0o700)
+            for unit in RESTORE_STOPS:
+                subprocess.run(['systemctl', 'stop', unit], capture_output=True)
+            for item in manifest['items']:
+                if item.startswith('/'):
+                    put_back(item.lstrip('/'), item, saved)
+                elif item.startswith('files/'):
+                    put_back(item, item[len('files'):], saved)
+            if LDIF in manifest['items']:
+                directory = vpnrouter_ldap.DB_DIRECTORY
+                if directory.exists():
+                    shutil.move(directory, saved / 'ldap')
+                directory.mkdir(mode=0o700)
+                command(['slapadd', '-b', vpnrouter_ldap.BASE, '-l', str(RESTORE_DIR / LDIF)])
+                command(['chown', '-R', 'openldap:openldap', str(directory)])
+            subprocess.run(['systemctl', 'start', 'slapd.service'], capture_output=True)
+            restored = vpnrouter.load_config()
+            if not restored['web']['database'].strip():
+                secret = vpnrouter_auth.DATABASE_SECRET
+                if secret.is_file() and vpnrouter_auth.psql("SELECT 1 FROM pg_roles WHERE rolname = :'v1'",
+                                                            vpnrouter_auth.DATABASE):
+                    password = secret.read_text(encoding='utf-8').strip()
+                    vpnrouter_auth.psql(f"ALTER ROLE :\"v1\" PASSWORD '{password}'", vpnrouter_auth.DATABASE)
+                vpnrouter_auth.provision(restored)
+            if DUMP in manifest['items']:
+                prefix, database = database_args(restored)
+                with open(RESTORE_DIR / DUMP, 'rb') as dump:
+                    result = subprocess.run([*prefix, 'pg_restore', '--clean', '--if-exists', *database], stdin=dump,
+                                            stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+                if result.returncode != 0:
+                    raise Failure(f'pg_restore failed: {result.stderr.decode(errors="replace").strip()}')
+            subprocess.run(['systemctl', 'start', 'vpn-router-web.service'], capture_output=True)
+            command(['systemctl', 'restart', 'vpn-router-setup.service'])
+            for unit in RESTORE_RESTARTS:
+                subprocess.run(['systemctl', 'try-restart', unit], capture_output=True)
+            subprocess.run(['ufw', 'reload'], capture_output=True)
+            return saved
+        finally:
+            shutil.rmtree(RESTORE_DIR, ignore_errors=True)
@@ -24,7 +24,9 @@ Pages:
  /site-to-site
           the site-to-site connections of the configuration: list, add, change and remove,
           applied by vpn-router-setup (see site_to_site())
-  /auth    sign-in with a user's password and TOTP code, refresh and sign-out (see auth())
+  /backup  the backup settings, the last run and the local archives: show, change and run
+           now (see backup())
+  /auth   sign-in with a user's password and TOTP code, refresh and sign-out (see auth())
  /metrics/history
           the metrics the metrics service keeps, for a view: hour, day, month or year (see
           metrics_history())
@@ -32,7 +34,7 @@ Pages:
           the databases of each PostgreSQL cluster on the router with their sizes, as JSON (see
           postgresql_databases()); /config has the clusters' settings

-/config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site and /account need an HS256 JWT in an Authorization: Bearer header signed
+/config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup and /account need an HS256 JWT in an Authorization: Bearer header signed
 with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub;
 /account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A
 token of vpn-router-token has the admin claim and names no user; it may call every path but
@@ -67,8 +69,10 @@ the router has no features, and /config shows no configuration or interfaces.
 """

 import collections
+import configparser
 import json
 import os
+import shutil
 import signal
 import sys
 import threading
@@ -83,6 +87,7 @@ import ldap

 import vpnrouter
 import vpnrouter_auth
+import vpnrouter_backup
 import vpnrouter_metrics
 import vpnrouter_net
 import vpnrouter_serve
@@ -676,6 +681,82 @@ def site_to_site(method, path, body):
    return (201 if method == 'POST' else 200), connection_of(name, cfg['site_to_site'][name], None)


+def backup_settings(settings):
+    """The backup section as /backup shows it: its settings but pbs_password, with pbs_password_set."""
+    return {**{option: value for option, value in settings.items() if option != 'pbs_password'},
+            'pbs_password_set': bool(settings['pbs_password'])}
+
+
+def rclone_remotes():
+    """The names of the remotes of vpnrouter_backup.RCLONE_CONFIG; [] when it cannot be read, as when it is encrypted."""
+    parser = configparser.ConfigParser(interpolation=None)
+    try:
+        parser.read(vpnrouter_backup.RCLONE_CONFIG, encoding='utf-8')
+    except (OSError, configparser.Error):
+        return []
+    return parser.sections()
+
+
+def backup(method, path, body):
+    """Answer a /backup request: the HTTP status and the JSON document.
+
+      GET  /backup      {"settings", "state", "archives", "next", "client", "rclone"}: the backup section without
+                        pbs_password, with pbs_password_set; the last run (vpnrouter_backup.state), null before the
+                        first; the local archives of this host, newest first, [{name, time, size}], [] with the pbs
+                        target; the next run of the timer as systemd shows it, null while backups are disabled;
+                        whether proxmox-backup-client is installed; and {"installed", "config", "remotes"} of rclone
+      PUT  /backup      change any of the settings of the backup section; an empty or absent pbs_password keeps the
+                        current secret
+      POST /backup/run  start a backup now, without waiting for it; 409 while one runs
+
+    A change is made as /site-to-site makes one, and applied by restarting vpn-router-setup without waiting for it.
+    """
+    if body == 'invalid' or (body is not None and not isinstance(body, dict)):
+        return 400, {'error': 'the body must be a JSON object'}
+    body = body or {}
+    if (method, path) == ('POST', '/backup/run'):
+        if vpnrouter_backup.running():
+            return 409, {'error': 'a backup is running'}
+        result = run('systemctl', 'start', '--no-block', vpnrouter_backup.UNIT)
+        if result is None or result.returncode != 0:
+            return 500, {'error': f'the backup cannot be started: {result.stderr.strip() if result else "no systemctl"}'}
+        return 202, {'started': True}
+    if (method, path) not in (('GET', '/backup'), ('PUT', '/backup')):
+        return 404, {'error': f'no route {method} {path}'}
+    try:
+        if method == 'GET':
+            settings = vpnrouter.load_config()['backup']
+            timer = run('systemctl', 'show', '-p', 'NextElapseUSecRealtime', '--value', vpnrouter_backup.TIMER)
+            next_run = timer.stdout.strip() if timer and timer.returncode == 0 else ''
+            return 200, {
+                'settings': backup_settings(settings),
+                'state': vpnrouter_backup.state() or None,
+                'archives': vpnrouter_backup.archives(settings['path']) if settings['target'] == 'local' else [],
+                'next': next_run if settings['enabled'] and next_run not in ('', 'n/a') else None,
+                'client': vpnrouter_backup.PBS_CLIENT.exists(),
+                'rclone': {'installed': shutil.which('rclone') is not None,
+                           'config': vpnrouter_backup.RCLONE_CONFIG.is_file(), 'remotes': rclone_remotes()},
+            }
+        settings = {option: body[option] for option in vpnrouter.SCHEMA['backup'] if option in body}
+        if not settings.get('pbs_password'):
+            settings.pop('pbs_password', None)
+        with vpnrouter.config_lock():
+            data = vpnrouter.read_settings()
+            current = data.get('backup')
+            data['backup'] = {**(current if isinstance(current, dict) else {}), **settings}
+            cfg = vpnrouter.with_defaults(data)
+            vpnrouter.validate(cfg)
+            vpnrouter.save_config(data)
+    except vpnrouter.ConfigError as exc:
+        return 400, {'error': str(exc)}
+    except OSError as exc:
+        return 500, {'error': f'the configuration cannot be written: {exc}'}
+    except SystemExit as exc:
+        return 500, {'error': str(exc.code)}
+    run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
+    return 200, {'settings': backup_settings(cfg['backup'])}
+
+
 def account(name, method, path, body):
    """Answer an /account request of the user named by the token with vpnrouter_users: the HTTP status and the JSON document.

@@ -864,7 +945,7 @@ class Handler(BaseHTTPRequestHandler):
        # A body that is not read must not be taken for the next request.
        if self.command not in ('GET', 'HEAD'):
            self.close_connection = True
-        directory = next((name for name in ('users', 'service-accounts', 'groups', 'account', 'site-to-site')
+        directory = next((name for name in ('users', 'service-accounts', 'groups', 'account', 'site-to-site', 'backup')
                          if path == f'/{name}' or path.startswith(f'/{name}/')), None)
        protected = path in ('/config', '/metrics/history', '/postgresql') or directory is not None
        signing = path in ('/auth', '/auth/refresh', '/auth/logout')
@@ -905,6 +986,9 @@ class Handler(BaseHTTPRequestHandler):
        elif directory == 'site-to-site':
            code, document = site_to_site(self.command, path, payload)
            body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
+        elif directory == 'backup':
+            code, document = backup(self.command, path, payload)
+            body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
        elif directory == 'service-accounts':
            code, document = service_accounts(self.command, path, payload)
            body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'