From 1eac15e640f80236ad7cfac291ea54205411633f Mon Sep 17 00:00:00 2001 From: Slawomir Koszewski Date: Thu, 29 Oct 2026 19:24:24 +0100 Subject: [PATCH] feature: the router serves S3-compatible object storage with versitygw, with one S3 account per user and buckets owned by users. --- ADR.md | 41 ++ README.md | 83 ++++ debian-package/debian/changelog | 21 + debian-package/debian/control | 6 +- debian-package/debian/install | 2 + debian-package/debian/rules | 1 + .../debian/vpn-router.vpn-router-s3.service | 22 + .../src/usr/lib/vpn-router/install-packages | 72 ++- debian-package/src/usr/lib/vpn-router/setup | 39 +- .../src/usr/lib/vpn-router/vpnrouter.py | 22 +- .../usr/lib/vpn-router/vpnrouter_backup.py | 14 +- .../src/usr/lib/vpn-router/vpnrouter_s3.py | 446 ++++++++++++++++++ .../lib/vpn-router/vpnrouter_web/__init__.py | 5 +- .../lib/vpn-router/vpnrouter_web/handler.py | 9 +- .../vpn-router/vpnrouter_web/identities.py | 10 +- .../usr/lib/vpn-router/vpnrouter_web/s3.py | 118 +++++ debian-package/src/usr/sbin/vpn-router-s3 | 137 ++++++ debian-package/src/usr/sbin/vpn-router-user | 3 + .../completions/vpn-router-s3.bash | 84 ++++ .../share/vpn-router/templates/Caddyfile.j2 | 5 + 20 files changed, 1115 insertions(+), 25 deletions(-) create mode 100644 debian-package/debian/vpn-router.vpn-router-s3.service create mode 100644 debian-package/src/usr/lib/vpn-router/vpnrouter_s3.py create mode 100644 debian-package/src/usr/lib/vpn-router/vpnrouter_web/s3.py create mode 100755 debian-package/src/usr/sbin/vpn-router-s3 create mode 100644 debian-package/src/usr/share/bash-completion/completions/vpn-router-s3.bash diff --git a/ADR.md b/ADR.md index 03a85d6..9d5c8a9 100644 --- a/ADR.md +++ b/ADR.md @@ -1278,3 +1278,44 @@ method. Consequences: A change to one family of paths touches one module and the routing in `handler`. State shared between requests, such as the open shells, lives in the module that owns it; other modules change it and never rebind it. + +## ADR-48: Object storage is versitygw with its POSIX backend, one S3 account per user + +Status: Accepted + +Context: The router's users and services want S3-compatible storage for backups of other +systems, Terraform state and artifacts. Ubuntu 24.04 has no S3 server; its rclone 1.60 predates +`rclone serve s3`. versitygw (Apache-2.0) is an S3 gateway that keeps buckets as plain folders +(POSIX backend) and has its own IAM. It publishes `.deb` files for amd64 and arm64, and an Admin +API, signed like S3, to manage accounts and bucket owners. Its LDAP IAM needs each secret stored in +plain text in the directory. S3 signs with a shared secret, so the directory's passwords cannot +serve as S3 credentials. + +Decision: + +- **Installation:** versitygw is pinned to a version and the SHA-256 of each architecture's + `.deb` (`vpnrouter_s3`). It is downloaded and installed on demand by + `vpn-router-install@s3.service`. +- **Service:** `vpn-router-s3.service` runs it with the internal IAM in + `/var/lib/vpn-router/s3/iam` and the buckets in `/var/lib/vpn-router/s3/data`. Its S3 API and + its Admin API are on loopback ports, and Caddy serves the S3 API at + `https://:` with path-style addresses, which need no wildcard name. +- **Root keys:** setup makes them once, in `/etc/vpn-router/s3.env`. +- **Calls:** the router calls the Admin API and S3's DeleteBucket with its own Signature Version 4 + signer, from the standard library, so no secret appears on a command line. +- **Accounts:** + - Each user or service account has at most one S3 account, of the role `user`, so a bucket's + owner is a user. + - `/etc/vpn-router/s3-accounts.json` records the owner, the key and the secret. + - The account follows its user: an unknown secret while the user is disabled, its own secret + when the user is enabled, and removal with the user. + +Consequences: + +- **Accounts:** a user has one key pair; replacing its secret replaces it for every client. +- **Secrets:** they are kept by the router in two root-only files, versitygw's IAM and the + record, as S3 needs them to check signatures. +- **Ownership:** a removed user's buckets keep the account as their owner until an admin gives + them to another user. +- **Backups and upgrades:** the buckets are in backups only with `backup.s3`. A new versitygw + comes with a new vpn-router release, not from apt. diff --git a/README.md b/README.md index 7783e74..759ada7 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,9 @@ must have the type shown below; an unknown section or setting is an error. | `sites` | `upload_max_mb` | integer | `512` | The largest upload of a site's file or zip archive, in MiB, 1 to 10240 | | `sites` | `webdav` | boolean | `false` | WebDAV of the sites' folders, which needs `https.enabled` and `wan.local_fqdn` (see WebDAV) | | `sites` | `webdav_port` | integer | `7445` | HTTPS port of WebDAV | +| `s3` | `enabled` | boolean | `false` | S3-compatible object storage, which needs `https.enabled` and `wan.local_fqdn` (see Object storage) | +| `s3` | `port` | integer | `7447` | HTTPS port of the object storage | +| `s3` | `region` | string | `us-east-1` | The region S3 clients sign their requests with | | `dns` | `enabled` | boolean | `false` | The primary name server of the router's domains, which needs `dns.nameserver` and `dns.addresses` (see Name server) | | `dns` | `nameserver` | string | | The name server's host name, the primary NS of every domain | | `dns` | `addresses` | list | | The public addresses of the name server, the domains' apexes and the sites' host names | @@ -188,6 +191,7 @@ must have the type shown below; an unknown section or setting is an error. | `backup` | `keep_monthly` | integer | `12` | Monthly local archives kept | | `backup` | `keep_yearly` | integer | `0` | Yearly local archives kept | | `backup` | `metrics` | boolean | `false` | The backup holds the database of the metrics and the refresh tokens | +| `backup` | `s3` | boolean | `false` | The backup holds the buckets of the object storage too; its accounts are always held | | `backup` | `rclone_remote` | string | | `remote:path` of `/etc/vpn-router/rclone.conf` the local archives are copied to | | `backup` | `rclone_mode` | string | `sync` | `sync` mirrors the archives, `copy` deletes nothing in the cloud | | `backup` | `pbs_repository` | string | | `user@realm!token@host[:port]:datastore` | @@ -1674,6 +1678,85 @@ vpn-config-edit set dns.addresses 203.0.113.10 vpn-config-edit set --restart dns.enabled true ``` +## Object storage + +While `s3.enabled` is `true` and HTTPS is ready, the router serves S3-compatible object storage +at `https://:` (7447 by default), with path-style addresses, for clients +such as `aws s3`, rclone, restic or Terraform's `s3` backend. The firewall opens the port where it +opens the other HTTPS ports. + +```sh +vpn-config-edit set --restart s3.enabled true +``` + +**The service** + +- **The server:** versitygw, pinned to one release, which a release of vpn-router raises. + `vpn-router-install@s3.service` downloads its `.deb` for amd64 or arm64 from GitHub, checks its + SHA-256 and installs it with apt-get, when another version or none is installed. +- **How it runs:** `vpn-router-s3.service` runs it as root, sandboxed. Its S3 API is on + `127.0.0.1:7448`, which Caddy serves, and its Admin API on `127.0.0.1:7449`. +- **Storage:** with the POSIX backend, each folder of `/var/lib/vpn-router/s3/data` is a bucket + and the files below it are its objects. +- **Settings:** `/etc/vpn-router/s3.env` (0600, written by setup) holds the root keys, the ports, + the IAM folder `/var/lib/vpn-router/s3/iam` and `s3.region`, which clients sign with. + +**Accounts** + +- **One account per user:** a user or service account of the directory has at most one S3 + account, an access key and its secret. +- **The secret is shown once,** when the account is made or its secret replaced. + `/etc/vpn-router/s3-accounts.json` (0600) records the accounts with their owners. +- **Following the user:** + - while the user is disabled, the account's secret is replaced by an unknown one; + - once the user is enabled, the account has its own secret again; + - removing the user removes the account. + + Setup brings every account in line at each run. + +**Buckets** + +- A bucket belongs to one account, which alone may use it. +- Buckets are made, given to another user and removed (when empty) by an admin. +- A user's buckets keep the account as their owner after the account is removed, until another + owner is chosen. + +```sh +vpn-router-s3 account add alice # prints the access key and the secret once +vpn-router-s3 bucket add tfstate --owner alice +aws --endpoint-url https://router.example.com:7447 s3 ls s3://tfstate +``` + +| Command | What it does | +| --- | --- | +| `vpn-router-s3 status` | Whether storage is on, the installed and the pinned versitygw, whether it runs, the endpoint and the region | +| `vpn-router-s3 account list [--owner U]` | The accounts with their owners and the time each was made | +| `vpn-router-s3 account add\|rotate\|remove U` | Make the user's account, replace its secret, or remove it | +| `vpn-router-s3 bucket list` | The buckets with their owners, sizes and numbers of objects | +| `vpn-router-s3 bucket add NAME --owner U` | Make a bucket of the user's account | +| `vpn-router-s3 bucket owner NAME U` | Give the bucket to another user's account; versitygw resets its ACL and policy | +| `vpn-router-s3 bucket remove NAME` | Remove the bucket, which must be empty | + +`vpn-router-s3` answers JSON with `--json`, as `vpn-router-site` does. + +**Web service API** (`/s3` for admins, `/account/s3` for the signed-in user) + +| Request | Body | Answer | +| --- | --- | --- | +| `GET /s3` | | `{"settings", "endpoint", "version", "installed", "running", "buckets", "accounts", "problem"}` | +| `PUT /s3` | any of the `s3` settings | `{"settings"}`, applied by setup | +| `POST /s3/buckets` | `{"name", "owner"}` | `{"name", "owner"}` | +| `PUT /s3/buckets/` | `{"owner"}` | `{"name", "owner"}` | +| `DELETE /s3/buckets/` | | `{"name", "removed": true}`; 409 while it is not empty | +| `POST /s3/accounts` | `{"owner"}` | `{"owner", "access_key", "secret_key", "created"}` | +| `PUT /s3/accounts/` | | the same, with a new `secret_key` | +| `DELETE /s3/accounts/` | | `{"owner", "access_key", "removed": true}` | +| `GET /account/s3` | | `{"enabled", "endpoint", "region", "account"}`, `account` null without one | +| `POST`, `PUT`, `DELETE /account/s3` | | Make, replace the secret of, or remove the user's own account | + +Backups always hold the IAM folder and `/etc/vpn-router`; with `backup.s3` they also hold the +buckets, which may be large. + ## Metrics While `metrics.enabled` is `true`, `vpn-router-setup` runs `vpn-router-metrics`, which every minute diff --git a/debian-package/debian/changelog b/debian-package/debian/changelog index 2b8154e..4edc7e3 100644 --- a/debian-package/debian/changelog +++ b/debian-package/debian/changelog @@ -1,3 +1,24 @@ +vpn-router (0.36.0-1) unstable; urgency=medium + + feature: the router serves S3-compatible object storage with versitygw, with one S3 account per user and buckets owned by users. + + * s3.enabled, s3.port (7447) and s3.region: while storage is on and + HTTPS is ready, Caddy serves versitygw's S3 API at + https://:, with path-style addresses. + * vpn-router-install@s3 downloads the pinned versitygw 1.10.0 .deb for + amd64 or arm64, checks its SHA-256 and installs it; + vpn-router-s3.service runs it with the buckets in + /var/lib/vpn-router/s3/data. + * vpn-router-s3 makes, replaces the secret of and removes a user's S3 + account, and makes, gives and removes buckets; GET, PUT /s3, + /s3/buckets and /s3/accounts do the same for admins, and /account/s3 + for the signed-in user's own account. + * A user's S3 account follows the user: it stops working while the user + is disabled and is removed with the user. + * Backups hold the S3 accounts; with backup.s3 also the buckets. + + -- Slawomir Koszewski Thu, 29 Oct 2026 19:16:59 +0100 + vpn-router (0.35.2-1) unstable; urgency=medium fix: the web service answers /, /status and /status/full again while IKEv2 runs. diff --git a/debian-package/debian/control b/debian-package/debian/control index 7df9cfd..1209559 100644 --- a/debian-package/debian/control +++ b/debian-package/debian/control @@ -57,8 +57,10 @@ Description: Linux VPN router with IPSec, IKEv2, OpenVPN and WireGuard support certificates and CRLs. Caddy, installed by the package from the Ubuntu archive or from Caddy's repository, serves HTTPS and static web sites, whose files are uploaded through the web service or WebDAV, an optional PowerDNS - name server answers for a zone of the sites, and an optional OpenID - Provider signs the directory's users in to other systems. It keeps metrics of the + name server answers for a zone of the sites, an optional OpenID + Provider signs the directory's users in to other systems, and optional + S3-compatible object storage, versitygw installed on demand, keeps buckets + of the directory's users. It keeps metrics of the router in PostgreSQL and serves them for an OpenTelemetry Collector, and makes scheduled backups to a local folder, copied to the cloud with rclone, or to a Proxmox Backup Server. Nothing in the package is specific to a cloud diff --git a/debian-package/debian/install b/debian-package/debian/install index 2ceabc9..314df10 100644 --- a/debian-package/debian/install +++ b/debian-package/debian/install @@ -13,6 +13,7 @@ src/usr/lib/vpn-router/vpnrouter_backup.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_pki.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_sites.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_dns.py usr/lib/vpn-router/ +src/usr/lib/vpn-router/vpnrouter_s3.py usr/lib/vpn-router/ src/usr/lib/vpn-router/install-packages usr/lib/vpn-router/ src/usr/lib/vpn-router/webdav-auth usr/lib/vpn-router/ src/usr/lib/vpn-router/openvpn-same-user usr/lib/vpn-router/ @@ -27,6 +28,7 @@ src/usr/sbin/vpn-router-backup usr/sbin/ src/usr/sbin/vpn-router-ca usr/sbin/ src/usr/sbin/vpn-router-caddy usr/sbin/ src/usr/sbin/vpn-router-site usr/sbin/ +src/usr/sbin/vpn-router-s3 usr/sbin/ src/usr/lib/vpn-router/configure usr/lib/vpn-router/ src/usr/lib/vpn-router/docker-rules usr/lib/vpn-router/ src/usr/lib/vpn-router/setup usr/lib/vpn-router/ diff --git a/debian-package/debian/rules b/debian-package/debian/rules index 46fa571..7db64c4 100755 --- a/debian-package/debian/rules +++ b/debian-package/debian/rules @@ -12,5 +12,6 @@ override_dh_installsystemd: dh_installsystemd --name=vpn-router-pki --no-enable --no-start dh_installsystemd --name=vpn-router-caddy --no-enable --no-start dh_installsystemd --name=vpn-router-webdav --no-enable --no-start + dh_installsystemd --name=vpn-router-s3 --no-enable --no-start dh_installsystemd --name=vpn-router-resolver --no-enable --no-start dh_installsystemd --name=vpn-router-install@ --no-enable --no-start diff --git a/debian-package/debian/vpn-router.vpn-router-s3.service b/debian-package/debian/vpn-router.vpn-router-s3.service new file mode 100644 index 0000000..27ac37c --- /dev/null +++ b/debian-package/debian/vpn-router.vpn-router-s3.service @@ -0,0 +1,22 @@ +[Unit] +Description=VPN Router S3-compatible object storage (versitygw) +Documentation=file:/usr/share/doc/vpn-router/README.md +After=network-online.target +Wants=network-online.target +ConditionFileIsExecutable=/usr/bin/versitygw + +[Service] +Type=simple +EnvironmentFile=/etc/vpn-router/s3.env +ExecStart=/usr/bin/versitygw posix /var/lib/vpn-router/s3/data +Restart=on-failure +RestartSec=15 +LimitNOFILE=65536 +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/vpn-router/s3 +ProtectHome=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/debian-package/src/usr/lib/vpn-router/install-packages b/debian-package/src/usr/lib/vpn-router/install-packages index c80ad7b..8a42134 100755 --- a/debian-package/src/usr/lib/vpn-router/install-packages +++ b/debian-package/src/usr/lib/vpn-router/install-packages @@ -1,33 +1,83 @@ #!/usr/bin/python3 -"""Install the packages of a feature from the Ubuntu archive, then restart vpn-router-setup without waiting. +"""Install the packages of a feature, then restart vpn-router-setup without waiting. Run by vpn-router-install@.service, which vpn-router-setup starts when the feature is on and its packages are -missing: webdav (rclone) and dns (PowerDNS with its gpgsql backend). Recommended packages are left out, so PowerDNS -comes without its bind backend. A failure is a message on standard error and the exit status 1. +missing: webdav (rclone) and dns (PowerDNS with its gpgsql backend) from the Ubuntu archive, and s3 (versitygw), whose +pinned release (vpnrouter_s3.VERSION) is downloaded for this host's architecture into a new folder, checked against +its pinned SHA-256 and installed with apt-get. Recommended packages are left out, so PowerDNS comes without its bind +backend. A failure is a message on standard error and the exit status 1. install-packages FEATURE """ +import hashlib import os import subprocess import sys +import tempfile +import urllib.request -#: The packages of each feature. +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) + +import vpnrouter_s3 + +#: The packages of each feature from the archive. FEATURES = {'webdav': ['rclone'], 'dns': ['pdns-server', 'pdns-backend-pgsql']} +#: The features installed from a downloaded package. +DOWNLOADS = ('s3',) + #: apt-get, waiting up to ten minutes for dpkg's lock, which a package installation that runs setup holds. APT = ['apt-get', '-o', 'DPkg::Lock::Timeout=600'] +#: The options of apt-get install: no questions, no recommended packages, and the installed configuration files kept. +INSTALL = ['-y', '--no-install-recommends', '-o', 'Dpkg::Options::=--force-confdef', '-o', + 'Dpkg::Options::=--force-confold', 'install'] + +#: Seconds a download may take. +DOWNLOAD_TIMEOUT = 300 + + +def command(args, environment): + print(f'$ {" ".join(args)}', flush=True) + if subprocess.run(args, env=environment).returncode != 0: + sys.exit(f'install-packages: {args[0]} failed') + + +def download_versitygw(folder): + """Download the pinned versitygw .deb of this host's architecture into the folder and check it; its path.""" + arch = vpnrouter_s3.architecture() + if arch not in vpnrouter_s3.SHA256: + sys.exit(f'install-packages: versitygw has no package for {arch}; only {", ".join(vpnrouter_s3.SHA256)}') + url = vpnrouter_s3.DEB_URL.format(version=vpnrouter_s3.VERSION, arch=arch) + path = os.path.join(folder, os.path.basename(url)) + print(f'Downloading {url}', flush=True) + digest = hashlib.sha256() + with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response, open(path, 'wb') as handle: + while chunk := response.read(1024 * 1024): + digest.update(chunk) + handle.write(chunk) + if digest.hexdigest() != vpnrouter_s3.SHA256[arch]: + sys.exit(f'install-packages: {url} has the SHA-256 {digest.hexdigest()}, not {vpnrouter_s3.SHA256[arch]}') + os.chmod(folder, 0o755) + os.chmod(path, 0o644) + return path + def main(): - if len(sys.argv) != 2 or sys.argv[1] not in FEATURES: - sys.exit(f'usage: install-packages {"|".join(FEATURES)}') + if len(sys.argv) != 2 or sys.argv[1] not in (*FEATURES, *DOWNLOADS): + sys.exit(f'usage: install-packages {"|".join((*FEATURES, *DOWNLOADS))}') environment = dict(os.environ, DEBIAN_FRONTEND='noninteractive') - for args in ([*APT, 'update'], [*APT, '-y', '--no-install-recommends', '-o', 'Dpkg::Options::=--force-confdef', - '-o', 'Dpkg::Options::=--force-confold', 'install', *FEATURES[sys.argv[1]]]): - print(f'$ {" ".join(args)}', flush=True) - if subprocess.run(args, env=environment).returncode != 0: - sys.exit(f'install-packages: {args[0]} failed') + if sys.argv[1] in FEATURES: + command([*APT, 'update'], environment) + command([*APT, *INSTALL, *FEATURES[sys.argv[1]]], environment) + else: + with tempfile.TemporaryDirectory() as folder: + try: + path = download_versitygw(folder) + except OSError as exc: + sys.exit(f'install-packages: the download failed: {exc}') + command([*APT, *INSTALL, path], environment) subprocess.run(['systemctl', 'restart', '--no-block', 'vpn-router-setup.service']) return 0 diff --git a/debian-package/src/usr/lib/vpn-router/setup b/debian-package/src/usr/lib/vpn-router/setup index 1dc5490..558d051 100755 --- a/debian-package/src/usr/lib/vpn-router/setup +++ b/debian-package/src/usr/lib/vpn-router/setup @@ -37,6 +37,7 @@ import vpnrouter_auth import vpnrouter_dns import vpnrouter_ldap import vpnrouter_pki +import vpnrouter_s3 import vpnrouter_sites import vpnrouter_users from vpnrouter import fail, warn @@ -301,6 +302,7 @@ def filter_block(ctx): (vpnrouter.MANAGER.exists(), ctx['https_manager_port']), (vpnrouter.metrics_ready(ctx), ctx['metrics_https_port']), (vpnrouter.webdav_ready(ctx), ctx['webdav_port']), + (vpnrouter.s3_ready(ctx), ctx['s3_port']), ('http-80' in publish, vpnrouter.ACME_HTTP_PORT), ('https-default' in publish, vpnrouter.HTTPS_DEFAULT_PORT)) if wanted] lines += [ @@ -698,8 +700,9 @@ def manage_caddy(cfg, ctx): stop it when not. While HTTPS is ready, CADDYFILE serves the web service on https.web_port, while the manager is installed the - manager on https.manager_port, while metrics are enabled the metrics service on metrics.https_port, and while - WebDAV is ready WebDAV on sites.webdav_port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all + manager on https.manager_port, while metrics are enabled the metrics service on metrics.https_port, while + WebDAV is ready WebDAV on sites.webdav_port, and while the object storage is ready and versitygw is installed its + S3 API on s3.port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all calling the service on its loopback address. It also serves the CA files of vpnrouter_pki.PUBLISH_DIR at /pki/ on the endpoints of pki.publish that are Caddy's, and an ACME server at /acme// on https.web_port for each issuing CA with acme and a certificate. @@ -785,6 +788,8 @@ def manage_caddy(cfg, ctx): 'key': str(CADDY_PKI_DIR / name / 'key.pem')} for name in authorities], acme_cas=[name for name in authorities if https and name in ctx['pki_acme']], webdav=vpnrouter.webdav_ready(ctx), webdav_loopback_port=vpnrouter.WEBDAV_LOOPBACK_PORT, + s3=vpnrouter.s3_ready(ctx) and vpnrouter_s3.installed() == vpnrouter_s3.VERSION, + s3_loopback_port=vpnrouter_s3.S3_PORT, sites=[{'name': site['name'], 'tls': site['tls'], 'ca': site['ca'], 'spa': site['spa'], 'index': site['index_file'], 'root': str(vpnrouter_sites.SITES_DIR / site['name']), 'addresses': [caddy_address(host, site['port'], site['tls']) for host in site['hosts']] @@ -820,6 +825,34 @@ def manage_webdav(ctx): run(['systemctl', 'enable', '--now', WEBDAV_UNIT], check=False) +def manage_s3(ctx): + """Run versitygw, the S3-compatible object storage (vpnrouter_s3), while it is ready, and stop it when not. + + The pinned versitygw is installed by vpn-router-install@s3 when another version or none is installed. ROOT_ENV is + written with the root keys it has, or new ones, and s3.region; a change restarts the service. Once it runs, the + S3 accounts are made to match their users (vpnrouter_s3.sync_all). + """ + if not vpnrouter.s3_ready(ctx): + if unit_is_active(vpnrouter_s3.UNIT): + run(['systemctl', 'disable', '--now', vpnrouter_s3.UNIT], check=False) + return + if vpnrouter_s3.installed() != vpnrouter_s3.VERSION: + install_packages('s3') + return + vpnrouter_s3.ensure_folders() + changed = vpnrouter_s3.write_env(ctx['s3_region']) + if not unit_is_active(vpnrouter_s3.UNIT): + run(['systemctl', 'enable', '--now', vpnrouter_s3.UNIT], check=False) + elif changed: + run(['systemctl', 'restart', vpnrouter_s3.UNIT], check=False) + for _ in range(10): + if unit_is_active(vpnrouter_s3.UNIT): + break + time.sleep(1) + for problem in vpnrouter_s3.sync_all(): + warn(problem) + + def manage_dns(cfg, ctx): """Run PowerDNS, the primary name server of the domains of vpnrouter_dns, on the addresses of dns.listen while DNS is enabled, and stop it when not; its packages are installed when they are missing. @@ -1072,6 +1105,8 @@ def main(): manage_caddy(cfg, ctx) progress('running', 'WebDAV') manage_webdav(ctx) + progress('running', 'object storage') + manage_s3(ctx) progress('running', 'name server') manage_dns(cfg, ctx) progress('running', 'PKI timer') diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter.py b/debian-package/src/usr/lib/vpn-router/vpnrouter.py index 47d028c..8444d0d 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter.py @@ -69,6 +69,7 @@ SCHEMA = { 'https': {'enabled': True, 'email': '', 'web_port': 7444, 'manager_port': 7443, 'caddy_source': 'ubuntu'}, 'oidc': {'enabled': False}, 'sites': {'enabled': False, 'upload_max_mb': 512, 'webdav': False, 'webdav_port': 7445}, + 's3': {'enabled': False, 'port': 7447, 'region': 'us-east-1'}, 'dns': {'enabled': False, 'nameserver': '', 'addresses': [], 'listen': 'external', 'ttl': 300}, 'pki': {'root_name': 'VPN Router Root CA', 'root_days': 7300, 'ca_days': 3650, 'key': 'rsa-4096', 'crl_days': 7, 'root_crl_days': 180, 'publish': ['http'], 'p2s_name': 'VPN Router P2S CA', @@ -80,7 +81,7 @@ SCHEMA = { 'keep_monthly': '1 year', 'keep_yearly': '10 years'}, 'backup': {'enabled': False, 'schedule': 'daily', 'target': 'local', 'path': '/var/backups/vpn-router', 'keep_hourly': 24, 'keep_daily': 7, 'keep_weekly': 4, 'keep_monthly': 12, 'keep_yearly': 0, - 'metrics': False, 'rclone_remote': '', 'rclone_mode': 'sync', + 'metrics': False, 's3': False, 'rclone_remote': '', 'rclone_mode': 'sync', 'pbs_repository': '', 'pbs_password': '', 'pbs_fingerprint': '', 'pbs_namespace': '', 'pbs_keyfile': ''}, } @@ -140,6 +141,9 @@ LOG_FILES_MAX = 32 #: The port rclone serves WebDAV on, on the loopback address, behind Caddy. WEBDAV_LOOPBACK_PORT = 7446 +#: A region name of s3.region: lowercase letters, digits and hyphens. +S3_REGION = re.compile(r'^[a-z0-9][a-z0-9-]{0,31}$') + #: The largest sites.upload_max_mb, 10 GiB. UPLOAD_MAX_MB = 10240 @@ -576,6 +580,11 @@ def validate(cfg): if _get(cfg, 'sites', 'webdav') and not (_get(cfg, 'https', 'enabled') and _get(cfg, 'wan', 'local_fqdn')): raise ConfigError('sites.webdav: WebDAV is served by Caddy at https://:, ' 'which needs https.enabled and wan.local_fqdn') + if _get(cfg, 's3', 'enabled') and not (_get(cfg, 'https', 'enabled') and _get(cfg, 'wan', 'local_fqdn')): + raise ConfigError('s3.enabled: the object storage is served by Caddy at https://:, ' + 'which needs https.enabled and wan.local_fqdn') + if not S3_REGION.match(_get(cfg, 's3', 'region')): + raise ConfigError(f's3.region: {_get(cfg, "s3", "region")!r} is not a region name, such as us-east-1') dns = cfg['dns'] if dns['enabled']: @@ -695,7 +704,8 @@ def validate(cfg): if _get(cfg, 'openvpn', 'protocol') == 'tcp' and _get(cfg, 'openvpn', 'port') == HTTPS_DEFAULT_PORT: raise ConfigError(f'pki.publish: https-default takes port {HTTPS_DEFAULT_PORT}, the port of openvpn.port') taken[HTTPS_DEFAULT_PORT] = 'pki.publish https-default' - for section_name, option in (('metrics', 'port'), ('metrics', 'https_port'), ('sites', 'webdav_port')): + for section_name, option in (('metrics', 'port'), ('metrics', 'https_port'), ('sites', 'webdav_port'), + ('s3', 'port')): port = _get(cfg, section_name, option) if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: raise ConfigError(f'{section_name}.{option}: {port!r} is not a valid port') @@ -1045,6 +1055,9 @@ def build_context(cfg, public_ip=None): 'sites_enabled': _get(cfg, 'sites', 'enabled'), 'webdav_enabled': _get(cfg, 'sites', 'webdav'), 'webdav_port': _get(cfg, 'sites', 'webdav_port'), + 's3_enabled': _get(cfg, 's3', 'enabled'), + 's3_port': _get(cfg, 's3', 'port'), + 's3_region': _get(cfg, 's3', 'region'), 'dns_enabled': _get(cfg, 'dns', 'enabled'), 'dns_listen': _get(cfg, 'dns', 'listen'), 'loopback': LOOPBACK, @@ -1091,6 +1104,11 @@ def webdav_ready(ctx): return bool(ctx['webdav_enabled'] and https_ready(ctx)) +def s3_ready(ctx): + """The object storage is served by Caddy, on the router's own name, while it is on and HTTPS is ready.""" + return bool(ctx['s3_enabled'] and https_ready(ctx)) + + def dns_ready(ctx): """The name server answers for its domains while dns.enabled is true.""" return bool(ctx['dns_enabled']) diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_backup.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_backup.py index 72c3b5f..400af69 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_backup.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_backup.py @@ -1,7 +1,8 @@ """The backups of the router, for vpn-router-backup and the web service. A backup is a consistent copy of the state that setup cannot make again, staged in STAGING: the -trees of TREES, the folders of the web sites among them, the files of site_to_site psk_file outside them under files/, +trees of TREES, the folders of the web sites and the accounts of the object storage among them, with backup.s3 its +buckets, the files of site_to_site psk_file outside them under files/, the directory as slapcat writes it (ldap/vpn-router.ldif), with backup.metrics the database as pg_dump writes it (postgresql/vpn-router.dump), the databases of the web sites and of the name server (DUMPS) once they exist, and manifest.json. The trees and the directory are copied while @@ -36,6 +37,7 @@ import vpnrouter import vpnrouter_auth import vpnrouter_dns import vpnrouter_ldap +import vpnrouter_s3 import vpnrouter_sites import vpnrouter_users @@ -51,9 +53,10 @@ PBS_CLIENT = pathlib.Path('/usr/bin/proxmox-backup-client') UNIT = 'vpn-router-backup.service' TIMER = 'vpn-router-backup.timer' -#: The state a backup copies: directories and files, each where it is on the system. +#: The state a backup copies: directories and files, each where it is on the system; with backup.s3 also the buckets +#: of the object storage (vpnrouter_s3.DATA_DIR). TREES = ('/etc/vpn-router', '/etc/wireguard', '/etc/ufw', '/etc/default/ufw', '/etc/vpn-router-manager', - str(vpnrouter_sites.SITES_DIR)) + str(vpnrouter_sites.SITES_DIR), str(vpnrouter_s3.IAM_DIR)) #: The databases a backup always dumps, each while it exists: the dump -> the database and its password file. DUMPS = {'postgresql/vpn-router-sites.dump': (vpnrouter_sites.DATABASE, vpnrouter_sites.DATABASE_SECRET), @@ -73,7 +76,7 @@ PERIODS = {'keep_hourly': '%Y-%m-%d %H', 'keep_daily': '%Y-%m-%d', 'keep_weekly' #: The units a restore stops while it replaces the directory and the database, and the ones it restarts after setup. RESTORE_STOPS = ('vpn-router-web.service', 'vpn-router-metrics.service', 'vpn-router-webdav.service', 'pdns.service', - 'slapd.service') + 'vpn-router-s3.service', 'slapd.service') RESTORE_RESTARTS = ('wg-quick@wg0.service', 'openvpn-server@vpn-router.service', 'strongswan.service', 'vpn-router-manager.service') @@ -220,7 +223,8 @@ def stage(cfg): """Copy what a backup holds to STAGING, emptied first, and return the manifest.""" shutil.rmtree(STAGING, ignore_errors=True) STAGING.mkdir(parents=True, mode=0o700) - trees = [tree for tree in TREES if os.path.exists(tree)] + trees = [tree for tree in (*TREES, *((str(vpnrouter_s3.DATA_DIR),) if cfg['backup']['s3'] else ())) + if os.path.exists(tree)] files = psk_files(cfg) vpnrouter_users.LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) with vpnrouter.config_lock(), open(vpnrouter_users.LOCK_FILE, 'w') as users: diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_s3.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_s3.py new file mode 100644 index 0000000..7fca90e --- /dev/null +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_s3.py @@ -0,0 +1,446 @@ +"""The router's S3-compatible object storage, served by versitygw, for setup, vpn-router-s3 and the web service. + +versitygw runs as vpn-router-s3.service with its POSIX backend: each folder of DATA_DIR is a bucket and the files +below it are its objects. Its internal IAM in IAM_DIR holds an account per access key; the root keys in ROOT_ENV are +not stored there. Its S3 API listens on S3_PORT and its Admin API on ADMIN_PORT, both on the loopback address; Caddy +serves the S3 API at https://:. + +Each user or service account of the router's directory has at most one S3 account, an access key and its secret, +recorded in ACCOUNTS with the time it was made. The secret is shown once, when the account is made or its secret +replaced; ACCOUNTS keeps it, as the IAM does, so the account can be switched off and on with its user (sync). A +bucket belongs to one S3 account, which alone may use it; the root keys reach every bucket. + +The Admin API and the S3 API are called with AWS Signature Version 4, signed with the root keys for the region +s3.region. A request that cannot be done raises Failure, whose status says why: 2 invalid, 3 no such user, account or +bucket, 4 it exists or the state does not allow it, 1 other. +""" + +import contextlib +import datetime +import fcntl +import hashlib +import hmac +import json +import os +import pathlib +import re +import secrets +import string +import subprocess +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ElementTree + +import ldap + +import vpnrouter +import vpnrouter_users + +#: The pinned release of versitygw: its version and the SHA-256 of its .deb for each Debian architecture. +VERSION = '1.10.0' +SHA256 = {'amd64': 'd8a39e9cc7da6f17467e60795cb703cfcb29b3fe5db8227e341d90bc64727140', + 'arm64': '5f81e0dec09d8f1f8ca3159de209543ceb03714dc0066cf7d353b8ef064a0136'} +DEB_URL = 'https://github.com/versity/versitygw/releases/download/v{version}/versitygw_{version}_linux_{arch}.deb' + +#: The versitygw binary the .deb installs. +BINARY = pathlib.Path('/usr/bin/versitygw') + +UNIT = 'vpn-router-s3.service' +INSTALL_UNIT = 'vpn-router-install@s3.service' + +#: The buckets, one folder each, and the accounts of versitygw's internal IAM. +S3_DIR = pathlib.Path('/var/lib/vpn-router/s3') +DATA_DIR = S3_DIR / 'data' +IAM_DIR = S3_DIR / 'iam' + +#: The environment of vpn-router-s3.service, which setup writes: the root keys, the ports, the IAM and the region. +ROOT_ENV = pathlib.Path('/etc/vpn-router/s3.env') + +#: The S3 account of each user: name -> {"access_key", "secret_key", "created"}. +ACCOUNTS = pathlib.Path('/etc/vpn-router/s3-accounts.json') +ACCOUNTS_LOCK = pathlib.Path('/run/vpn-router/s3-accounts.lock') + +S3_PORT = 7448 +ADMIN_PORT = 7449 + +#: The folder versitygw keeps the parts of multipart uploads in, inside each bucket. +TEMPORARY = '.sgwtmp' + +#: A bucket name versitygw and S3 accept: 3 to 63 lowercase letters, digits, dots and hyphens, from and to a letter or +#: a digit, and not an IPv4 address. +BUCKET_NAME = re.compile(r'^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$') + +#: Seconds a call of versitygw may take. +TIMEOUT = 30 + +#: The characters of the access keys and the secrets made here. +KEY_CHARACTERS = string.ascii_uppercase + string.digits +SECRET_CHARACTERS = string.ascii_letters + string.digits + + +class Failure(Exception): + """A request that cannot be done; status is 2 invalid, 3 no such user, account or bucket, 4 exists or not allowed, + 1 other.""" + + def __init__(self, message, status=1): + super().__init__(message) + self.status = status + + +def architecture(): + """The Debian architecture of this host, such as amd64.""" + return subprocess.run(['dpkg', '--print-architecture'], capture_output=True, text=True, check=True).stdout.strip() + + +def installed(): + """The installed version of versitygw, or None when it is not installed.""" + result = subprocess.run(['dpkg-query', '-W', '-f=${Status}\t${Version}', 'versitygw'], capture_output=True, text=True) + status, _, version = result.stdout.partition('\t') + return version if result.returncode == 0 and status == 'install ok installed' else None + + +def running(): + """Whether vpn-router-s3.service is active.""" + result = subprocess.run(['systemctl', 'is-active', UNIT], capture_output=True, text=True) + return result.stdout.strip() == 'active' + + +def read_env(): + """The settings of ROOT_ENV, {} while it does not exist.""" + try: + text = ROOT_ENV.read_text(encoding='utf-8') + except FileNotFoundError: + return {} + return dict(line.split('=', 1) for line in text.splitlines() if '=' in line and not line.startswith('#')) + + +def write_env(region): + """Write ROOT_ENV with the root keys it has, or new ones, and the region; whether it changed.""" + current = read_env() + values = {'ROOT_ACCESS_KEY_ID': current.get('ROOT_ACCESS_KEY_ID') or 'VRROOT' + random_text(KEY_CHARACTERS, 14), + 'ROOT_SECRET_ACCESS_KEY': current.get('ROOT_SECRET_ACCESS_KEY') or random_text(SECRET_CHARACTERS, 40), + 'VGW_PORT': f'{vpnrouter.LOOPBACK}:{S3_PORT}', 'VGW_ADMIN_PORT': f'{vpnrouter.LOOPBACK}:{ADMIN_PORT}', + 'VGW_IAM_DIR': str(IAM_DIR), 'VGW_REGION': region} + if values == current: + return False + ROOT_ENV.parent.mkdir(parents=True, exist_ok=True) + temporary = ROOT_ENV.with_name(ROOT_ENV.name + '.new') + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: + handle.write('# Written by vpn-router-setup for vpn-router-s3.service; edits are overwritten.\n') + handle.writelines(f'{key}={value}\n' for key, value in values.items()) + os.replace(temporary, ROOT_ENV) + return True + + +def random_text(characters, length): + return ''.join(secrets.choice(characters) for _ in range(length)) + + +def ensure_folders(): + """Make DATA_DIR and IAM_DIR, readable by root only.""" + for path in (S3_DIR, DATA_DIR, IAM_DIR): + path.mkdir(mode=0o700, parents=True, exist_ok=True) + + +@contextlib.contextmanager +def accounts_lock(): + """Hold the lock that serializes the changes of ACCOUNTS.""" + ACCOUNTS_LOCK.parent.mkdir(parents=True, exist_ok=True) + with open(ACCOUNTS_LOCK, 'w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +def accounts(): + """The S3 accounts of ACCOUNTS by user name, {} while it does not exist.""" + try: + data = json.loads(ACCOUNTS.read_text(encoding='utf-8')) + except FileNotFoundError: + return {} + except ValueError as exc: + raise Failure(f'{ACCOUNTS} cannot be read: {exc}') + return data if isinstance(data, dict) else {} + + +def save_accounts(data): + """Write ACCOUNTS by a rename, readable by root only.""" + ACCOUNTS.parent.mkdir(parents=True, exist_ok=True) + temporary = ACCOUNTS.with_name(ACCOUNTS.name + '.new') + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: + json.dump(data, handle, indent=2) + handle.write('\n') + os.replace(temporary, ACCOUNTS) + + +def uri_encode(text, slash=True): + """Text encoded as Signature Version 4 encodes it: every byte but A-Z, a-z, 0-9, -, ., _ and ~, and / when slash.""" + return urllib.parse.quote(text, safe='-._~' + ('' if slash else '/')) + + +def signed_headers(method, host, path, query, body, headers, access, secret, region, now): + """The headers of a request signed with AWS Signature Version 4 for the service s3 at the UTC time now: host, + x-amz-content-sha256, x-amz-date and the headers given, all signed, and Authorization.""" + stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + digest = hashlib.sha256(body).hexdigest() + signed = {'host': host, 'x-amz-content-sha256': digest, 'x-amz-date': stamp, + **{key.lower(): value for key, value in headers.items()}} + names = sorted(signed) + canonical = '\n'.join([method, uri_encode(path, slash=False), + '&'.join(f'{uri_encode(key)}={uri_encode(value)}' for key, value in sorted(query.items())), + ''.join(f'{name}:{" ".join(str(signed[name]).split())}\n' for name in names), + ';'.join(names), digest]) + scope = f'{day}/{region}/s3/aws4_request' + to_sign = '\n'.join(['AWS4-HMAC-SHA256', stamp, scope, hashlib.sha256(canonical.encode()).hexdigest()]) + key = ('AWS4' + secret).encode() + for part in (day, region, 's3', 'aws4_request'): + key = hmac.new(key, part.encode(), hashlib.sha256).digest() + signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + return {**signed, 'Authorization': f'AWS4-HMAC-SHA256 Credential={access}/{scope}, ' + f'SignedHeaders={";".join(names)}, Signature={signature}'} + + +def call(method, port, path, query=None, body=b'', headers=None): + """Make a request of versitygw on the port, signed with the root keys; the status and the body of the answer. + + Failure with versitygw's message when it answers an error, and when it cannot be reached. + """ + env = read_env() + access, secret, region = env.get('ROOT_ACCESS_KEY_ID'), env.get('ROOT_SECRET_ACCESS_KEY'), env.get('VGW_REGION') + if not access or not secret or not region: + raise Failure('the object storage is not set up; setup writes its keys once s3.enabled is on', 4) + host = f'{vpnrouter.LOOPBACK}:{port}' + signed = signed_headers(method, host, path, query or {}, body, headers or {}, access, secret, region, + datetime.datetime.now(datetime.timezone.utc)) + canonical_query = '&'.join(f'{uri_encode(key)}={uri_encode(value)}' for key, value in sorted((query or {}).items())) + url = f'http://{host}{uri_encode(path, slash=False)}' + (f'?{canonical_query}' if canonical_query else '') + request = urllib.request.Request(url, data=body if body else None, method=method, + headers={key: value for key, value in signed.items() if key != 'host'}) + try: + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + return response.status, response.read() + except urllib.error.HTTPError as exc: + answer = exc.read() + code, message = error_of(answer) + status = 3 if exc.code == 404 or code in ('NoSuchBucket', 'XAdminUserNotFound') else \ + 4 if exc.code == 409 or code in ('XAdminUserExists', 'BucketAlreadyExists', 'BucketNotEmpty') else \ + 2 if exc.code == 400 else 1 + raise Failure(f'versitygw refused {method} {path}: {message or code or exc.reason}', status) + except OSError as exc: + raise Failure(f'versitygw cannot be reached on {host}: {exc}; is {UNIT} running?') + + +def local_name(element): + return element.tag.rsplit('}', 1)[-1] + + +def error_of(body): + """The code and the message of an S3 or Admin API error document, empty strings when there is none.""" + try: + root = ElementTree.fromstring(body) + except ElementTree.ParseError: + return '', body.decode(errors='replace').strip() + fields = {local_name(child): (child.text or '').strip() for child in root} + return fields.get('Code', ''), fields.get('Message', '') + + +def admin(path, query=None, body=b''): + """Call the Admin API: PATCH of the path; the body of the answer.""" + headers = {'Content-Type': 'application/xml'} if body else None + return call('PATCH', ADMIN_PORT, path, query, body, headers)[1] + + +def account_xml(root, **fields): + """An XML document of the root element with the fields as its children.""" + element = ElementTree.Element(root) + for name, value in fields.items(): + ElementTree.SubElement(element, name).text = value + return ElementTree.tostring(element) + + +def user_state(connection, name): + """Whether the user or service account exists and is enabled: None when it does not exist, else True or False.""" + try: + return vpnrouter_users.summary(vpnrouter_users.entry(connection, name))['enabled'] + except vpnrouter_users.Failure as failure: + if failure.status == 3: + return None + raise + + +def shown(name, account): + """An account as the answers show it, without its secret.""" + return {'owner': name, 'access_key': account['access_key'], 'created': account['created']} + + +def list_accounts(owner=None): + """The accounts, each {owner, access_key, created}, sorted by owner; with owner, only that user's.""" + return [shown(name, account) for name, account in sorted(accounts().items()) if owner in (None, name)] + + +def account_add(name): + """Make the S3 account of the user, who must exist and have none: {owner, access_key, secret_key, created}.""" + with vpnrouter_users.session() as connection: + if user_state(connection, name) is None: + raise Failure(f'user {name} does not exist', 3) + with accounts_lock(): + data = accounts() + if name in data: + raise Failure(f'{name} has an S3 account, {data[name]["access_key"]}; replace its secret or remove it', 4) + account = {'access_key': 'VR' + random_text(KEY_CHARACTERS, 18), + 'secret_key': random_text(SECRET_CHARACTERS, 40), + 'created': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')} + admin('/create-user', body=account_xml('Account', Access=account['access_key'], Secret=account['secret_key'], + Role='user')) + data[name] = account + save_accounts(data) + return {**shown(name, account), 'secret_key': account['secret_key']} + + +def account_rotate(name): + """Replace the secret of the user's S3 account: {owner, access_key, secret_key, created}.""" + with accounts_lock(): + data = accounts() + if name not in data: + raise Failure(f'{name} has no S3 account', 3) + account = dict(data[name], secret_key=random_text(SECRET_CHARACTERS, 40)) + admin('/update-user', {'access': account['access_key']}, + account_xml('MutableProps', Secret=account['secret_key'])) + data[name] = account + save_accounts(data) + sync(name) + return {**shown(name, account), 'secret_key': account['secret_key']} + + +def account_remove(name): + """Remove the user's S3 account; its buckets keep it as their owner until another is chosen.""" + with accounts_lock(): + data = accounts() + if name not in data: + raise Failure(f'{name} has no S3 account', 3) + try: + admin('/delete-user', {'access': data[name]['access_key']}) + except Failure as failure: + if failure.status != 3: + raise + account = data.pop(name) + save_accounts(data) + return {**shown(name, account), 'removed': True} + + +def sync(name): + """Bring the user's S3 account in line with the user: removed with a user that no longer exists, its secret + replaced by an unknown one while the user is disabled, and its own secret while it is enabled. Nothing for a user + without an account, or while versitygw does not run.""" + if name not in accounts() or not running(): + return + with vpnrouter_users.session() as connection: + state = user_state(connection, name) + if state is None: + account_remove(name) + return + secret = accounts()[name]['secret_key'] if state else random_text(SECRET_CHARACTERS, 40) + admin('/update-user', {'access': accounts()[name]['access_key']}, account_xml('MutableProps', Secret=secret)) + + +#: The vpnrouter_users operations after which the user's S3 account follows the user (follow). +FOLLOWING = ('disable', 'enable', 'remove') + + +def follow(name): + """sync() the user's account after a vpnrouter_users operation of FOLLOWING; why it failed, or None. A failure + leaves the account as it is until setup's next run synchronizes it.""" + try: + sync(name) + except (Failure, vpnrouter_users.Failure, ldap.LDAPError, OSError) as exc: + return f'the S3 account of {name} cannot follow the user: {exc}; setup synchronizes it at its next run' + return None + + +def sync_all(): + """sync() each account, as setup does once versitygw runs; the accounts versitygw lacks, as after a restore of the + configuration only, are made again. The failures are answered as messages.""" + problems = [] + try: + root = ElementTree.fromstring(admin('/list-users')) + known = {(child.findtext('Access') or '').strip() for child in root if local_name(child) == 'Account'} + for name, account in accounts().items(): + if account['access_key'] not in known: + admin('/create-user', body=account_xml('Account', Access=account['access_key'], + Secret=account['secret_key'], Role='user')) + except (Failure, ElementTree.ParseError) as exc: + return [f'the S3 accounts cannot be checked: {exc}'] + for name in list(accounts()): + try: + sync(name) + except (Failure, vpnrouter_users.Failure, ldap.LDAPError, OSError) as exc: + problems.append(f'the S3 account of {name} cannot be synchronized: {exc}') + return problems + + +def check_bucket(name): + if not isinstance(name, str) or not BUCKET_NAME.match(name) or '..' in name or \ + re.fullmatch(r'\d+\.\d+\.\d+\.\d+', name): + raise Failure(f'{name!r} is not a bucket name: 3 to 63 lowercase letters, digits, dots and hyphens, from and to ' + 'a letter or a digit', 2) + + +def usage(bucket): + """The size in bytes and the number of objects of the bucket, from its folder, without the uploads in progress.""" + size = count = 0 + for folder, folders, files in os.walk(DATA_DIR / bucket): + if folder == str(DATA_DIR / bucket): + folders[:] = [item for item in folders if item != TEMPORARY] + for item in files: + try: + size += os.lstat(os.path.join(folder, item)).st_size + count += 1 + except OSError: + pass + return size, count + + +def buckets(): + """The buckets, each {name, owner, access_key, size, objects}, sorted by name; owner is the user of the account + that owns it, None for an account no user has.""" + root = ElementTree.fromstring(admin('/list-buckets')) + owners = {account['access_key']: name for name, account in accounts().items()} + found = [] + for child in root: + if local_name(child) != 'Bucket': + continue + name, access = (child.findtext('Name') or '').strip(), (child.findtext('Owner') or '').strip() + size, count = usage(name) + found.append({'name': name, 'owner': owners.get(access), 'access_key': access, 'size': size, 'objects': count}) + return sorted(found, key=lambda bucket: bucket['name']) + + +def owner_key(owner): + """The access key of the S3 account of the user; Failure when the user has none.""" + account = accounts().get(owner) + if account is None: + raise Failure(f'{owner} has no S3 account; make one first', 3) + return account['access_key'] + + +def bucket_add(name, owner): + """Make the bucket, owned by the user's S3 account.""" + check_bucket(name) + call('PATCH', ADMIN_PORT, f'/{name}/create', headers={'x-vgw-owner': owner_key(owner)}) + return {'name': name, 'owner': owner} + + +def bucket_owner(name, owner): + """Give the bucket to the user's S3 account; versitygw replaces its ACL and policy with the default ones.""" + check_bucket(name) + admin('/change-bucket-owner', {'bucket': name, 'owner': owner_key(owner)}) + return {'name': name, 'owner': owner} + + +def bucket_remove(name): + """Remove the bucket, which must be empty.""" + check_bucket(name) + call('DELETE', S3_PORT, f'/{name}') + return {'name': name, 'removed': True} diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/__init__.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/__init__.py index cd2b7e7..84a1404 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/__init__.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/__init__.py @@ -55,6 +55,8 @@ Pages: remove (see Handler.site_files()); a member of a site's editors group may use the site's files /dns the primary name server of the router's domains: its settings, and the domains: add, change and remove, each with its secondaries, the records it serves, and its manual records (see dns()) + /s3 the S3-compatible object storage (vpnrouter_s3): its settings, its buckets and the users' S3 accounts; + /account/s3 is the signed-in user's own account (see s3() and account_s3()) /postgresql the databases of each PostgreSQL cluster on the router with their sizes, as JSON (see postgresql_databases()); /config has the clusters' settings @@ -63,7 +65,7 @@ Pages: loopback address, where Caddy passes the requests of the HTTPS site; POST /oidc/login counts towards the limit of /auth calls -/status/full, /config, /metrics/history, /metrics/settings, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /sites, /dns, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed +/status/full, /config, /metrics/history, /metrics/settings, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /sites, /dns, /s3, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub; /account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A token of vpn-router-token has the admin claim and names no user; it may call every path but @@ -95,6 +97,7 @@ Modules, one per family of paths, with what they share in common: logs /logs sites /sites, with the files of each site dns /dns + s3 /s3 and /account/s3 metrics /metrics/history and /metrics/settings shell /shell, its WebSocket, and the editor of the shell with its socket handler Handler, which reads a request, checks its token and passes it to the module of its path diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/handler.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/handler.py index db6b6ae..baf1d17 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/handler.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/handler.py @@ -24,6 +24,7 @@ from .identities import account, groups, service_accounts, users from .logs import logs from .metrics import metrics_history, metrics_settings from .pki import PUBLISHED, ca +from .s3 import account_s3, s3 from .shell import EDITOR_BODY_LIMIT, shell, shell_editor, shell_ticket from .sites import SITE_FILES, site_files, sites from .status import cached, details, html_of, postgresql_databases, shown, text_of @@ -64,7 +65,7 @@ class Handler(BaseHTTPRequestHandler): if self.command not in ('GET', 'HEAD'): self.close_connection = True directory = next((name for name in ('users', 'service-accounts', 'groups', 'account', 'site-to-site', 'backup', - 'ca', 'shell', 'remote-access', 'logs', 'sites', 'dns') + 'ca', 'shell', 'remote-access', 'logs', 'sites', 'dns', 's3') if path == f'/{name}' or path.startswith(f'/{name}/')), None) protected = path in ('/status/full', '/config', '/metrics/history', '/metrics/settings', '/postgresql') or \ directory is not None @@ -124,6 +125,9 @@ class Handler(BaseHTTPRequestHandler): if code != 200: log(f'shell ticket of {claims.get("sub")} from {self.client()} refused: {document["error"]}') body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' + elif directory == 'account' and path == '/account/s3': + code, document = account_s3(claims['sub'], self.command) + body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' elif directory == 'account': code, document = account(claims['sub'], self.command, path, payload) body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' @@ -154,6 +158,9 @@ class Handler(BaseHTTPRequestHandler): elif directory == 'dns': code, document = dns(self.command, path, query, payload) body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' + elif directory == 's3': + code, document = s3(self.command, path, payload) + body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' elif directory == 'service-accounts': code, document = service_accounts(self.command, path, payload) body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/identities.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/identities.py index 7def30c..4ccd8ca 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/identities.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/identities.py @@ -7,9 +7,13 @@ import ldap import vpnrouter import vpnrouter_auth import vpnrouter_passkeys +import vpnrouter_s3 import vpnrouter_users -from .common import USER_STATUS +from .common import USER_STATUS, log + +#: The routes of /users and /service-accounts after which the S3 account of the user follows it (vpnrouter_s3.follow). +FOLLOWING = (('POST', 3, 'disable'), ('POST', 3, 'enable'), ('DELETE', 2, None)) def users(state, method, path, body): @@ -77,6 +81,8 @@ def users(state, method, path, body): document = actions[route](connection) if operations.get(route) in vpnrouter_auth.REVOKING: vpnrouter_auth.revoke(vpnrouter.load_config(), name) + if route in FOLLOWING and (problem := vpnrouter_s3.follow(name)): + log(problem) except vpnrouter_users.Failure as failure: return USER_STATUS.get(failure.status, 500), {'error': str(failure)} except (ldap.LDAPError, OSError) as exc: @@ -146,6 +152,8 @@ def service_accounts(method, path, body): if name is not None and not vpnrouter_users.service(vpnrouter_users.entry(connection, name)): raise vpnrouter_users.Failure(f'{name} is not a service account', 3) document = actions[route](connection) + if route in FOLLOWING and (problem := vpnrouter_s3.follow(name)): + log(problem) except vpnrouter_users.Failure as failure: return USER_STATUS.get(failure.status, 500), {'error': str(failure)} except (ldap.LDAPError, OSError) as exc: diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_web/s3.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/s3.py new file mode 100644 index 0000000..87f635b --- /dev/null +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_web/s3.py @@ -0,0 +1,118 @@ +"""/s3 and /account/s3: the S3-compatible object storage of the router (vpnrouter_s3), its buckets and its accounts. + + GET /s3 {"settings", "endpoint", "version", "installed", "running", "buckets", "accounts", + "problem"}: the s3 section, the URL clients call, the pinned and the installed versions + of versitygw, whether it runs, the buckets, each {name, owner, access_key, size, + objects}, while it runs, the accounts, each {owner, access_key, created}, and why the + buckets cannot be read, null when they can + PUT /s3 change any of the settings of the s3 section, applied by vpn-router-setup + POST /s3/buckets make a bucket: {"name", "owner"}, owner a user with an S3 account + PUT /s3/buckets/ {"owner"} gives the bucket to another user's account + DELETE /s3/buckets/ remove the bucket, which must be empty + POST /s3/accounts make the S3 account of a user: {"owner"}; answers its secret_key, shown only here + PUT /s3/accounts/ replace the account's secret; answers the new secret_key + DELETE /s3/accounts/ remove the account + + GET /account/s3 {"enabled", "endpoint", "region", "account"}: the signed-in user's account, or null + POST /account/s3 make the signed-in user's account; answers its secret_key + PUT /account/s3 replace its secret; answers the new secret_key + DELETE /account/s3 remove it + +A change of the settings is applied by restarting vpn-router-setup without waiting for it. +""" + +import ldap + +import vpnrouter +import vpnrouter_s3 +import vpnrouter_users + +from .common import USER_STATUS, section_change + +#: The HTTP status for each status of a vpnrouter_s3.Failure; any other is 500. +S3_STATUS = {2: 400, 3: 404, 4: 409} + + +def endpoint(cfg): + """The URL S3 clients call: Caddy's site of the object storage on wan.local_fqdn.""" + return f'https://{cfg["wan"]["local_fqdn"].strip()}:{cfg["s3"]["port"]}' + + +def answered(work): + """The HTTP status and the JSON document of work(), a function that answers a document; its failures as errors.""" + try: + return 200, work() + except vpnrouter_s3.Failure as failure: + return S3_STATUS.get(failure.status, 500), {'error': str(failure)} + except vpnrouter_users.Failure as failure: + return USER_STATUS.get(failure.status, 500), {'error': str(failure)} + except (ldap.LDAPError, OSError) as exc: + return 500, {'error': f'the object storage failed: {exc}'} + except SystemExit as exc: + return 500, {'error': str(exc.code)} + + +def overview(cfg): + """The document of GET /s3.""" + running = vpnrouter_s3.running() + found, problem = [], None + if running: + try: + found = vpnrouter_s3.buckets() + except vpnrouter_s3.Failure as failure: + problem = str(failure) + elif cfg['s3']['enabled']: + problem = f'{vpnrouter_s3.UNIT} does not run' + return {'settings': cfg['s3'], 'endpoint': endpoint(cfg), 'version': vpnrouter_s3.VERSION, + 'installed': vpnrouter_s3.installed(), 'running': running, 'buckets': found, + 'accounts': vpnrouter_s3.list_accounts(), 'problem': problem} + + +def s3(method, path, body): + """Answer a /s3 request: the HTTP status and the JSON document.""" + if body == 'invalid' or (body is not None and not isinstance(body, dict)): + return 400, {'error': 'the body must be a JSON object'} + body = body or {} + parts = path.strip('/').split('/') + route = (method, *parts[1:2], len(parts)) + try: + cfg = vpnrouter.load_config() + except SystemExit as exc: + return 500, {'error': str(exc.code)} + if route == ('GET', 1): + return answered(lambda: overview(cfg)) + if route == ('PUT', 1): + changed = section_change('s3', body) + return changed if isinstance(changed, tuple) else (200, {'settings': changed}) + if route == ('POST', 'buckets', 2): + return answered(lambda: vpnrouter_s3.bucket_add(body.get('name'), body.get('owner'))) + if route == ('PUT', 'buckets', 3): + return answered(lambda: vpnrouter_s3.bucket_owner(parts[2], body.get('owner'))) + if route == ('DELETE', 'buckets', 3): + return answered(lambda: vpnrouter_s3.bucket_remove(parts[2])) + if route == ('POST', 'accounts', 2): + return answered(lambda: vpnrouter_s3.account_add(body.get('owner'))) + if route == ('PUT', 'accounts', 3): + return answered(lambda: vpnrouter_s3.account_rotate(parts[2])) + if route == ('DELETE', 'accounts', 3): + return answered(lambda: vpnrouter_s3.account_remove(parts[2])) + return 404, {'error': f'no route {method} {path}'} + + +def account_s3(name, method): + """Answer a request of /account/s3 for the signed-in user: the HTTP status and the JSON document.""" + try: + cfg = vpnrouter.load_config() + except SystemExit as exc: + return 500, {'error': str(exc.code)} + if method == 'GET': + return answered(lambda: {'enabled': cfg['s3']['enabled'], 'endpoint': endpoint(cfg), + 'region': cfg['s3']['region'], + 'account': next(iter(vpnrouter_s3.list_accounts(name)), None)}) + if not cfg['s3']['enabled']: + return 409, {'error': 'the object storage is off; vpn-config-edit set s3.enabled true switches it on'} + actions = {'POST': vpnrouter_s3.account_add, 'PUT': vpnrouter_s3.account_rotate, + 'DELETE': vpnrouter_s3.account_remove} + if method not in actions: + return 405, {'error': f'{method} is not allowed on /account/s3'} + return answered(lambda: actions[method](name)) diff --git a/debian-package/src/usr/sbin/vpn-router-s3 b/debian-package/src/usr/sbin/vpn-router-s3 new file mode 100755 index 0000000..7d6979f --- /dev/null +++ b/debian-package/src/usr/sbin/vpn-router-s3 @@ -0,0 +1,137 @@ +#!/usr/bin/python3 +"""Manage the router's S3-compatible object storage from the command line, with vpnrouter_s3. + +The storage is versitygw, which runs while s3.enabled is true (vpn-config-edit set --restart s3.enabled true). Each +user or service account of the directory has at most one S3 account, whose secret is shown once, when it is made or +replaced; a bucket belongs to one account. The answer is text for people on standard output, and a failure is a +message on standard error. With --json the answer is JSON on standard output, and a failure is {"error": ...} there +too. The exit status says what failed: 2 the request is invalid, 3 the user, account or bucket does not exist, 4 it +exists or the state does not allow it, 1 anything else. -h or --help shows the commands and their options. + + vpn-router-s3 [--json] status + vpn-router-s3 [--json] account list [--owner USER] + vpn-router-s3 [--json] account add USER + vpn-router-s3 [--json] account rotate USER + vpn-router-s3 [--json] account remove USER + vpn-router-s3 [--json] bucket list + vpn-router-s3 [--json] bucket add NAME --owner USER + vpn-router-s3 [--json] bucket owner NAME USER + vpn-router-s3 [--json] bucket remove NAME +""" + +import argparse +import json +import sys + +sys.path.insert(0, '/usr/lib/vpn-router') + +import ldap + +import vpnrouter +import vpnrouter_s3 +import vpnrouter_users +from vpnrouter_s3 import Failure + + +class Parser(argparse.ArgumentParser): + """An argument parser whose errors are Failures with status 2, so that --json answers them as JSON.""" + + def error(self, message): + raise Failure(f'{message}; see {self.prog} --help', 2) + + +def parse(argv): + parser = Parser(prog='vpn-router-s3', description="Manage the router's object storage.") + parser.add_argument('--json', action='store_true', help='answer in JSON') + commands = parser.add_subparsers(dest='command', required=True, metavar='command', parser_class=Parser) + commands.add_parser('status', help='show whether the storage is on, installed and running, and its endpoint') + account = commands.add_parser('account', help="the users' S3 accounts") + accounts = account.add_subparsers(dest='action', required=True, metavar='action', parser_class=Parser) + accounts.add_parser('list', help='list the accounts').add_argument('--owner', metavar='USER', help="only this user's") + for action, text in (('add', "make the user's account and show its secret once"), + ('rotate', "replace the secret of the user's account and show it once"), + ('remove', "remove the user's account; its buckets keep it as their owner")): + accounts.add_parser(action, help=text).add_argument('user', help='the user or service account') + bucket = commands.add_parser('bucket', help='the buckets') + buckets = bucket.add_subparsers(dest='action', required=True, metavar='action', parser_class=Parser) + buckets.add_parser('list', help='list the buckets with their owners, sizes and objects') + adder = buckets.add_parser('add', help='make a bucket owned by a user with an S3 account') + adder.add_argument('name', help='the bucket name') + adder.add_argument('--owner', required=True, metavar='USER', help='the owner') + owner = buckets.add_parser('owner', help='give the bucket to another user') + owner.add_argument('name', help='the bucket name') + owner.add_argument('user', help='the new owner') + buckets.add_parser('remove', help='remove the bucket, which must be empty').add_argument('name', help='the bucket name') + return parser.parse_args(argv) + + +def run(options): + """The answer of the command the options name.""" + if options.command == 'status': + cfg = vpnrouter.load_config() + return {'enabled': cfg['s3']['enabled'], 'version': vpnrouter_s3.VERSION, 'installed': vpnrouter_s3.installed(), + 'running': vpnrouter_s3.running(), 'endpoint': f'https://{cfg["wan"]["local_fqdn"]}:{cfg["s3"]["port"]}', + 'region': cfg['s3']['region']} + if options.command == 'account': + if options.action == 'list': + return {'accounts': vpnrouter_s3.list_accounts(options.owner)} + return {'add': vpnrouter_s3.account_add, 'rotate': vpnrouter_s3.account_rotate, + 'remove': vpnrouter_s3.account_remove}[options.action](options.user) + if options.action == 'list': + return {'buckets': vpnrouter_s3.buckets()} + if options.action == 'add': + return vpnrouter_s3.bucket_add(options.name, options.owner) + if options.action == 'owner': + return vpnrouter_s3.bucket_owner(options.name, options.user) + return vpnrouter_s3.bucket_remove(options.name) + + +def table(head, rows): + """Rows as text for people, in columns under the head.""" + widths = [max(len(row[column]) for row in (head, *rows)) for column in range(len(head))] + return '\n'.join(' '.join(cell.ljust(width) for cell, width in zip(row, widths)).rstrip() for row in (head, *rows)) + + +def text_of(answer): + """The answer as text for people.""" + if 'endpoint' in answer: + state = 'running' if answer['running'] else 'not running' + return (f'Enabled: {"yes" if answer["enabled"] else "no"}\nversitygw: {answer["installed"] or "not installed"}' + f' (pinned {answer["version"]}), {state}\nEndpoint: {answer["endpoint"]}\nRegion: {answer["region"]}') + if 'secret_key' in answer: + return f'owner {answer["owner"]}\naccess key {answer["access_key"]}\nsecret key {answer["secret_key"]}' + if 'accounts' in answer: + rows = [(item['owner'], item['access_key'], item['created']) for item in answer['accounts']] + return table(('OWNER', 'ACCESS KEY', 'CREATED'), rows) if rows else 'no accounts' + if 'buckets' in answer: + rows = [(item['name'], item['owner'] or f'({item["access_key"]})', str(item['size']), str(item['objects'])) + for item in answer['buckets']] + return table(('BUCKET', 'OWNER', 'BYTES', 'OBJECTS'), rows) if rows else 'no buckets' + if answer.get('removed'): + return f'{answer.get("name") or answer.get("owner")} removed' + return f'bucket {answer["name"]} owned by {answer["owner"]}' + + +def main(argv): + # A request that cannot be parsed is answered in JSON when --json is among the arguments. + as_json = '--json' in argv + try: + options = parse(argv) + as_json = options.json + answer = run(options) + except (Failure, vpnrouter_users.Failure) as failure: + message, status = str(failure), failure.status + except (ldap.LDAPError, OSError) as exc: + message, status = f'the object storage failed: {exc}', 1 + else: + print(json.dumps(answer) if as_json else text_of(answer)) + return 0 + if as_json: + print(json.dumps({'error': message})) + else: + print(f'vpn-router-s3: {message}', file=sys.stderr) + return status + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/debian-package/src/usr/sbin/vpn-router-user b/debian-package/src/usr/sbin/vpn-router-user index e661395..bd9e18c 100755 --- a/debian-package/src/usr/sbin/vpn-router-user +++ b/debian-package/src/usr/sbin/vpn-router-user @@ -45,6 +45,7 @@ import ldap import vpnrouter import vpnrouter_auth import vpnrouter_passkeys +import vpnrouter_s3 import vpnrouter_users from vpnrouter_users import Failure @@ -201,6 +202,8 @@ def main(argv): answer = run(connection, options, secret) if options.command in vpnrouter_auth.REVOKING: vpnrouter_auth.revoke(vpnrouter.load_config(), options.name) + if options.command in vpnrouter_s3.FOLLOWING and (problem := vpnrouter_s3.follow(options.name)): + print(f'vpn-router-user: {problem}', file=sys.stderr) except Failure as failure: message, status = str(failure), failure.status except (ldap.LDAPError, OSError) as exc: diff --git a/debian-package/src/usr/share/bash-completion/completions/vpn-router-s3.bash b/debian-package/src/usr/share/bash-completion/completions/vpn-router-s3.bash new file mode 100644 index 0000000..b006b01 --- /dev/null +++ b/debian-package/src/usr/share/bash-completion/completions/vpn-router-s3.bash @@ -0,0 +1,84 @@ +# bash completion for vpn-router-s3 -*- shell-script -*- + +# The buckets and the users with or without an S3 account, read as root; nothing for other users. +_vpn_router_s3_buckets() +{ + vpn-router-s3 --json bucket list 2>/dev/null | + python3 -c 'import json, sys; print("\n".join(item["name"] for item in json.load(sys.stdin)["buckets"]))' 2>/dev/null +} + +_vpn_router_s3_owners() +{ + vpn-router-s3 --json account list 2>/dev/null | + python3 -c 'import json, sys; print("\n".join(item["owner"] for item in json.load(sys.stdin)["accounts"]))' 2>/dev/null +} + +_vpn_router_s3_users() +{ + vpn-router-user --json list 2>/dev/null | + python3 -c 'import json, sys; print("\n".join(user["name"] for user in json.load(sys.stdin)["users"]))' 2>/dev/null +} + +_vpn_router_s3() +{ + local cur prev words cword split + _init_completion -s || return + + # The command and its action are the first two words that are not options; the arguments are the words after them. + local i command='' action='' args=0 + for ((i = 1; i < cword; i++)); do + case ${words[i]} in + --owner) ((i++)) ;; + -*) ;; + *) + if [[ -z $command ]]; then + command=${words[i]} + elif [[ -z $action && $command != status ]]; then + action=${words[i]} + else + ((args++)) + fi + ;; + esac + done + + if [[ -z $command ]]; then + if [[ $cur == -* ]]; then + COMPREPLY=($(compgen -W '--json --help' -- "$cur")) + else + COMPREPLY=($(compgen -W 'status account bucket' -- "$cur")) + fi + return + fi + [[ $command == status ]] && return + if [[ -z $action ]]; then + case $command in + account) COMPREPLY=($(compgen -W 'list add rotate remove' -- "$cur")) ;; + bucket) COMPREPLY=($(compgen -W 'list add owner remove' -- "$cur")) ;; + esac + return + fi + + if [[ $prev == --owner ]]; then + COMPREPLY=($(compgen -W "$(_vpn_router_s3_owners)" -- "$cur")) + return + fi + $split && return + + if [[ $cur == -* ]]; then + local options='--help' + [[ $command/$action == @(account/list|bucket/add) ]] && options+=' --owner' + COMPREPLY=($(compgen -W "$options" -- "$cur")) + return + fi + + case $command/$action/$args in + account/add/0) COMPREPLY=($(compgen -W "$(_vpn_router_s3_users)" -- "$cur")) ;; + account/@(rotate|remove)/0) COMPREPLY=($(compgen -W "$(_vpn_router_s3_owners)" -- "$cur")) ;; + bucket/@(owner|remove)/0) COMPREPLY=($(compgen -W "$(_vpn_router_s3_buckets)" -- "$cur")) ;; + bucket/owner/1) COMPREPLY=($(compgen -W "$(_vpn_router_s3_owners)" -- "$cur")) ;; + esac +} && + complete -F _vpn_router_s3 vpn-router-s3 + +# ex: filetype=sh \ No newline at end of file diff --git a/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 b/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 index d49b698..b909cf8 100644 --- a/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 +++ b/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 @@ -84,6 +84,11 @@ http://{{ local_fqdn }} { reverse_proxy {{ loopback }}:{{ webdav_loopback_port }} } {% endif %} +{%- if s3 %} +{{ local_fqdn }}:{{ s3_port }} { + reverse_proxy {{ loopback }}:{{ s3_loopback_port }} +} +{% endif %} {%- for site in sites %} # Site {{ site.name }} {{ site.addresses | join(', ') }} {