From 38151bf5bebd7171fa8104d4068568baec725e7b Mon Sep 17 00:00:00 2001 From: Slawomir Koszewski Date: Sun, 25 Oct 2026 14:57:47 +0100 Subject: [PATCH] feature: the router keeps its metrics in PostgreSQL, serves them for an OpenTelemetry Collector and pushes them over OTLP. --- ADR.md | 58 ++- README.md | 127 +++++- debian-package/debian/changelog | 29 ++ debian-package/debian/control | 7 +- debian-package/debian/install | 3 + debian-package/debian/postrm | 5 +- debian-package/debian/prerm | 9 +- debian-package/debian/rules | 1 + .../vpn-router.vpn-router-metrics.service | 22 + debian-package/src/usr/lib/vpn-router/metrics | 414 ++++++++++++++++++ debian-package/src/usr/lib/vpn-router/setup | 98 ++++- .../src/usr/lib/vpn-router/vpnrouter.py | 47 ++ .../src/usr/lib/vpn-router/vpnrouter_ldap.py | 2 +- .../usr/lib/vpn-router/vpnrouter_metrics.py | 173 ++++++++ .../usr/lib/vpn-router/vpnrouter_status.py | 111 +++++ .../src/usr/lib/vpn-router/vpnrouter_users.py | 25 +- debian-package/src/usr/lib/vpn-router/web | 128 ++---- debian-package/src/usr/sbin/vpn-router-user | 2 +- .../doc/vpn-router/vpn-router.json.example | 15 + .../share/vpn-router/templates/Caddyfile.j2 | 5 + 20 files changed, 1153 insertions(+), 128 deletions(-) create mode 100644 debian-package/debian/vpn-router.vpn-router-metrics.service create mode 100755 debian-package/src/usr/lib/vpn-router/metrics create mode 100644 debian-package/src/usr/lib/vpn-router/vpnrouter_metrics.py create mode 100644 debian-package/src/usr/lib/vpn-router/vpnrouter_status.py diff --git a/ADR.md b/ADR.md index 5120ae6..5defe5a 100644 --- a/ADR.md +++ b/ADR.md @@ -467,8 +467,8 @@ e-mail address can be added later without changing their classes; VPN users get used. Consequences: The schema check stays strict. New attributes extend `vpnRouterAccount`, and their -OIDs continue the arcs. Nothing reads `vpnRouterScope` yet. The OID must never be reused for -anything else. +OIDs continue the arcs. The scope `service` marks a service account (ADR-23). The OID must never +be reused for anything else. ## ADR-20: Users manage their own account through /account, proving who they are again @@ -523,3 +523,57 @@ settings set it, so `internal` keeps passwords off the internet. Caddy's admin e on `localhost:2019`, which `caddy reload` uses. Port 80 and the HTTPS ports are not available to Docker containers while HTTPS is on. The Manager's HTTPS port is a setting of the router, not of the manager. + +## ADR-22: Metrics are kept in PostgreSQL and served in the Prometheus format, pushed with OTLP + +Status: Accepted + +Context: The router should keep its CPU, memory, disk, network, load and VPN client metrics for +hour, day, month and year views, as Proxmox VE does, and expose them to OpenTelemetry. Ubuntu +noble has no OpenTelemetry Python SDK and no OpenTelemetry Collector package. OTLP itself is +push-only; a collector reads a scraped endpoint in the Prometheus format with its `prometheus` +receiver. The router already runs PostgreSQL for the refresh tokens (ADR-15). + +Decision: A service of its own, `vpn-router-metrics`, takes a sample every minute with `psutil` +(`python3-psutil`) and the VPN readers it shares with the web service (`vpnrouter_status`). It +keeps the samples in the router's database, in three detail tables (`metrics_system`, +`metrics_network`, `metrics_vpn`). Each detail table has hourly, daily, monthly and yearly summary +tables, with the average and the maximum of each value per period, written when the clock passes a +period from the level below with `INSERT ... ON CONFLICT DO NOTHING`, so a stopped service catches +up. Garbage collection runs on start and once a day, not with every write, and deletes the rows +older than the retention of their level, `metrics.keep_*`, PostgreSQL intervals. The service serves +the last sample as `/metrics` in the Prometheus text format with `prometheus_client` +(`python3-prometheus-client`) and pushes it as OTLP/HTTP JSON, built with the standard library, to +`metrics.otlp_endpoint`. The names follow the OpenTelemetry semantic conventions. The web service +answers the views from the tables at `/metrics/history`, protected like `/config`, for the Router +Manager. + +Consequences: The metrics live in the database of the refresh tokens and are dropped with it on +purge. A view shows only what its level keeps, so a shorter retention empties the older part of a +view. A month view needs the hourly rows and a year view the monthly rows. `/metrics` has only the +last sample; history is not scraped. + +## ADR-23: /metrics takes HTTP Basic authentication of a service account that setup owns + +Status: Accepted + +Context: An OpenTelemetry Collector's `prometheus` receiver authenticates with HTTP Basic, not with +the router's tokens, which last minutes (ADR-3). The scraper should not be able to sign in to the +API or the Manager, and its password should be checked and locked out like any other. The account +belongs to the package's metrics, not to the people using the router. + +Decision: A service account is a Manager-only account (ADR-18) with `vpnRouterScope` `service`. +`vpn-router-setup` owns it: while metrics are enabled it adds the account `metrics.service_account` +names, `metrics` by default, with a random password it writes to `/etc/vpn-router/metrics.secret` +(root only), and sets a new random password, ending a lockout, when the file is missing or no +longer holds the password. `vpn-router-user` and `/users` cannot add one. `/auth` refuses it +with the message of a wrong password. `/metrics` accepts HTTP Basic only for that account, and +only when it is a service account; the password is checked with a bind as the account, so the +password policy's lockout applies (ADR-14). TLS is Caddy's, on `metrics.https_port` (ADR-21); +plain HTTP is served on the interfaces `metrics.listen` names, `internal` by default. + +Consequences: Every scrape binds to slapd once. The collector's operator copies the password from +the router; deleting the file and running setup changes it. A user who already has the name is not +taken over; setup fails until `metrics.service_account` names another. A disabled account stays +disabled. A scraper on the external interface over plain HTTP sends the password in the clear, so +`metrics.listen` stays `internal` unless HTTPS is used. diff --git a/README.md b/README.md index 10349c7..6f1bf04 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ A Debian package that configures a Linux host or virtual machine as a VPN router - optional WireGuard endpoint - routing and NAT for a protected subnet, TCP MSS clamping, and UFW firewall rules - firewall rules for the ports Docker publishes, when Docker is installed +- metrics of the router kept in PostgreSQL, served for an OpenTelemetry Collector and pushed + over OTLP Nothing in the package is specific to a cloud provider. It is built on the external and internal NIC names, and a `mode` setting says how much of the rest you are supplying and @@ -133,6 +135,19 @@ must have the type shown below; an unknown section or setting is an error. | `https` | `email` | string | | The e-mail address of the Let's Encrypt account | | `https` | `web_port` | integer | `7444` | HTTPS port of the status web service | | `https` | `manager_port` | integer | `7443` | HTTPS port of the Router Manager | +| `metrics` | `enabled` | boolean | `true` | The metrics service: collection, history and `/metrics` | +| `metrics` | `listen` | string | `internal` | The interfaces `/metrics` is served on: `internal`, `external` or `both` | +| `metrics` | `port` | integer | `9464` | TCP port of `/metrics` | +| `metrics` | `https_port` | integer | `9465` | HTTPS port of `/metrics`, served by Caddy while `https.enabled` is true | +| `metrics` | `service_account` | string | `metrics` | The service account `/metrics` takes, which `vpn-router-setup` adds and keeps | +| `metrics` | `otlp_endpoint` | string | | An OTLP/HTTP URL the metrics are pushed to every minute, for example `https://collector.example.com:4318/v1/metrics` | +| `metrics` | `otlp_username` | string | | The user name of HTTP Basic authentication at the OTLP endpoint | +| `metrics` | `otlp_password` | string | | Its password | +| `metrics` | `keep_details` | string | `24 hours` | How long the per-minute metrics are kept | +| `metrics` | `keep_hourly` | string | `1 month` | How long the hourly summaries are kept | +| `metrics` | `keep_daily` | string | `3 months` | How long the daily summaries are kept | +| `metrics` | `keep_monthly` | string | `1 year` | How long the monthly summaries are kept | +| `metrics` | `keep_yearly` | string | `10 years` | How long the yearly summaries are kept | A minimal site-to-site configuration in `auto` mode: @@ -352,8 +367,9 @@ is switched on: `account` for a Manager-only account (see Managing users); - the package's own schema, `cn=vpn-router,cn=schema,cn=config`, under the OID `2.25.98477352595735834532404256531620013001` of the UUID arc: the auxiliary class - `vpnRouterAccount` with `MAY ( mail $ vpnRouterScope )`, which Manager-only accounts have, and - the attribute `vpnRouterScope`, authorization scopes that nothing reads yet; + `vpnRouterAccount` with `MAY ( mail $ vpnRouterScope )`, which Manager-only and service accounts + have, and the attribute `vpnRouterScope`, authorization scopes, of which `service` marks a + service account; - `ou=groups` holds one `groupOfNames` per group, named by `cn`, with its members' DNs in `member` and an optional `description`; - `cn=user-manager,ou=services` is the account that changes users and groups; it may write @@ -375,8 +391,8 @@ The directory is kept on purge. ### Managing users -`vpn-router-user` manages the users. A user is a directory entry of one of two kinds, fixed when it -is added: +`vpn-router-user` manages the users. A user is a directory entry of one of three kinds, fixed when +it is added: - a VPN user, an `inetOrgPerson` with a first name, a last name and an e-mail address (`givenName`, `sn` and `mail`; `cn` is the first and the last name), and a client certificate @@ -384,7 +400,10 @@ is added: - a Manager-only account, an `account`, `simpleSecurityObject` and `vpnRouterAccount` with the name, the password and an optional e-mail address, for notifications: no names and no certificate, so no VPN access. It can sign in to `/auth` and, as a member of `admins`, manage - the router. `add` without the first and the last name adds one; its names cannot be set later. + the router. `add` without the first and the last name adds one; its names cannot be set later; +- a service account, a Manager-only account whose `vpnRouterScope` is `service`, which + `vpn-router-setup` adds and keeps for the metrics service (see Metrics); it cannot sign in to + `/auth`, and `vpn-router-user` and `/users` cannot add one. Passwords and TOTP secrets are read from the first line of standard input. The answer is text: a table of the users, the client profile, a TOTP secret or a short message, and a failure is a @@ -729,8 +748,8 @@ and the facts for `/config` from `/run/vpn-router/web.json` (readable by root on ## HTTPS Caddy, from Ubuntu's `caddy` package, serves the status web service on `https.web_port` (7444 by -default) and, while the Router Manager is installed, the manager on `https.manager_port` (7443), -both for `wan.local_fqdn`, while `https.enabled` is `true` and `wan.local_fqdn` is set. It obtains +default), while the Router Manager is installed the manager on `https.manager_port` (7443), and +while metrics are enabled the metrics service on `metrics.https_port` (9465), all for `wan.local_fqdn`, while `https.enabled` is `true` and `wan.local_fqdn` is set. It obtains and renews the certificate from Let's Encrypt with the HTTP challenge, so `wan.local_fqdn` must resolve to the router's public address and port 80 must be reachable from the internet; the certificate and the ACME account are kept in `/var/lib/caddy`. Port 80 answers the challenge only; @@ -753,6 +772,92 @@ the HTTPS ports while `https.enabled` is `true`. curl https://router.example.com:7444/status ``` +## Metrics + +While `metrics.enabled` is `true`, `vpn-router-setup` runs `vpn-router-metrics`, which every minute +takes a sample of the router with `psutil`: + +- the CPU utilization; +- the memory used and in total; +- the free and the total space of `/`; +- the bytes received and sent per second on the external and the internal interface; +- the 1, 5 and 15 minute load averages; +- the VPN clients of each enabled VPN system: road-warrior (IPsec P2S) and OpenVPN clients, + WireGuard peers with a handshake in the last 3 minutes, and the site-to-site tunnel, 1 while it is + up. + +### History + +The samples are kept in the router's PostgreSQL database, the one of the refresh tokens +(`web.database`), in three detail tables, one row per minute, and each is summarized hourly, daily, +monthly and yearly. A summary row has the start of its period, in UTC, and the average and the +maximum of each value: + +| Level | Tables | Kept | +|---|---|---| +| Details | `metrics_system`, `metrics_network`, `metrics_vpn` | `metrics.keep_details` | +| Hourly | `metrics_system_hourly`, `metrics_network_hourly`, `metrics_vpn_hourly` | `metrics.keep_hourly` | +| Daily | `metrics_system_daily`, `metrics_network_daily`, `metrics_vpn_daily` | `metrics.keep_daily` | +| Monthly | `metrics_system_monthly`, `metrics_network_monthly`, `metrics_vpn_monthly` | `metrics.keep_monthly` | +| Yearly | `metrics_system_yearly`, `metrics_network_yearly`, `metrics_vpn_yearly` | `metrics.keep_yearly` | + +How it works: + +- **Summaries:** a period is summarized once it is complete, from the level below. A stopped service catches up on the periods it missed, as long as their rows are still kept. +- **Garbage collection:** runs on start and once a day. It deletes the rows older than their retention, a count and a unit such as `24 hours`, `3 months` or `10 years`. +- **History API:** `GET /metrics/history?range=hour|day|month|year` of the status web service, protected like `/config`, answers the views of the Router Manager's Metrics tab: + + | Range | Bucket | Source | + |---|---|---| + | `hour` | 1 minute | details | + | `day` | 10 minutes | details | + | `month` | 6 hours | hourly | + | `year` | 1 month | monthly | + +### Scraping + +`GET /metrics` of the metrics service serves the last sample in the Prometheus text format, written by `prometheus_client`. + +- **Metric names:** the OpenTelemetry semantic convention names in their Prometheus form: + - `system_cpu_utilization`; + - `system_memory_usage_bytes` and `system_memory_limit_bytes`; + - `system_filesystem_usage_bytes` and `system_filesystem_utilization`; + - `system_network_io_bytes_total`; + - `system_cpu_load_average_1m`, `_5m` and `_15m`; + - `vpn_router_clients`. +- **Listening:** on the interfaces `metrics.listen` names, `internal` by default, at `metrics.port`, and on `127.0.0.1`. While HTTPS is on, Caddy serves it at `metrics.https_port` for `wan.local_fqdn`. `vpn-router-setup` opens both ports on the internal and the external interface. +- **Authentication:** HTTP Basic as the service account `metrics.service_account` names, `metrics` by default. The password is checked with a bind to the user directory, so 5 failed attempts lock the account as for any user. +- **The account:** `vpn-router-setup` owns it while metrics are enabled. + - It adds the account with a random password, which it writes to `/etc/vpn-router/metrics.secret` (root only). Give that password to the collector. + - When the file is missing or no longer holds the account's password, setup sets a new random password and ends any lockout. Delete the file and restart `vpn-router-setup` to change the password. + - A disabled account is left as it is. + - A user of that name who is not a service account is a setup failure. + +An OpenTelemetry Collector scrapes it with its `prometheus` receiver: + +```yaml +receivers: + prometheus: + config: + scrape_configs: + - job_name: vpn-router + scrape_interval: 60s + scheme: https + basic_auth: + username: metrics + password_file: /etc/otelcol/vpn-router-metrics.secret # a copy of /etc/vpn-router/metrics.secret + static_configs: + - targets: ['router.example.com:9465'] +``` + +### Pushing + +When `metrics.otlp_endpoint` is set, every sample is also pushed there as an OTLP/HTTP JSON request. + +- **Content:** the semantic convention names (`system.cpu.utilization`, `system.network.io` and so on), and the resource attributes `service.name` `vpn-router` and `host.name`. +- **Authentication:** HTTP Basic with `metrics.otlp_username` and `metrics.otlp_password` when they are set. +- **Failures:** a failed push is logged and not repeated; the next sample is pushed a minute later. + ## Router Manager The `vpn-router-manager` package, built in the repository `linux-cloud-router-manager`, is a web @@ -766,15 +871,17 @@ interfaces it listens on. ## Removal `apt-get remove` stops `vpn-router-docker`, which removes the Docker rules. It terminates the -tunnels and takes `wg-quick@wg0`, `openvpn-server@vpn-router` and `caddy` down, but leaves the +tunnels and takes `wg-quick@wg0`, `openvpn-server@vpn-router`, `caddy` and `vpn-router-metrics` +down, but leaves the configuration in place. `strongswan` is left enabled, since it is a shared service that may be in use by something else. `apt-get purge` additionally removes the generated files and the firewall rules, restoring `/etc/ufw/before.rules` to its original content, and hands `vpn-router.json` back to `ucf` before deleting it so a later reinstall starts clean. It also removes the web service's -signing key, and the refresh tokens' database, its role and their password file when the package -created them, and Caddy's configuration `/etc/caddy/vpn-router.Caddyfile`; the certificate in +signing key, the password file of the metrics service account, and the database of the refresh +tokens and the metrics, its role and their password +file when the package created them, and Caddy's configuration `/etc/caddy/vpn-router.Caddyfile`; the certificate in `/var/lib/caddy` belongs to the `caddy` package. Key material and users are never deleted: `/etc/vpn-router/pki`, `/etc/vpn-router/ldap`, the user directory in slapd, the WireGuard key pair and everything under `/etc/swanctl` are left alone. diff --git a/debian-package/debian/changelog b/debian-package/debian/changelog index 3905a91..df43874 100644 --- a/debian-package/debian/changelog +++ b/debian-package/debian/changelog @@ -1,3 +1,32 @@ +vpn-router (0.16.0-1) unstable; urgency=medium + + feature: the router keeps its metrics in PostgreSQL, serves them for an OpenTelemetry Collector and pushes them over OTLP. + + * vpn-router-metrics takes a sample every minute: CPU, memory, free + space of /, bytes per second on the external and the internal + interface, the load averages, and the VPN clients of each enabled + system. + * The samples are kept in metrics_system, metrics_network and + metrics_vpn, and summarized hourly, daily, monthly and yearly in + their _hourly, _daily, _monthly and _yearly tables; once a day the + rows older than metrics.keep_details, keep_hourly, keep_daily, + keep_monthly and keep_yearly are deleted. + * /metrics serves the last sample in the Prometheus format on the + interfaces metrics.listen names (internal by default) at + metrics.port, and over HTTPS by Caddy at metrics.https_port, with + HTTP Basic authentication of the service account + metrics.service_account names (metrics by default). + * metrics.otlp_endpoint pushes every sample as OTLP/HTTP JSON, with + metrics.otlp_username and metrics.otlp_password. + * /metrics/history of the web service answers the hour, day, month + and year views for the Router Manager. + * vpn-router-setup adds and keeps that service account, which cannot + sign in to /auth, with a random password in + /etc/vpn-router/metrics.secret, set anew when the file is missing. + * Depends on python3-psutil and python3-prometheus-client. + + -- Slawomir Koszewski Sun, 25 Oct 2026 19:00:00 +0200 + vpn-router (0.15.0-1) unstable; urgency=medium feature: vpn-config-edit shows and changes the router's configuration. diff --git a/debian-package/debian/control b/debian-package/debian/control index e586dae..db289cd 100644 --- a/debian-package/debian/control +++ b/debian-package/debian/control @@ -23,6 +23,8 @@ Depends: ${misc:Depends}, python3-ldap, python3-pyotp, python3-psycopg, + python3-psutil, + python3-prometheus-client, postgresql, slapd, openvpn, @@ -35,8 +37,9 @@ Description: Linux VPN router with IPSec, road-warrior, OpenVPN and WireGuard su IKEv2 IPSec (strongSwan swanctl), optional road-warrior access using EAP-TLS, optional OpenVPN with a certificate and a password per user, kept in a local LDAP directory, optional WireGuard, and the routing, NAT, MSS clamping and - firewall rules a router needs. Nothing in the package is specific to a cloud - provider. + firewall rules a router needs. It keeps metrics of the router in PostgreSQL + and serves them for an OpenTelemetry Collector. Nothing in the package is + specific to a cloud provider. . Settings live in /etc/vpn-router/vpn-router.json, which is created on first install and owned by the administrator afterwards. Editing it and restarting diff --git a/debian-package/debian/install b/debian-package/debian/install index 9c91392..7d60082 100644 --- a/debian-package/debian/install +++ b/debian-package/debian/install @@ -5,6 +5,8 @@ src/usr/lib/vpn-router/vpnrouter_net.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_serve.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_users.py usr/lib/vpn-router/ src/usr/lib/vpn-router/vpnrouter_auth.py usr/lib/vpn-router/ +src/usr/lib/vpn-router/vpnrouter_metrics.py usr/lib/vpn-router/ +src/usr/lib/vpn-router/vpnrouter_status.py usr/lib/vpn-router/ src/usr/lib/vpn-router/openvpn-same-user usr/lib/vpn-router/ src/usr/sbin/vpn-router-user usr/sbin/ src/usr/sbin/vpn-router-group usr/sbin/ @@ -15,6 +17,7 @@ src/usr/lib/vpn-router/configure usr/lib/vpn-router/ src/usr/lib/vpn-router/docker-rules usr/lib/vpn-router/ src/usr/lib/vpn-router/setup usr/lib/vpn-router/ src/usr/lib/vpn-router/web usr/lib/vpn-router/ +src/usr/lib/vpn-router/metrics usr/lib/vpn-router/ src/usr/lib/vpn-router/bootstrap usr/lib/vpn-router/ src/usr/lib/vpn-router/wait-for-libpq usr/lib/vpn-router/ src/usr/lib/vpn-router/generate-config usr/lib/vpn-router/ diff --git a/debian-package/debian/postrm b/debian-package/debian/postrm index 37a1d16..1e3a6b0 100755 --- a/debian-package/debian/postrm +++ b/debian-package/debian/postrm @@ -11,10 +11,10 @@ strip_ufw_blocks() { tmp="$(mktemp)" awk ' - /^# (IPSEC|WIREGUARD|OPENVPN|P2S DNS|WEB|BOOTSTRAP|MANAGER|HTTPS|ROUTER FORWARD|ROUTER NAT|ROUTER MSS) RULES START$/ { + /^# (IPSEC|WIREGUARD|OPENVPN|P2S DNS|WEB|BOOTSTRAP|MANAGER|HTTPS|METRICS|ROUTER FORWARD|ROUTER NAT|ROUTER MSS) RULES START$/ { skip = 1; pending = 0; next } - /^# (IPSEC|WIREGUARD|OPENVPN|P2S DNS|WEB|BOOTSTRAP|MANAGER|HTTPS|ROUTER FORWARD|ROUTER NAT|ROUTER MSS) RULES END$/ { + /^# (IPSEC|WIREGUARD|OPENVPN|P2S DNS|WEB|BOOTSTRAP|MANAGER|HTTPS|METRICS|ROUTER FORWARD|ROUTER NAT|ROUTER MSS) RULES END$/ { skip = 0; next } skip { next } @@ -52,6 +52,7 @@ case "$1" in /etc/caddy/vpn-router.Caddyfile \ /etc/vpn-router/vpn-router.json \ /etc/vpn-router/web.key \ + /etc/vpn-router/metrics.secret \ /var/lib/vpn-router/setup.done rmdir --ignore-fail-on-non-empty /var/lib/vpn-router 2>/dev/null || true diff --git a/debian-package/debian/prerm b/debian-package/debian/prerm index 7e229da..721d034 100755 --- a/debian-package/debian/prerm +++ b/debian-package/debian/prerm @@ -11,10 +11,11 @@ case "$1" in swanctl --terminate --ike road-warrior >/dev/null 2>&1 || true fi - # This package enabled wg-quick@wg0, openvpn-server@vpn-router and - # caddy, so it takes them down again. Use the debhelper wrappers rather - # than systemctl, so the script behaves on a machine without systemd. - for unit in wg-quick@wg0.service openvpn-server@vpn-router.service caddy.service; do + # vpn-router-setup enabled wg-quick@wg0, openvpn-server@vpn-router, + # caddy and vpn-router-metrics, so the package takes them down again. + # Use the debhelper wrappers rather than systemctl, so the script + # behaves on a machine without systemd. + for unit in wg-quick@wg0.service openvpn-server@vpn-router.service caddy.service vpn-router-metrics.service; do if command -v deb-systemd-invoke >/dev/null 2>&1; then deb-systemd-invoke stop "$unit" >/dev/null 2>&1 || true fi diff --git a/debian-package/debian/rules b/debian-package/debian/rules index ba17514..28e83bf 100755 --- a/debian-package/debian/rules +++ b/debian-package/debian/rules @@ -6,4 +6,5 @@ override_dh_installsystemd: dh_installsystemd --name=vpn-router-setup dh_installsystemd --name=vpn-router-docker dh_installsystemd --name=vpn-router-web --no-start + dh_installsystemd --name=vpn-router-metrics --no-enable --no-start dh_installsystemd --name=vpn-router-bootstrap --no-enable --no-start diff --git a/debian-package/debian/vpn-router.vpn-router-metrics.service b/debian-package/debian/vpn-router.vpn-router-metrics.service new file mode 100644 index 0000000..9d75aa4 --- /dev/null +++ b/debian-package/debian/vpn-router.vpn-router-metrics.service @@ -0,0 +1,22 @@ +[Unit] +Description=VPN Router metrics service +Documentation=file:/usr/share/doc/vpn-router/README.md +After=network-online.target postgresql.service vpn-router-web.service +Wants=network-online.target + +[Service] +Type=simple +ExecStartPre=/usr/lib/vpn-router/wait-for-libpq +ExecStart=/usr/bin/python3 /usr/lib/vpn-router/metrics +TimeoutStartSec=5min +Restart=on-failure +RestartSec=15 +NoNewPrivileges=yes +ProtectSystem=strict +RuntimeDirectory=vpn-router +RuntimeDirectoryPreserve=yes +ProtectHome=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/debian-package/src/usr/lib/vpn-router/metrics b/debian-package/src/usr/lib/vpn-router/metrics new file mode 100755 index 0000000..8d29b40 --- /dev/null +++ b/debian-package/src/usr/lib/vpn-router/metrics @@ -0,0 +1,414 @@ +#!/usr/bin/python3 +"""Collect the metrics of the router, keep them in PostgreSQL, serve them for scraping and push them with OTLP. + +Run by vpn-router-metrics.service while metrics.enabled is true; vpn-router-setup enables and +disables the unit. + +Every SAMPLE_SECONDS, on the minute, a sample is taken with psutil: the CPU utilization since the +previous sample, the memory used and in total, the free and the total space of /, the bytes +received and sent per second on the external and the internal interface, and the load averages; +and the VPN clients of each enabled VPN system (vpnrouter_status): road-warrior (p2s) and +OpenVPN clients, WireGuard peers with a recent handshake, and the site-to-site tunnel, 1 while it +is up. The interfaces and the enabled systems are those vpn-router-setup resolved, read from +/run/vpn-router/web.json. + +The sample is written to the detail tables of vpnrouter_metrics. When the clock has passed into +a new hour, day, month or year, the complete periods are summarized, and once a day, after the +daily summary, the rows older than their retention are deleted; both also run on start. A +failure of the database is logged and tried again with the next sample. + +GET /metrics answers the last sample in the Prometheus text format, written by +prometheus_client, with OpenTelemetry semantic convention names in their Prometheus form, for an +OpenTelemetry Collector's prometheus receiver. It needs HTTP Basic authentication as the service +account metrics.service_account names, which vpn-router-setup keeps with the password in +/etc/vpn-router/metrics.secret, checked with a bind to the router's directory, so failed attempts +lock the account as for any user. The service listens on the interfaces metrics.listen +names and on the loopback address, where Caddy serves it over HTTPS at metrics.https_port; the +address of a call through Caddy is the last entry of its X-Forwarded-For header. + +When metrics.otlp_endpoint is set, every sample is also pushed there as an OTLP/HTTP JSON +ExportMetricsServiceRequest, with HTTP Basic authentication when metrics.otlp_username is set. +""" + +import base64 +import datetime +import json +import os +import signal +import socket +import sys +import threading +import time +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) + +import ldap +import psutil +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest +from prometheus_client.core import CollectorRegistry, CounterMetricFamily, GaugeMetricFamily + +import vpnrouter +import vpnrouter_metrics +import vpnrouter_net +import vpnrouter_serve +import vpnrouter_status +import vpnrouter_users + +STATE_FILE = '/run/vpn-router/web.json' +CONFIG_FILE = '/etc/vpn-router/vpn-router.json' + +SAMPLE_SECONDS = 60 + +#: Seconds between two discoveries of where the service should listen. +DISCOVERY_SECONDS = 30 + +#: Seconds an OTLP push may take. +OTLP_TIMEOUT = 10 + +#: The mount point whose space is measured. +MOUNTPOINT = '/' + +#: The last sample, as sample() makes it, or None before the first. +_latest = {'sample': None} +_latest_lock = threading.Lock() + + +def log(message): + print(f'vpn-router-metrics: {message}', file=sys.stderr, flush=True) + + +def read_json(path): + """The object in a JSON file, or {} when the file is missing or holds no object.""" + try: + with open(path, encoding='utf-8') as handle: + data = json.load(handle) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def section(configuration, name): + """A section of the configuration, or {} when it is missing or not an object.""" + value = configuration.get(name) + return value if isinstance(value, dict) else {} + + +def clients(state): + """The VPN clients of each enabled VPN system, None where it cannot be asked.""" + counts = {} + if state.get('p2s'): + counts['p2s'] = vpnrouter_status.established('road-warrior') + if state.get('openvpn'): + counts['openvpn'] = vpnrouter_status.openvpn_clients() + if state.get('wireguard'): + counts['wireguard'] = vpnrouter_status.wireguard_connected() + if state.get('site_to_site'): + count = vpnrouter_status.established('remote-site') + counts['site_to_site'] = None if count is None else int(count > 0) + return counts + + +def sample(previous): + """A sample of the router, with the rates since the previous sample, None for the first.""" + state = read_json(STATE_FILE) + network = section(state, 'network') + names = [name for name in dict.fromkeys((network.get('external_interface'), network.get('internal_interface'))) + if isinstance(name, str) and name] + counters = psutil.net_io_counters(pernic=True) + memory = psutil.virtual_memory() + disk = psutil.disk_usage(MOUNTPOINT) + load1, load5, load15 = os.getloadavg() + now = time.time() + current = { + 'time': now, + 'monotonic': time.monotonic(), + 'system': { + 'cpu': psutil.cpu_percent(interval=None), + 'memory_used': memory.total - memory.available, + 'memory_total': memory.total, + 'disk_free': disk.free, + 'disk_total': disk.total, + 'load1': load1, + 'load5': load5, + 'load15': load15, + }, + 'counters': {name: (counters[name].bytes_recv, counters[name].bytes_sent) for name in names if name in counters}, + 'clients': clients(state), + 'rates': {}, + } + if previous is not None: + elapsed = current['monotonic'] - previous['monotonic'] + for name, (received, sent) in current['counters'].items(): + if name in previous['counters'] and elapsed > 0: + before_received, before_sent = previous['counters'][name] + current['rates'][name] = (max(received - before_received, 0) / elapsed, max(sent - before_sent, 0) / elapsed) + return current + + +def periods(moment): + """The hour, the day, the month and the year of a UTC moment, keyed by the levels of vpnrouter_metrics.LEVELS.""" + return {'hourly': moment.strftime('%Y-%m-%dT%H'), 'daily': moment.strftime('%Y-%m-%d'), + 'monthly': moment.strftime('%Y-%m'), 'yearly': moment.strftime('%Y')} + + +def store(cfg, current, done): + """Write the sample, summarize the periods the clock has passed and collect the garbage once a day. + + done holds the periods summarized so far, and garbage the day of the last garbage collection; + it is updated only for what succeeded. + """ + now = periods(datetime.datetime.now(datetime.timezone.utc)) + with vpnrouter_metrics.database(cfg) as connection: + vpnrouter_metrics.write(connection, current['system'], current['rates'], current['clients']) + for level, _, _ in vpnrouter_metrics.LEVELS: + if done.get(level) != now[level]: + vpnrouter_metrics.summarize(connection, level) + done[level] = now[level] + if done.get('garbage') != now['daily']: + vpnrouter_metrics.collect_garbage(connection, cfg) + done['garbage'] = now['daily'] + + +def attributes(**values): + """OTLP attributes of string values.""" + return [{'key': key, 'value': {'stringValue': value}} for key, value in values.items()] + + +def otlp_request(current, started): + """The OTLP/HTTP JSON ExportMetricsServiceRequest of a sample; started is when the counters were first read, in Unix seconds.""" + stamp = str(int(current['time'] * 1e9)) + start = str(int(started * 1e9)) + system = current['system'] + + def gauge(name, unit, points): + return {'name': name, 'unit': unit, 'gauge': {'dataPoints': [ + {'timeUnixNano': stamp, 'asDouble': float(value), 'attributes': attributes(**labels)} + for value, labels in points if value is not None]}} + + metrics = [ + gauge('system.cpu.utilization', '1', [(system['cpu'] / 100, {})]), + gauge('system.memory.usage', 'By', [(system['memory_used'], {'state': 'used'})]), + gauge('system.memory.limit', 'By', [(system['memory_total'], {})]), + gauge('system.filesystem.usage', 'By', [(system['disk_free'], {'mountpoint': MOUNTPOINT, 'state': 'free'}), + (system['disk_total'] - system['disk_free'], {'mountpoint': MOUNTPOINT, 'state': 'used'})]), + gauge('system.filesystem.utilization', '1', + [(1 - system['disk_free'] / system['disk_total'] if system['disk_total'] else None, {'mountpoint': MOUNTPOINT})]), + gauge('system.cpu.load_average.1m', '{thread}', [(system['load1'], {})]), + gauge('system.cpu.load_average.5m', '{thread}', [(system['load5'], {})]), + gauge('system.cpu.load_average.15m', '{thread}', [(system['load15'], {})]), + gauge('vpn_router.clients', '{client}', [(count, {'kind': kind}) for kind, count in current['clients'].items()]), + {'name': 'system.network.io', 'unit': 'By', 'sum': { + 'aggregationTemporality': 2, 'isMonotonic': True, 'dataPoints': [ + {'startTimeUnixNano': start, 'timeUnixNano': stamp, 'asInt': str(value), + 'attributes': attributes(device=name, direction=direction)} + for name, counters in current['counters'].items() + for value, direction in zip(counters, ('receive', 'transmit'))]}}, + ] + return {'resourceMetrics': [{ + 'resource': {'attributes': attributes(**{'service.name': 'vpn-router', 'host.name': socket.gethostname()})}, + 'scopeMetrics': [{'scope': {'name': 'vpn-router-metrics'}, 'metrics': metrics}], + }]} + + +def push(metrics, current, started): + """Push the sample to metrics.otlp_endpoint; raises OSError when the collector does not take it.""" + request = urllib.request.Request(metrics['otlp_endpoint'], data=json.dumps(otlp_request(current, started)).encode(), + headers={'Content-Type': 'application/json'}, method='POST') + if metrics['otlp_username']: + credentials = f"{metrics['otlp_username']}:{metrics['otlp_password']}".encode() + request.add_header('Authorization', 'Basic ' + base64.b64encode(credentials).decode()) + with urllib.request.urlopen(request, timeout=OTLP_TIMEOUT) as response: + response.read() + + +def collector_loop(): + """Take a sample on every minute, store it and push it; every new failure is logged once.""" + previous, done, reported = None, {}, {} + started = time.time() + psutil.cpu_percent(interval=None) + + def report(kind, message): + if message != reported.get(kind): + if message: + log(message) + elif reported.get(kind): + log(f'{kind} works again') + reported[kind] = message + + while True: + time.sleep(SAMPLE_SECONDS - time.time() % SAMPLE_SECONDS) + try: + cfg = vpnrouter.load_config() + except SystemExit as exc: + report('configuration', str(exc.code)) + continue + report('configuration', '') + current = sample(previous) + previous = current + with _latest_lock: + _latest['sample'] = current + try: + store(cfg, current, done) + report('database', '') + except vpnrouter_metrics.DatabaseError as exc: + report('database', str(exc)) + metrics = cfg['metrics'] + if metrics['otlp_endpoint']: + try: + push(metrics, current, started) + report('OTLP push', '') + except (OSError, ValueError) as exc: + report('OTLP push', f"pushing to {metrics['otlp_endpoint']} failed: {exc}") + + +class Sample: + """A prometheus_client collector of a sample.""" + + def __init__(self, current): + self.current = current + + def collect(self): + system = self.current['system'] + yield GaugeMetricFamily('system_cpu_utilization', 'CPU utilization since the previous sample, 0 to 1', + value=system['cpu'] / 100) + memory = GaugeMetricFamily('system_memory_usage_bytes', 'Memory in use', labels=['state']) + memory.add_metric(['used'], system['memory_used']) + yield memory + yield GaugeMetricFamily('system_memory_limit_bytes', 'Total memory', value=system['memory_total']) + usage = GaugeMetricFamily('system_filesystem_usage_bytes', 'Space of the file system', labels=['mountpoint', 'state']) + usage.add_metric([MOUNTPOINT, 'free'], system['disk_free']) + usage.add_metric([MOUNTPOINT, 'used'], system['disk_total'] - system['disk_free']) + yield usage + if system['disk_total']: + utilization = GaugeMetricFamily('system_filesystem_utilization', 'Used part of the file system, 0 to 1', + labels=['mountpoint']) + utilization.add_metric([MOUNTPOINT], 1 - system['disk_free'] / system['disk_total']) + yield utilization + for period in ('1m', '5m', '15m'): + yield GaugeMetricFamily(f'system_cpu_load_average_{period}', f'Load average over {period}', + value=system[f'load{period[:-1]}']) + network = CounterMetricFamily('system_network_io_bytes', 'Bytes received and sent', labels=['device', 'direction']) + for name, (received, sent) in self.current['counters'].items(): + network.add_metric([name, 'receive'], received) + network.add_metric([name, 'transmit'], sent) + yield network + vpn = GaugeMetricFamily('vpn_router_clients', 'Connected VPN clients; site_to_site is 1 while the tunnel is up', + labels=['kind']) + for kind, count in self.current['clients'].items(): + if count is not None: + vpn.add_metric([kind], count) + yield vpn + + +def authorized(header, account): + """Whether an Authorization header holds the password of the service account; LDAP errors propagate.""" + if not header.startswith('Basic '): + return False + try: + name, _, password = base64.b64decode(header[6:].strip(), validate=True).decode().partition(':') + except ValueError: + return False + if name != account: + return False + try: + with vpnrouter_users.session() as connection: + if not vpnrouter_users.service(vpnrouter_users.entry(connection, name)): + return False + vpnrouter_users.login(name, password, None) + except vpnrouter_users.Failure: + return False + return True + + +class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def client(self): + """The client address: the last X-Forwarded-For entry of a call through Caddy on the loopback address, else the peer.""" + peer = self.client_address[0] + forwarded = self.headers.get('X-Forwarded-For', '') + if peer == vpnrouter_net.LOOPBACK and forwarded.strip(): + return forwarded.split(',')[-1].strip() + return peer + + def answer(self, send_body): + self.principal = None + extra = [] + path = self.path.split('?', 1)[0] + account = str(section(read_json(CONFIG_FILE), 'metrics').get('service_account', + vpnrouter.SCHEMA['metrics']['service_account'])).strip() + with _latest_lock: + current = _latest['sample'] + if path != '/metrics': + body, kind, code = b'Not found\n', 'text/plain; charset=utf-8', 404 + elif not account: + body, kind, code = b'metrics.service_account is not set\n', 'text/plain; charset=utf-8', 503 + else: + try: + allowed = authorized(self.headers.get('Authorization', ''), account) + except (ldap.LDAPError, OSError) as exc: + allowed = None + body, kind, code = f'the user directory failed: {exc}\n'.encode(), 'text/plain; charset=utf-8', 500 + if allowed is False: + body, kind, code = b'Unauthorized\n', 'text/plain; charset=utf-8', 401 + extra.append(('WWW-Authenticate', 'Basic realm="vpn-router metrics", charset="UTF-8"')) + elif allowed: + self.principal = account + if current is None: + body, kind, code = b'no sample yet\n', 'text/plain; charset=utf-8', 503 + else: + registry = CollectorRegistry() + registry.register(Sample(current)) + body, kind, code = generate_latest(registry), CONTENT_TYPE_LATEST, 200 + self.send_response(code) + self.send_header('Content-Type', kind) + self.send_header('Content-Length', str(len(body))) + self.send_header('Cache-Control', 'no-store') + for name, value in extra: + self.send_header(name, value) + self.end_headers() + if send_body: + self.wfile.write(body) + + def do_GET(self): + self.answer(True) + + def do_HEAD(self): + self.answer(False) + + def log_message(self, format, *args): + principal = getattr(self, 'principal', None) or '-' + sys.stderr.write('%s - %s [%s] %s\n' % (self.client(), principal, self.log_date_time_string(), format % args)) + + +def wanted(): + """Where the service listens: the addresses of the interfaces metrics.listen names and LOOPBACK, at metrics.port.""" + configuration = read_json(CONFIG_FILE) + metrics = section(configuration, 'metrics') + defaults = vpnrouter.SCHEMA['metrics'] + if metrics.get('enabled', defaults['enabled']) is False: + return [], 'metrics are disabled in the configuration' + port = metrics.get('port', defaults['port']) + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + port = defaults['port'] + addresses = vpnrouter_net.listen_addresses(configuration, metrics.get('listen', defaults['listen'])) + if not addresses: + return [], 'the addresses to listen on are not known yet' + return [(address, port) for address in [*addresses, vpnrouter_net.LOOPBACK]], '' + + +def main(): + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + threading.Thread(target=collector_loop, name='collector', daemon=True).start() + try: + vpnrouter_serve.serve(wanted, Handler, log, DISCOVERY_SECONDS) + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/debian-package/src/usr/lib/vpn-router/setup b/debian-package/src/usr/lib/vpn-router/setup index c42400d..004bf0e 100755 --- a/debian-package/src/usr/lib/vpn-router/setup +++ b/debian-package/src/usr/lib/vpn-router/setup @@ -21,6 +21,7 @@ import json import os import pathlib import re +import secrets import socket import subprocess import sys @@ -59,6 +60,9 @@ MANAGER_CONFIG = pathlib.Path('/etc/vpn-router-manager/manager.json') MANAGER_ADMIN = pathlib.Path('/etc/vpn-router-manager/admin.json') MANAGER_PORT = 7080 +#: The password of the metrics service account, which setup makes and keeps (root only). +METRICS_SECRET = pathlib.Path('/etc/vpn-router/metrics.secret') + #: The configuration of Caddy, the HTTPS proxy, which the package's drop-in for caddy.service names; #: Caddy runs as the user and group CADDY_GROUP, which the caddy package creates. CADDYFILE = pathlib.Path('/etc/caddy/vpn-router.Caddyfile') @@ -79,6 +83,7 @@ BLOCKS = ( ('# BOOTSTRAP RULES START', '# BOOTSTRAP RULES END'), ('# MANAGER RULES START', '# MANAGER RULES END'), ('# HTTPS RULES START', '# HTTPS RULES END'), + ('# METRICS RULES START', '# METRICS RULES END'), ('# ROUTER FORWARD RULES START', '# ROUTER FORWARD RULES END'), ('# ROUTER NAT RULES START', '# ROUTER NAT RULES END'), ('# ROUTER MSS RULES START', '# ROUTER MSS RULES END'), @@ -223,7 +228,7 @@ def filter_block(ctx): ] lines += ['# P2S DNS RULES END'] - # The web service and the Router Manager choose their interfaces themselves, so their ports are open on both. + # The web service, the Router Manager and the metrics service choose their interfaces themselves, so their ports are open on both. interfaces = [name for name in (ctx['int_iface'], ctx['wan_iface']) if name] if ctx['web_enabled'] and interfaces: lines += [ @@ -250,10 +255,19 @@ def filter_block(ctx): '# MANAGER RULES END', ] + if vpnrouter.metrics_ready(ctx) and interfaces: + lines += [ + '', + '# METRICS RULES START', + *(f'-A ufw-before-input -i {name} -p tcp --dport {ctx["metrics_port"]} -j ACCEPT' for name in interfaces), + '# METRICS RULES END', + ] + # Let's Encrypt reaches the HTTP challenge on the external interface; the HTTPS ports are open on both. if vpnrouter.https_ready(ctx) and interfaces: ports = [port for wanted, port in ((ctx['web_enabled'], ctx['https_web_port']), - (MANAGER.exists(), ctx['https_manager_port'])) if wanted] + (MANAGER.exists(), ctx['https_manager_port']), + (vpnrouter.metrics_ready(ctx), ctx['metrics_https_port'])) if wanted] lines += [ '', '# HTTPS RULES START', @@ -439,6 +453,57 @@ def manage_admin(): return '' +def manage_metrics_account(ctx): + """Keep the service account metrics.service_account names, whose password is in METRICS_SECRET; the problem, or ''. + + While metrics are enabled, setup owns the account: it adds it as a service account with a random password when + it is missing, and gives it a new random password, ending any lockout, when METRICS_SECRET is missing or does not + hold its password. A disabled account is left as it is. An account of that name that is not a service account is + left alone and is a problem. + """ + if not vpnrouter.metrics_ready(ctx): + return '' + name = ctx['metrics_service_account'] + try: + vpnrouter_users.check_name(name) + try: + password = METRICS_SECRET.read_text(encoding='utf-8').strip() + except OSError: + password = '' + with vpnrouter_users.session() as connection: + try: + current = vpnrouter_users.entry(connection, name) + except vpnrouter_users.Failure as failure: + if failure.status != 3: + raise + current = None + if current is not None and not vpnrouter_users.service(current): + return f'metrics.service_account: {name} is a user, not a service account' + # An administrator who disabled the account meant it; its password is left as it is. + if current is not None and not vpnrouter_users.summary(current)['enabled']: + return '' + valid = False + if current is not None and password: + try: + vpnrouter_users.login(name, password, None) + valid = True + except vpnrouter_users.Failure: + pass + if not valid: + password = secrets.token_urlsafe(32) + METRICS_SECRET.touch(mode=0o600) + METRICS_SECRET.chmod(0o600) + METRICS_SECRET.write_text(password + '\n', encoding='utf-8') + if current is None: + vpnrouter_users.add(connection, name, password, {}, service=True) + else: + vpnrouter_users.passwd(connection, name, password) + vpnrouter_users.unlock(connection, name) + except (OSError, ldap.LDAPError, vpnrouter_users.Failure) as exc: + return f'the metrics service account cannot be kept: {exc}' + return '' + + def manage_openvpn(ctx, changed): """Run the OpenVPN server when it is configured, and stop it when it is not.""" unit = 'openvpn-server@vpn-router' @@ -455,11 +520,12 @@ def manage_openvpn(ctx, changed): def manage_caddy(ctx): - """Run Caddy, the HTTPS proxy of the web service and the Router Manager, while HTTPS is ready, and stop it when not. + """Run Caddy, the HTTPS proxy of the web service, the Router Manager and the metrics service, while HTTPS is ready, and stop it when not. - CADDYFILE serves the web service on https.web_port and, while the manager is installed, the - manager on https.manager_port, both for wan.local_fqdn with a certificate from Let's Encrypt, - and both calling the service on its loopback address. + CADDYFILE serves the web service on https.web_port, while the manager is installed the + manager on https.manager_port, and while metrics are enabled the metrics service on + metrics.https_port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all + calling the service on its loopback address. """ if not vpnrouter.https_ready(ctx): vpnrouter.remove(CADDYFILE) @@ -480,6 +546,20 @@ def manage_caddy(ctx): run(['systemctl', 'reload', CADDY_UNIT], check=False) +def manage_metrics(ctx): + """Run the metrics service while metrics are enabled, and stop it when not. + + The service reads its settings itself, so it is restarted only to run the code of a new package version, which + every run of setup after an upgrade does. + """ + unit = 'vpn-router-metrics.service' + if not vpnrouter.metrics_ready(ctx): + run(['systemctl', 'disable', '--now', unit], check=False) + return + run(['systemctl', 'enable', unit], check=False) + run(['systemctl', 'restart', unit], check=False) + + def run_platform_setup(cfg, ctx): """Let the platform module apply its own state, if it defines any.""" module = vpnrouter.platform_module(cfg) @@ -621,6 +701,10 @@ def main(): failed_step = 'Router Manager admin' progress('running', failed_step) problem = manage_admin() + if not problem: + failed_step = 'metrics service account' + progress('running', failed_step) + problem = manage_metrics_account(ctx) progress('running', 'OpenVPN') manage_openvpn(ctx, changed) progress('running', 'HTTPS proxy') @@ -636,6 +720,8 @@ def main(): refresh_docker_rules() progress('running', 'web service state') refresh_web(cfg, ctx) + progress('running', 'metrics service') + manage_metrics(ctx) if problem: progress('failed', failed_step, problem) print(f'vpn-router: error: {problem}', file=sys.stderr) diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter.py b/debian-package/src/usr/lib/vpn-router/vpnrouter.py index 1bb7e07..9680393 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter.py @@ -12,6 +12,7 @@ import re import shutil import subprocess import sys +import urllib.parse import jinja2 @@ -57,8 +58,18 @@ SCHEMA = { 'web': {'enabled': True, 'listen': DEFAULT_LISTEN, 'port': 7081, 'database': '', 'trusted_proxies': [], 'status_content': 'basic'}, 'https': {'enabled': True, 'email': '', 'web_port': 7444, 'manager_port': 7443}, + 'metrics': {'enabled': True, 'listen': 'internal', 'port': 9464, 'https_port': 9465, 'service_account': 'metrics', + 'otlp_endpoint': '', 'otlp_username': '', 'otlp_password': '', + 'keep_details': '24 hours', 'keep_hourly': '1 month', 'keep_daily': '3 months', + 'keep_monthly': '1 year', 'keep_yearly': '10 years'}, } +#: The retention settings of the metrics, from the details to the yearly summaries. +METRICS_KEEP = ('keep_details', 'keep_hourly', 'keep_daily', 'keep_monthly', 'keep_yearly') + +#: A retention period as PostgreSQL reads an interval: a count and a unit. +METRICS_INTERVAL = re.compile(r'^[1-9]\d{0,4} (hour|day|month|year)s?$') + #: The port of Let's Encrypt's HTTP challenge, which Caddy answers while HTTPS is on. ACME_HTTP_PORT = 80 @@ -291,6 +302,33 @@ def validate(cfg): for item in _get_list(cfg, 'web', 'trusted_proxies'): _check_address(item, 'web.trusted_proxies') + taken = {web_port: 'web.port', ACME_HTTP_PORT: 'the HTTP challenge', + https_ports[0]: 'https.web_port', https_ports[1]: 'https.manager_port'} + for option in ('port', 'https_port'): + port = _get(cfg, 'metrics', option) + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + raise ConfigError(f'metrics.{option}: {port!r} is not a valid port') + if port in taken: + raise ConfigError(f'metrics.{option}: {port} is the port of {taken[port]}') + taken[port] = f'metrics.{option}' + + metrics_listen = _get(cfg, 'metrics', 'listen') + if metrics_listen not in LISTEN_CHOICES: + raise ConfigError( + f'metrics.listen: {metrics_listen!r} is not one of {", ".join(LISTEN_CHOICES)}') + + endpoint = _get(cfg, 'metrics', 'otlp_endpoint') + if endpoint and urllib.parse.urlsplit(endpoint).scheme not in ('http', 'https'): + raise ConfigError(f'metrics.otlp_endpoint: {endpoint!r} is not an http or https URL') + if bool(_get(cfg, 'metrics', 'otlp_username')) != bool(_get(cfg, 'metrics', 'otlp_password')): + raise ConfigError('metrics.otlp_username and metrics.otlp_password are given together') + + for option in METRICS_KEEP: + keep = _get(cfg, 'metrics', option) + if not METRICS_INTERVAL.match(keep): + raise ConfigError( + f'metrics.{option}: {keep!r} is not a count and a unit, such as "24 hours" or "3 months"') + docker_mode = _get(cfg, 'docker', 'mode') if docker_mode not in DOCKER_MODES: raise ConfigError( @@ -483,6 +521,10 @@ def build_context(cfg, local_id_override=None): 'https_email': _get(cfg, 'https', 'email'), 'https_web_port': _get(cfg, 'https', 'web_port'), 'https_manager_port': _get(cfg, 'https', 'manager_port'), + 'metrics_enabled': _get(cfg, 'metrics', 'enabled'), + 'metrics_port': _get(cfg, 'metrics', 'port'), + 'metrics_https_port': _get(cfg, 'metrics', 'https_port'), + 'metrics_service_account': _get(cfg, 'metrics', 'service_account'), 'loopback': LOOPBACK, } @@ -518,6 +560,11 @@ def https_ready(ctx): return bool(ctx['https_enabled'] and ctx['local_fqdn']) +def metrics_ready(ctx): + """The metrics service collects, keeps and serves the metrics while metrics.enabled is true.""" + return bool(ctx['metrics_enabled']) + + def _jinja_env(): return jinja2.Environment( loader=jinja2.FileSystemLoader(str(TEMPLATE_DIR)), diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_ldap.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_ldap.py index 3936a5d..7033a80 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_ldap.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_ldap.py @@ -53,7 +53,7 @@ OVERLAYS = ('ppolicy', 'otp') #: The package's own schema, under an OID of the UUID arc 2.25 (ITU-T X.667): attribute types #: under .1 and object classes under .2. vpnRouterAccount, auxiliary, adds properties of the #: package to a user of either kind: mail, for a Manager-only account, and vpnRouterScope, -#: authorization scopes that nothing reads yet. +#: authorization scopes, of which service marks a service account. SCHEMA_OID = '2.25.98477352595735834532404256531620013001' SCHEMA_NAME = 'vpn-router' SCHEMA_ATTRIBUTE_TYPES = ( diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_metrics.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_metrics.py new file mode 100644 index 0000000..efab224 --- /dev/null +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_metrics.py @@ -0,0 +1,173 @@ +"""The metrics of the router in PostgreSQL, for the metrics service and the web service. + +The metrics service writes one detail row per minute to the three detail tables, of TABLES: +metrics_system (CPU, memory, disk and load averages), metrics_network (bytes per second +received and sent per interface) and metrics_vpn (clients per kind). Each detail table has four +summary tables, of LEVELS: _hourly from the details,
_daily from the hourly, +
_monthly from the daily and
_yearly from the monthly rows. A summary row has the +start of its period, in UTC, and the average and the maximum of each value over the period: of +the details for the hourly rows, else the average of the averages and the maximum of the maxima +below. A period is summarized once it is complete; summarize() catches up every complete period +after the last summarized one, so a stopped service loses no summary of what was written. + +collect_garbage() deletes the rows older than the retention of their level, metrics.keep_*; +the service runs it on start and once a day. history() answers the views of the Metrics tab. + +The tables are in the database of the refresh tokens (vpnrouter_auth.connection_string) and are +created when missing. Every failure of the database raises DatabaseError. +""" + +import contextlib + +import psycopg + +import vpnrouter_auth + +#: The detail tables: their key columns besides time, and their value columns. +TABLES = { + 'metrics_system': ((), ('cpu', 'memory_used', 'memory_total', 'disk_free', 'disk_total', 'load1', 'load5', 'load15')), + 'metrics_network': (('interface',), ('rx', 'tx')), + 'metrics_vpn': (('kind',), ('clients',)), +} + +#: The summary levels, from the finest: the suffix of the tables, the period of date_trunc and the retention setting. +LEVELS = ( + ('hourly', 'hour', 'keep_hourly'), + ('daily', 'day', 'keep_daily'), + ('monthly', 'month', 'keep_monthly'), + ('yearly', 'year', 'keep_yearly'), +) + +#: The kinds of VPN clients of metrics_vpn; site_to_site is 1 while the tunnel is up. +VPN_KINDS = ('p2s', 'openvpn', 'wireguard', 'site_to_site') + +#: The views of history(): the summary level read (None: the details), the bucket and the span. +RANGES = { + 'hour': (None, '1 minute', '1 hour'), + 'day': (None, '10 minutes', '1 day'), + 'month': ('hourly', '6 hours', '1 month'), + 'year': ('monthly', '1 month', '1 year'), +} + + +class DatabaseError(Exception): + """The metrics database failed.""" + + +def schema(): + """The statements that create the detail and the summary tables.""" + statements = [] + for table, (keys, values) in TABLES.items(): + key_columns = ''.join(f', {key} text NOT NULL' for key in keys) + primary = ', '.join(('time', *keys)) + detail = ''.join(f', {value} double precision' for value in values) + statements.append(f'CREATE TABLE IF NOT EXISTS {table} (time timestamptz NOT NULL{key_columns}{detail}, ' + f'PRIMARY KEY ({primary}))') + summary = ''.join(f', {value}_avg double precision, {value}_max double precision' for value in values) + for suffix, _, _ in LEVELS: + statements.append(f'CREATE TABLE IF NOT EXISTS {table}_{suffix} (time timestamptz NOT NULL{key_columns}{summary}, ' + f'PRIMARY KEY ({primary}))') + return statements + + +@contextlib.contextmanager +def database(cfg): + """A connection to the metrics database, with its tables, closed at the end; DatabaseError on any failure.""" + try: + with psycopg.connect(vpnrouter_auth.connection_string(cfg), autocommit=True) as connection: + for statement in schema(): + connection.execute(statement) + yield connection + except (psycopg.Error, OSError) as exc: + raise DatabaseError(f'the metrics database failed: {exc}') from exc + + +def write(connection, system, network, vpn): + """Write a sample as the detail rows of the current minute. + + system maps the values of metrics_system to numbers; network maps an interface to its rx and + tx; vpn maps a kind of VPN_KINDS to its clients. A value that is None is written as NULL. + """ + with connection.transaction(): + minute = connection.execute("SELECT date_trunc('minute', now(), 'UTC')").fetchone()[0] + names = TABLES['metrics_system'][1] + connection.execute(f"INSERT INTO metrics_system (time, {', '.join(names)}) " + f"VALUES (%s{', %s' * len(names)}) ON CONFLICT DO NOTHING", + (minute, *(system.get(name) for name in names))) + for interface, (rx, tx) in network.items(): + connection.execute('INSERT INTO metrics_network (time, interface, rx, tx) VALUES (%s, %s, %s, %s) ' + 'ON CONFLICT DO NOTHING', (minute, interface, rx, tx)) + for kind, clients in vpn.items(): + connection.execute('INSERT INTO metrics_vpn (time, kind, clients) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING', + (minute, kind, clients)) + + +def summarize(connection, level): + """Summarize the complete periods of a level of LEVELS that are not summarized yet, for every detail table.""" + index = [suffix for suffix, _, _ in LEVELS].index(level) + _, period, _ = LEVELS[index] + for table, (keys, values) in TABLES.items(): + target = f'{table}_{level}' + source = table if index == 0 else f'{table}_{LEVELS[index - 1][0]}' + if index == 0: + aggregates = ', '.join(f'avg({value}), max({value})' for value in values) + else: + aggregates = ', '.join(f'avg({value}_avg), max({value}_max)' for value in values) + columns = ', '.join(('time', *keys, *(f'{value}_{kind}' for value in values for kind in ('avg', 'max')))) + groups = ''.join(f', {key}' for key in keys) + connection.execute( + f"INSERT INTO {target} ({columns}) " + f"SELECT date_trunc('{period}', time, 'UTC') AS period{groups}, {aggregates} FROM {source} " + f"WHERE time < date_trunc('{period}', now(), 'UTC') " + f"AND time >= coalesce((SELECT max(time) FROM {target}) + interval '1 {period}', '-infinity') " + f"GROUP BY period{groups} ON CONFLICT DO NOTHING") + + +def collect_garbage(connection, cfg): + """Delete the rows older than the retention of their level: metrics.keep_details, keep_hourly and so on.""" + metrics = cfg['metrics'] + for table in TABLES: + connection.execute(f'DELETE FROM {table} WHERE time < now() - %s::interval', (metrics['keep_details'],)) + for suffix, _, keep in LEVELS: + connection.execute(f'DELETE FROM {table}_{suffix} WHERE time < now() - %s::interval', (metrics[keep],)) + + +def history(connection, view): + """The points of a view of RANGES, oldest first, each with the start of its bucket in Unix seconds. + + A point has the average and the maximum of each value of metrics_system, network, which maps + each interface to its rx and tx, and clients, which maps each kind of VPN client to its + clients; a value without data is None. + """ + level, bucket, span = RANGES[view] + + def rows(table): + keys, values = TABLES[table] + source = table if level is None else f'{table}_{level}' + if level is None: + aggregates = ', '.join(f'avg({value}), max({value})' for value in values) + else: + aggregates = ', '.join(f'avg({value}_avg), max({value}_max)' for value in values) + start = (f"date_bin('{bucket}', time, timestamptz '2000-01-01 00:00:00+00')" if view != 'year' + else "date_trunc('month', time, 'UTC')") + groups = ''.join(f', {key}' for key in keys) + return connection.execute( + f"SELECT extract(epoch FROM {start})::bigint AS bucket{groups}, {aggregates} FROM {source} " + f"WHERE time >= now() - interval '{span}' GROUP BY bucket{groups} ORDER BY bucket").fetchall() + + def pairs(row, values): + return {value: {'avg': row[2 * index], 'max': row[2 * index + 1]} for index, value in enumerate(values)} + + points = {} + + def point(start): + return points.setdefault(start, {'time': start, 'network': {}, 'clients': {}}) + + for row in rows('metrics_system'): + point(row[0]).update(pairs(row[1:], TABLES['metrics_system'][1])) + for row in rows('metrics_network'): + point(row[0])['network'][row[1]] = pairs(row[2:], TABLES['metrics_network'][1]) + for row in rows('metrics_vpn'): + point(row[0])['clients'][row[1]] = pairs(row[2:], TABLES['metrics_vpn'][1])['clients'] + empty = {value: {'avg': None, 'max': None} for value in TABLES['metrics_system'][1]} + return {'range': view, 'bucket': bucket, 'points': [{**empty, **points[start]} for start in sorted(points)]} diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_status.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_status.py new file mode 100644 index 0000000..ef0edbe --- /dev/null +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_status.py @@ -0,0 +1,111 @@ +"""What the router's VPN systems report, for the web service and the metrics service. + +Each reader asks the running system once and answers None when it cannot be asked: swanctl for +the IKE SAs, wg for the WireGuard peers and the OpenVPN server's status file for its clients. +""" + +import subprocess +import time + +COMMAND_TIMEOUT = 5 + +OPENVPN_STATUS = '/run/openvpn-server/status-vpn-router.log' + +#: Seconds since its last handshake within which a WireGuard peer counts as connected. +WIREGUARD_CONNECTED_SECONDS = 180 + + +def run(*args): + """The finished process of a command, or None when it cannot be run.""" + try: + return subprocess.run(args, capture_output=True, text=True, timeout=COMMAND_TIMEOUT) + except (OSError, subprocess.TimeoutExpired): + return None + + +def swanctl(command): + """The entries of a swanctl list command, or None when swanctl cannot be asked. + + The --pretty output has one item per line: "name {" opens a section, "key = [" a list + whose items follow one per line, "}" and "]" close them, and "key = value" is a pair + whose value may contain spaces. Every "... event {" section at the top holds one + connection or SA; each becomes an entry with its name under "name". The closing + "... reply {" section is left out. Values stay strings, as swanctl prints them. + Output with more closing than opening lines gives None. + """ + result = run('swanctl', command, '--pretty') + if result is None or result.returncode != 0: + return None + events = [] + stack = [] + for line in result.stdout.splitlines(): + line = line.strip() + if not line: + continue + if line in ('}', ']'): + if not stack: + return None + stack.pop() + elif not stack: + section = {} + if line.endswith(' event {'): + events.append(section) + stack.append(section) + elif isinstance(stack[-1], list): + stack[-1].append(line) + elif line.endswith(' {'): + stack[-1][line[:-2]] = section = {} + stack.append(section) + elif line.endswith(' = ['): + stack[-1][line[:-4]] = items = [] + stack.append(items) + else: + key, _, value = line.partition(' = ') + stack[-1][key] = value + return [dict(body, name=name) for event in events for name, body in event.items()] + + +def established(connection): + """The number of established IKE SAs of the connection, or None when swanctl cannot be asked.""" + sas = swanctl('--list-sas') + if sas is None: + return None + return sum(1 for sa in sas if sa['name'] == connection and sa.get('state') == 'ESTABLISHED') + + +def wireguard_peers(): + """One entry per WireGuard peer, with its handshake age and traffic, or None when wg0 is down.""" + result = run('wg', 'show', 'wg0', 'dump') + if result is None or result.returncode != 0: + return None + now = int(time.time()) + peers = [] + for line in result.stdout.splitlines()[1:]: + fields = line.split('\t') + if len(fields) < 7: + continue + handshake = int(fields[4]) + peers.append({ + 'handshake_seconds_ago': now - handshake if handshake else None, + 'received_bytes': int(fields[5]), + 'sent_bytes': int(fields[6]), + }) + return peers + + +def wireguard_connected(): + """The number of WireGuard peers with a handshake in the last WIREGUARD_CONNECTED_SECONDS, or None when wg0 is down.""" + peers = wireguard_peers() + if peers is None: + return None + return sum(1 for peer in peers + if peer['handshake_seconds_ago'] is not None and peer['handshake_seconds_ago'] < WIREGUARD_CONNECTED_SECONDS) + + +def openvpn_clients(): + """The number of connected OpenVPN clients, from the server's status file, or None when it cannot be read.""" + try: + with open(OPENVPN_STATUS, encoding='utf-8') as handle: + return sum(1 for line in handle if line.startswith('CLIENT_LIST,')) + except OSError: + return None diff --git a/debian-package/src/usr/lib/vpn-router/vpnrouter_users.py b/debian-package/src/usr/lib/vpn-router/vpnrouter_users.py index 18cde28..985bdc7 100644 --- a/debian-package/src/usr/lib/vpn-router/vpnrouter_users.py +++ b/debian-package/src/usr/lib/vpn-router/vpnrouter_users.py @@ -6,8 +6,10 @@ password OpenVPN checks, and a client certificate of the router CA with the user common name, which OpenVPN and IKEv2 accept. A Manager-only account is an account, simpleSecurityObject and vpnRouterAccount (the package's schema) with a name, a password and optionally an e-mail address, no names, no certificate and so no VPN access; it is added when -neither the first nor the last name is given, and can sign in to the web service. Users exist -whether or not a P2S system is enabled. +neither the first nor the last name is given, and can sign in to the web service. A service +account is a Manager-only account with the scope SERVICE_SCOPE in vpnRouterScope, which +vpn-router-setup adds and keeps for the metrics service, as metrics.service_account names it; it +cannot sign in to the web service. Users exist whether or not a P2S system is enabled. A group is a groupOfNames in ou=groups with a name, which follows the rules of user names, an optional description and at least one member; a group cannot be empty, so the last member @@ -62,7 +64,9 @@ EMAIL_MAX = 254 #: The keys of the person attributes and their LDAP attributes. PERSON = (('first_name', 'givenName'), ('last_name', 'sn'), ('email', 'mail')) #: The attributes read for a user. -ATTRIBUTES = ['uid', 'objectClass', 'pwdAccountLockedTime', 'givenName', 'sn', 'mail', 'oathTOTPToken'] +ATTRIBUTES = ['uid', 'objectClass', 'pwdAccountLockedTime', 'givenName', 'sn', 'mail', 'oathTOTPToken', 'vpnRouterScope'] +#: The vpnRouterScope of a service account. +SERVICE_SCOPE = 'service' #: The object classes of a VPN user and of a Manager-only account. VPN_USER_CLASSES = [b'inetOrgPerson'] MANAGER_ONLY_CLASSES = [b'account', b'simpleSecurityObject', b'vpnRouterAccount'] @@ -183,6 +187,11 @@ def manager_only(attributes): return b'account' in (value.lower() for value in attributes.get('objectClass', [])) +def service(attributes): + """Whether the entry is a service account.""" + return SERVICE_SCOPE.encode() in (value.lower() for value in attributes.get('vpnRouterScope', [])) + + def summary(attributes): """A user as the answers show it; a person attribute the entry does not have is None.""" name = attributes['uid'][0].decode() @@ -190,6 +199,7 @@ def summary(attributes): user = { 'name': name, 'manager_only': manager_only(attributes), + 'service': service(attributes), 'enabled': lock != LOCKED, 'locked': lock not in (None, LOCKED) and locked_since(lock) > time.time() - vpnrouter_ldap.LOCKOUT_SECONDS, 'certificate': certificate(name)[0].is_file(), @@ -213,12 +223,15 @@ def list_users(connection): return {'users': sorted((summary(attributes) for _, attributes in found), key=lambda user: user['name'])} -def add(connection, name, password, values): +def add(connection, name, password, values, service=False): """Add a user: a VPN user with its certificate when values gives all three person attributes, a Manager-only account - with at most the e-mail address when it gives neither name.""" + with at most the e-mail address when it gives neither name, and, with service, which only vpn-router-setup uses, a + service account, with at most the e-mail address too.""" check_name(name) details = person(values, required=False) vpn_user = 'givenName' in details or 'sn' in details + if service and vpn_user: + raise Failure('a service account has no first or last name', 2) if vpn_user and len(details) != len(PERSON): raise Failure('give first_name, last_name and email for a VPN user, or at most email for a Manager-only account', 2) check_password(password) @@ -229,6 +242,8 @@ def add(connection, name, password, values): else: # simpleSecurityObject requires a userPassword; passwd_s sets the real one at once. record = {'objectClass': MANAGER_ONLY_CLASSES, 'uid': [name.encode()], 'userPassword': [b'{CRYPT}!']} + if service: + record['vpnRouterScope'] = [SERVICE_SCOPE.encode()] record.update({attribute: [value.encode()] for attribute, value in details.items()}) try: connection.add_s(dn_of(name), ldap.modlist.addModlist(record)) diff --git a/debian-package/src/usr/lib/vpn-router/web b/debian-package/src/usr/lib/vpn-router/web index f7a0db8..84debb1 100644 --- a/debian-package/src/usr/lib/vpn-router/web +++ b/debian-package/src/usr/lib/vpn-router/web @@ -18,8 +18,11 @@ Pages: /account the account of the signed-in user: its details, its password and its TOTP enrolment (see account()) /auth sign-in with a user's password and TOTP code, refresh and sign-out (see auth()) + /metrics/history + the metrics the metrics service keeps, for a view: hour, day, month or year (see + metrics_history()) -/config, /users, /groups and /account need an HS256 JWT in an Authorization: Bearer header signed +/config, /metrics/history, /users, /groups and /account need an HS256 JWT in an Authorization: Bearer header signed with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub; /account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A token of vpn-router-token has the admin claim and names no user; it may call every path but @@ -57,10 +60,10 @@ import collections import json import os import signal -import subprocess import sys import threading import time +import urllib.parse from html import escape from http.server import BaseHTTPRequestHandler @@ -70,9 +73,11 @@ import ldap import vpnrouter import vpnrouter_auth +import vpnrouter_metrics import vpnrouter_net import vpnrouter_serve import vpnrouter_users +from vpnrouter_status import established, openvpn_clients, run, swanctl, wireguard_peers STATE_FILE = '/run/vpn-router/web.json' CONFIG_FILE = '/etc/vpn-router/vpn-router.json' @@ -85,7 +90,6 @@ DEFAULT_PORT = 7081 #: Seconds between two discoveries of where the service should listen. DISCOVERY_SECONDS = 30 -COMMAND_TIMEOUT = 5 CACHE_SECONDS = 2 #: The calls of /auth allowed per client address in AUTH_WINDOW seconds. @@ -97,8 +101,6 @@ TAILWIND = 'https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4' #: Seconds after which the status page in a browser reloads itself. REFRESH_SECONDS = 60 -OPENVPN_STATUS = '/run/openvpn-server/status-vpn-router.log' - #: The largest request body accepted, in bytes. BODY_LIMIT = 4096 @@ -138,14 +140,6 @@ def current(): return dict(read_json(STATE_FILE), key=vpnrouter_auth.key()) -def run(*args): - """The finished process of a command, or None when it cannot be run.""" - try: - return subprocess.run(args, capture_output=True, text=True, timeout=COMMAND_TIMEOUT) - except (OSError, subprocess.TimeoutExpired): - return None - - def unit_state(unit): result = run('systemctl', 'is-active', unit) return result.stdout.strip() if result and result.stdout.strip() else 'unknown' @@ -169,85 +163,6 @@ def docker_status(): } -def swanctl(command): - """The entries of a swanctl list command, or None when swanctl cannot be asked. - - The --pretty output has one item per line: "name {" opens a section, "key = [" a list - whose items follow one per line, "}" and "]" close them, and "key = value" is a pair - whose value may contain spaces. Every "... event {" section at the top holds one - connection or SA; each becomes an entry with its name under "name". The closing - "... reply {" section is left out. Values stay strings, as swanctl prints them. - Output with more closing than opening lines gives None. - """ - result = run('swanctl', command, '--pretty') - if result is None or result.returncode != 0: - return None - events = [] - stack = [] - for line in result.stdout.splitlines(): - line = line.strip() - if not line: - continue - if line in ('}', ']'): - if not stack: - return None - stack.pop() - elif not stack: - section = {} - if line.endswith(' event {'): - events.append(section) - stack.append(section) - elif isinstance(stack[-1], list): - stack[-1].append(line) - elif line.endswith(' {'): - stack[-1][line[:-2]] = section = {} - stack.append(section) - elif line.endswith(' = ['): - stack[-1][line[:-4]] = items = [] - stack.append(items) - else: - key, _, value = line.partition(' = ') - stack[-1][key] = value - return [dict(body, name=name) for event in events for name, body in event.items()] - - -def established(connection): - """The number of established IKE SAs of the connection, or None when swanctl cannot be asked.""" - sas = swanctl('--list-sas') - if sas is None: - return None - return sum(1 for sa in sas if sa['name'] == connection and sa.get('state') == 'ESTABLISHED') - - -def wireguard_peers(): - """One entry per WireGuard peer, with its handshake age and traffic, or None when wg0 is down.""" - result = run('wg', 'show', 'wg0', 'dump') - if result is None or result.returncode != 0: - return None - now = int(time.time()) - peers = [] - for line in result.stdout.splitlines()[1:]: - fields = line.split('\t') - if len(fields) < 7: - continue - handshake = int(fields[4]) - peers.append({ - 'handshake_seconds_ago': now - handshake if handshake else None, - 'received_bytes': int(fields[5]), - 'sent_bytes': int(fields[6]), - }) - return peers - - -def openvpn_clients(): - """The number of connected OpenVPN clients, from the server's status file, or None when it cannot be read.""" - try: - with open(OPENVPN_STATUS, encoding='utf-8') as handle: - return sum(1 for line in handle if line.startswith('CLIENT_LIST,')) - except OSError: - return None - - def postgresql_clusters(): """The state of each PostgreSQL cluster unit (postgresql@-), or {'postgresql': 'no cluster'} without one. @@ -453,7 +368,7 @@ def users(state, method, path, body): GET /users the users POST /users add a user: {"name", "password", "first_name", "last_name", "email"}; - without the last three a Manager-only account + without the names a Manager-only account PUT /users/ change any of {"first_name", "last_name", "email"} PUT /users//password set the password: {"password": ...} POST /users//disable lock the user out of OpenVPN @@ -593,6 +508,22 @@ def account(name, method, path, body): return 200, document +def metrics_history(view): + """Answer GET /metrics/history?range= with vpnrouter_metrics.history: the HTTP status and the JSON document. + + view is hour, the default, day, month or year (vpnrouter_metrics.RANGES). + """ + if view not in vpnrouter_metrics.RANGES: + return 400, {'error': f'range must be one of {", ".join(vpnrouter_metrics.RANGES)}'} + try: + with vpnrouter_metrics.database(vpnrouter.load_config()) as connection: + return 200, vpnrouter_metrics.history(connection, view) + except vpnrouter_metrics.DatabaseError as exc: + return 500, {'error': str(exc)} + except SystemExit as exc: + return 500, {'error': str(exc.code)} + + def changes_admins(connection, method, parts, body): """Whether a /users or /groups request creates, renames to, changes or removes ADMIN_GROUP, or removes a member of it.""" admins = vpnrouter_users.ADMIN_GROUP @@ -638,6 +569,9 @@ def auth(path, body): cfg = vpnrouter.load_config() if path == '/auth': vpnrouter_users.login(body.get('name'), body.get('password'), body.get('code')) + with vpnrouter_users.session() as connection: + if vpnrouter_users.service(vpnrouter_users.entry(connection, body['name'])): + raise vpnrouter_users.Failure('the name, the password or the code is wrong', 5) return 200, vpnrouter_auth.issue(cfg, body['name']) name = vpnrouter_auth.take(cfg, body.get('refresh_token')) if path == '/auth/logout': @@ -714,7 +648,7 @@ class Handler(BaseHTTPRequestHandler): if self.command not in ('GET', 'HEAD'): self.close_connection = True directory = next((name for name in ('users', 'groups', 'account') if path == f'/{name}' or path.startswith(f'/{name}/')), None) - protected = path == '/config' or directory is not None + protected = path in ('/config', '/metrics/history') or directory is not None signing = path in ('/auth', '/auth/refresh', '/auth/logout') state = current() if protected or signing else {} claims = self.claims(state['key']) if protected and state['key'] else None @@ -741,10 +675,14 @@ class Handler(BaseHTTPRequestHandler): body = json.dumps({'error': refused[1]}, indent=2).encode() + b'\n' elif path == '/config' and self.command in ('GET', 'HEAD'): body, kind, code = json.dumps(details(state), indent=2).encode() + b'\n', 'application/json', 200 + elif path == '/metrics/history' and self.command in ('GET', 'HEAD'): + query = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query) + code, document = metrics_history(query.get('range', ['hour'])[0]) + body, kind = json.dumps(document).encode() + b'\n', 'application/json' elif directory == 'account': code, document = account(claims['sub'], self.command, path, payload) body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' - elif protected and path != '/config': + elif directory is not None: code, document = (users if directory == 'users' else groups)(state, self.command, path, payload) body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json' elif self.command not in ('GET', 'HEAD'): diff --git a/debian-package/src/usr/sbin/vpn-router-user b/debian-package/src/usr/sbin/vpn-router-user index 388b5a2..ac887a4 100755 --- a/debian-package/src/usr/sbin/vpn-router-user +++ b/debian-package/src/usr/sbin/vpn-router-user @@ -115,7 +115,7 @@ def text_of(answer): if not users: return 'no users' head = ('USER', 'KIND', 'STATE', 'CERTIFICATE', 'OTP', 'FIRST NAME', 'LAST NAME', 'E-MAIL') - rows = [(user['name'], 'manager' if user['manager_only'] else 'vpn', + rows = [(user['name'], 'service' if user['service'] else 'manager' if user['manager_only'] else 'vpn', 'disabled' if not user['enabled'] else 'locked' if user['locked'] else 'enabled', 'issued' if user['certificate'] else 'missing', 'enrolled' if user['otp'] else '-', user['first_name'] or '-', user['last_name'] or '-', user['email'] or '-') diff --git a/debian-package/src/usr/share/doc/vpn-router/vpn-router.json.example b/debian-package/src/usr/share/doc/vpn-router/vpn-router.json.example index b413d59..e8b2422 100644 --- a/debian-package/src/usr/share/doc/vpn-router/vpn-router.json.example +++ b/debian-package/src/usr/share/doc/vpn-router/vpn-router.json.example @@ -56,5 +56,20 @@ "email": "", "web_port": 7444, "manager_port": 7443 + }, + "metrics": { + "enabled": true, + "listen": "internal", + "port": 9464, + "https_port": 9465, + "service_account": "metrics", + "otlp_endpoint": "", + "otlp_username": "", + "otlp_password": "", + "keep_details": "24 hours", + "keep_hourly": "1 month", + "keep_daily": "3 months", + "keep_monthly": "1 year", + "keep_yearly": "10 years" } } diff --git a/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 b/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 index bc0315a..2d79a9d 100644 --- a/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 +++ b/debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2 @@ -15,3 +15,8 @@ reverse_proxy {{ loopback }}:{{ manager_port }} } {% endif %} +{%- if metrics_enabled %} +{{ local_fqdn }}:{{ metrics_https_port }} { + reverse_proxy {{ loopback }}:{{ metrics_port }} +} +{% endif %}