feature: the router keeps its metrics in PostgreSQL, serves them for an OpenTelemetry Collector and pushes them over OTLP.

debian-package/src/usr/lib/vpn-router/metrics

@@ -0,0 +1,414 @@
+#!/usr/bin/python3
+"""Collect the metrics of the router, keep them in PostgreSQL, serve them for scraping and push them with OTLP.
+
+Run by vpn-router-metrics.service while metrics.enabled is true; vpn-router-setup enables and
+disables the unit.
+
+Every SAMPLE_SECONDS, on the minute, a sample is taken with psutil: the CPU utilization since the
+previous sample, the memory used and in total, the free and the total space of /, the bytes
+received and sent per second on the external and the internal interface, and the load averages;
+and the VPN clients of each enabled VPN system (vpnrouter_status): road-warrior (p2s) and
+OpenVPN clients, WireGuard peers with a recent handshake, and the site-to-site tunnel, 1 while it
+is up. The interfaces and the enabled systems are those vpn-router-setup resolved, read from
+/run/vpn-router/web.json.
+
+The sample is written to the detail tables of vpnrouter_metrics. When the clock has passed into
+a new hour, day, month or year, the complete periods are summarized, and once a day, after the
+daily summary, the rows older than their retention are deleted; both also run on start. A
+failure of the database is logged and tried again with the next sample.
+
+GET /metrics answers the last sample in the Prometheus text format, written by
+prometheus_client, with OpenTelemetry semantic convention names in their Prometheus form, for an
+OpenTelemetry Collector's prometheus receiver. It needs HTTP Basic authentication as the service
+account metrics.service_account names, which vpn-router-setup keeps with the password in
+/etc/vpn-router/metrics.secret, checked with a bind to the router's directory, so failed attempts
+lock the account as for any user. The service listens on the interfaces metrics.listen
+names and on the loopback address, where Caddy serves it over HTTPS at metrics.https_port; the
+address of a call through Caddy is the last entry of its X-Forwarded-For header.
+
+When metrics.otlp_endpoint is set, every sample is also pushed there as an OTLP/HTTP JSON
+ExportMetricsServiceRequest, with HTTP Basic authentication when metrics.otlp_username is set.
+"""
+
+import base64
+import datetime
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler
+
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+
+import ldap
+import psutil
+from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+from prometheus_client.core import CollectorRegistry, CounterMetricFamily, GaugeMetricFamily
+
+import vpnrouter
+import vpnrouter_metrics
+import vpnrouter_net
+import vpnrouter_serve
+import vpnrouter_status
+import vpnrouter_users
+
+STATE_FILE = '/run/vpn-router/web.json'
+CONFIG_FILE = '/etc/vpn-router/vpn-router.json'
+
+SAMPLE_SECONDS = 60
+
+#: Seconds between two discoveries of where the service should listen.
+DISCOVERY_SECONDS = 30
+
+#: Seconds an OTLP push may take.
+OTLP_TIMEOUT = 10
+
+#: The mount point whose space is measured.
+MOUNTPOINT = '/'
+
+#: The last sample, as sample() makes it, or None before the first.
+_latest = {'sample': None}
+_latest_lock = threading.Lock()
+
+
+def log(message):
+    print(f'vpn-router-metrics: {message}', file=sys.stderr, flush=True)
+
+
+def read_json(path):
+    """The object in a JSON file, or {} when the file is missing or holds no object."""
+    try:
+        with open(path, encoding='utf-8') as handle:
+            data = json.load(handle)
+    except (OSError, ValueError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def section(configuration, name):
+    """A section of the configuration, or {} when it is missing or not an object."""
+    value = configuration.get(name)
+    return value if isinstance(value, dict) else {}
+
+
+def clients(state):
+    """The VPN clients of each enabled VPN system, None where it cannot be asked."""
+    counts = {}
+    if state.get('p2s'):
+        counts['p2s'] = vpnrouter_status.established('road-warrior')
+    if state.get('openvpn'):
+        counts['openvpn'] = vpnrouter_status.openvpn_clients()
+    if state.get('wireguard'):
+        counts['wireguard'] = vpnrouter_status.wireguard_connected()
+    if state.get('site_to_site'):
+        count = vpnrouter_status.established('remote-site')
+        counts['site_to_site'] = None if count is None else int(count > 0)
+    return counts
+
+
+def sample(previous):
+    """A sample of the router, with the rates since the previous sample, None for the first."""
+    state = read_json(STATE_FILE)
+    network = section(state, 'network')
+    names = [name for name in dict.fromkeys((network.get('external_interface'), network.get('internal_interface')))
+             if isinstance(name, str) and name]
+    counters = psutil.net_io_counters(pernic=True)
+    memory = psutil.virtual_memory()
+    disk = psutil.disk_usage(MOUNTPOINT)
+    load1, load5, load15 = os.getloadavg()
+    now = time.time()
+    current = {
+        'time': now,
+        'monotonic': time.monotonic(),
+        'system': {
+            'cpu': psutil.cpu_percent(interval=None),
+            'memory_used': memory.total - memory.available,
+            'memory_total': memory.total,
+            'disk_free': disk.free,
+            'disk_total': disk.total,
+            'load1': load1,
+            'load5': load5,
+            'load15': load15,
+        },
+        'counters': {name: (counters[name].bytes_recv, counters[name].bytes_sent) for name in names if name in counters},
+        'clients': clients(state),
+        'rates': {},
+    }
+    if previous is not None:
+        elapsed = current['monotonic'] - previous['monotonic']
+        for name, (received, sent) in current['counters'].items():
+            if name in previous['counters'] and elapsed > 0:
+                before_received, before_sent = previous['counters'][name]
+                current['rates'][name] = (max(received - before_received, 0) / elapsed, max(sent - before_sent, 0) / elapsed)
+    return current
+
+
+def periods(moment):
+    """The hour, the day, the month and the year of a UTC moment, keyed by the levels of vpnrouter_metrics.LEVELS."""
+    return {'hourly': moment.strftime('%Y-%m-%dT%H'), 'daily': moment.strftime('%Y-%m-%d'),
+            'monthly': moment.strftime('%Y-%m'), 'yearly': moment.strftime('%Y')}
+
+
+def store(cfg, current, done):
+    """Write the sample, summarize the periods the clock has passed and collect the garbage once a day.
+
+    done holds the periods summarized so far, and garbage the day of the last garbage collection;
+    it is updated only for what succeeded.
+    """
+    now = periods(datetime.datetime.now(datetime.timezone.utc))
+    with vpnrouter_metrics.database(cfg) as connection:
+        vpnrouter_metrics.write(connection, current['system'], current['rates'], current['clients'])
+        for level, _, _ in vpnrouter_metrics.LEVELS:
+            if done.get(level) != now[level]:
+                vpnrouter_metrics.summarize(connection, level)
+                done[level] = now[level]
+        if done.get('garbage') != now['daily']:
+            vpnrouter_metrics.collect_garbage(connection, cfg)
+            done['garbage'] = now['daily']
+
+
+def attributes(**values):
+    """OTLP attributes of string values."""
+    return [{'key': key, 'value': {'stringValue': value}} for key, value in values.items()]
+
+
+def otlp_request(current, started):
+    """The OTLP/HTTP JSON ExportMetricsServiceRequest of a sample; started is when the counters were first read, in Unix seconds."""
+    stamp = str(int(current['time'] * 1e9))
+    start = str(int(started * 1e9))
+    system = current['system']
+
+    def gauge(name, unit, points):
+        return {'name': name, 'unit': unit, 'gauge': {'dataPoints': [
+            {'timeUnixNano': stamp, 'asDouble': float(value), 'attributes': attributes(**labels)}
+            for value, labels in points if value is not None]}}
+
+    metrics = [
+        gauge('system.cpu.utilization', '1', [(system['cpu'] / 100, {})]),
+        gauge('system.memory.usage', 'By', [(system['memory_used'], {'state': 'used'})]),
+        gauge('system.memory.limit', 'By', [(system['memory_total'], {})]),
+        gauge('system.filesystem.usage', 'By', [(system['disk_free'], {'mountpoint': MOUNTPOINT, 'state': 'free'}),
+                                               (system['disk_total'] - system['disk_free'], {'mountpoint': MOUNTPOINT, 'state': 'used'})]),
+        gauge('system.filesystem.utilization', '1',
+              [(1 - system['disk_free'] / system['disk_total'] if system['disk_total'] else None, {'mountpoint': MOUNTPOINT})]),
+        gauge('system.cpu.load_average.1m', '{thread}', [(system['load1'], {})]),
+        gauge('system.cpu.load_average.5m', '{thread}', [(system['load5'], {})]),
+        gauge('system.cpu.load_average.15m', '{thread}', [(system['load15'], {})]),
+        gauge('vpn_router.clients', '{client}', [(count, {'kind': kind}) for kind, count in current['clients'].items()]),
+        {'name': 'system.network.io', 'unit': 'By', 'sum': {
+            'aggregationTemporality': 2, 'isMonotonic': True, 'dataPoints': [
+                {'startTimeUnixNano': start, 'timeUnixNano': stamp, 'asInt': str(value),
+                 'attributes': attributes(device=name, direction=direction)}
+                for name, counters in current['counters'].items()
+                for value, direction in zip(counters, ('receive', 'transmit'))]}},
+    ]
+    return {'resourceMetrics': [{
+        'resource': {'attributes': attributes(**{'service.name': 'vpn-router', 'host.name': socket.gethostname()})},
+        'scopeMetrics': [{'scope': {'name': 'vpn-router-metrics'}, 'metrics': metrics}],
+    }]}
+
+
+def push(metrics, current, started):
+    """Push the sample to metrics.otlp_endpoint; raises OSError when the collector does not take it."""
+    request = urllib.request.Request(metrics['otlp_endpoint'], data=json.dumps(otlp_request(current, started)).encode(),
+                                     headers={'Content-Type': 'application/json'}, method='POST')
+    if metrics['otlp_username']:
+        credentials = f"{metrics['otlp_username']}:{metrics['otlp_password']}".encode()
+        request.add_header('Authorization', 'Basic ' + base64.b64encode(credentials).decode())
+    with urllib.request.urlopen(request, timeout=OTLP_TIMEOUT) as response:
+        response.read()
+
+
+def collector_loop():
+    """Take a sample on every minute, store it and push it; every new failure is logged once."""
+    previous, done, reported = None, {}, {}
+    started = time.time()
+    psutil.cpu_percent(interval=None)
+
+    def report(kind, message):
+        if message != reported.get(kind):
+            if message:
+                log(message)
+            elif reported.get(kind):
+                log(f'{kind} works again')
+            reported[kind] = message
+
+    while True:
+        time.sleep(SAMPLE_SECONDS - time.time() % SAMPLE_SECONDS)
+        try:
+            cfg = vpnrouter.load_config()
+        except SystemExit as exc:
+            report('configuration', str(exc.code))
+            continue
+        report('configuration', '')
+        current = sample(previous)
+        previous = current
+        with _latest_lock:
+            _latest['sample'] = current
+        try:
+            store(cfg, current, done)
+            report('database', '')
+        except vpnrouter_metrics.DatabaseError as exc:
+            report('database', str(exc))
+        metrics = cfg['metrics']
+        if metrics['otlp_endpoint']:
+            try:
+                push(metrics, current, started)
+                report('OTLP push', '')
+            except (OSError, ValueError) as exc:
+                report('OTLP push', f"pushing to {metrics['otlp_endpoint']} failed: {exc}")
+
+
+class Sample:
+    """A prometheus_client collector of a sample."""
+
+    def __init__(self, current):
+        self.current = current
+
+    def collect(self):
+        system = self.current['system']
+        yield GaugeMetricFamily('system_cpu_utilization', 'CPU utilization since the previous sample, 0 to 1',
+                                value=system['cpu'] / 100)
+        memory = GaugeMetricFamily('system_memory_usage_bytes', 'Memory in use', labels=['state'])
+        memory.add_metric(['used'], system['memory_used'])
+        yield memory
+        yield GaugeMetricFamily('system_memory_limit_bytes', 'Total memory', value=system['memory_total'])
+        usage = GaugeMetricFamily('system_filesystem_usage_bytes', 'Space of the file system', labels=['mountpoint', 'state'])
+        usage.add_metric([MOUNTPOINT, 'free'], system['disk_free'])
+        usage.add_metric([MOUNTPOINT, 'used'], system['disk_total'] - system['disk_free'])
+        yield usage
+        if system['disk_total']:
+            utilization = GaugeMetricFamily('system_filesystem_utilization', 'Used part of the file system, 0 to 1',
+                                            labels=['mountpoint'])
+            utilization.add_metric([MOUNTPOINT], 1 - system['disk_free'] / system['disk_total'])
+            yield utilization
+        for period in ('1m', '5m', '15m'):
+            yield GaugeMetricFamily(f'system_cpu_load_average_{period}', f'Load average over {period}',
+                                    value=system[f'load{period[:-1]}'])
+        network = CounterMetricFamily('system_network_io_bytes', 'Bytes received and sent', labels=['device', 'direction'])
+        for name, (received, sent) in self.current['counters'].items():
+            network.add_metric([name, 'receive'], received)
+            network.add_metric([name, 'transmit'], sent)
+        yield network
+        vpn = GaugeMetricFamily('vpn_router_clients', 'Connected VPN clients; site_to_site is 1 while the tunnel is up',
+                                labels=['kind'])
+        for kind, count in self.current['clients'].items():
+            if count is not None:
+                vpn.add_metric([kind], count)
+        yield vpn
+
+
+def authorized(header, account):
+    """Whether an Authorization header holds the password of the service account; LDAP errors propagate."""
+    if not header.startswith('Basic '):
+        return False
+    try:
+        name, _, password = base64.b64decode(header[6:].strip(), validate=True).decode().partition(':')
+    except ValueError:
+        return False
+    if name != account:
+        return False
+    try:
+        with vpnrouter_users.session() as connection:
+            if not vpnrouter_users.service(vpnrouter_users.entry(connection, name)):
+                return False
+        vpnrouter_users.login(name, password, None)
+    except vpnrouter_users.Failure:
+        return False
+    return True
+
+
+class Handler(BaseHTTPRequestHandler):
+    protocol_version = 'HTTP/1.1'
+
+    def client(self):
+        """The client address: the last X-Forwarded-For entry of a call through Caddy on the loopback address, else the peer."""
+        peer = self.client_address[0]
+        forwarded = self.headers.get('X-Forwarded-For', '')
+        if peer == vpnrouter_net.LOOPBACK and forwarded.strip():
+            return forwarded.split(',')[-1].strip()
+        return peer
+
+    def answer(self, send_body):
+        self.principal = None
+        extra = []
+        path = self.path.split('?', 1)[0]
+        account = str(section(read_json(CONFIG_FILE), 'metrics').get('service_account',
+                                                                       vpnrouter.SCHEMA['metrics']['service_account'])).strip()
+        with _latest_lock:
+            current = _latest['sample']
+        if path != '/metrics':
+            body, kind, code = b'Not found\n', 'text/plain; charset=utf-8', 404
+        elif not account:
+            body, kind, code = b'metrics.service_account is not set\n', 'text/plain; charset=utf-8', 503
+        else:
+            try:
+                allowed = authorized(self.headers.get('Authorization', ''), account)
+            except (ldap.LDAPError, OSError) as exc:
+                allowed = None
+                body, kind, code = f'the user directory failed: {exc}\n'.encode(), 'text/plain; charset=utf-8', 500
+            if allowed is False:
+                body, kind, code = b'Unauthorized\n', 'text/plain; charset=utf-8', 401
+                extra.append(('WWW-Authenticate', 'Basic realm="vpn-router metrics", charset="UTF-8"'))
+            elif allowed:
+                self.principal = account
+                if current is None:
+                    body, kind, code = b'no sample yet\n', 'text/plain; charset=utf-8', 503
+                else:
+                    registry = CollectorRegistry()
+                    registry.register(Sample(current))
+                    body, kind, code = generate_latest(registry), CONTENT_TYPE_LATEST, 200
+        self.send_response(code)
+        self.send_header('Content-Type', kind)
+        self.send_header('Content-Length', str(len(body)))
+        self.send_header('Cache-Control', 'no-store')
+        for name, value in extra:
+            self.send_header(name, value)
+        self.end_headers()
+        if send_body:
+            self.wfile.write(body)
+
+    def do_GET(self):
+        self.answer(True)
+
+    def do_HEAD(self):
+        self.answer(False)
+
+    def log_message(self, format, *args):
+        principal = getattr(self, 'principal', None) or '-'
+        sys.stderr.write('%s - %s [%s] %s\n' % (self.client(), principal, self.log_date_time_string(), format % args))
+
+
+def wanted():
+    """Where the service listens: the addresses of the interfaces metrics.listen names and LOOPBACK, at metrics.port."""
+    configuration = read_json(CONFIG_FILE)
+    metrics = section(configuration, 'metrics')
+    defaults = vpnrouter.SCHEMA['metrics']
+    if metrics.get('enabled', defaults['enabled']) is False:
+        return [], 'metrics are disabled in the configuration'
+    port = metrics.get('port', defaults['port'])
+    if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
+        port = defaults['port']
+    addresses = vpnrouter_net.listen_addresses(configuration, metrics.get('listen', defaults['listen']))
+    if not addresses:
+        return [], 'the addresses to listen on are not known yet'
+    return [(address, port) for address in [*addresses, vpnrouter_net.LOOPBACK]], ''
+
+
+def main():
+    signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+    threading.Thread(target=collector_loop, name='collector', daemon=True).start()
+    try:
+        vpnrouter_serve.serve(wanted, Handler, log, DISCOVERY_SECONDS)
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

debian-package/src/usr/lib/vpn-router/setup

@@ -21,6 +21,7 @@ import json
 import os
 import pathlib
 import re
+import secrets
 import socket
 import subprocess
 import sys
@@ -59,6 +60,9 @@ MANAGER_CONFIG = pathlib.Path('/etc/vpn-router-manager/manager.json')
 MANAGER_ADMIN  = pathlib.Path('/etc/vpn-router-manager/admin.json')
 MANAGER_PORT   = 7080
 
+#: The password of the metrics service account, which setup makes and keeps (root only).
+METRICS_SECRET = pathlib.Path('/etc/vpn-router/metrics.secret')
+
 #: The configuration of Caddy, the HTTPS proxy, which the package's drop-in for caddy.service names;
 #: Caddy runs as the user and group CADDY_GROUP, which the caddy package creates.
 CADDYFILE   = pathlib.Path('/etc/caddy/vpn-router.Caddyfile')
@@ -79,6 +83,7 @@ BLOCKS = (
     ('# BOOTSTRAP RULES START',     '# BOOTSTRAP RULES END'),
     ('# MANAGER RULES START',       '# MANAGER RULES END'),
     ('# HTTPS RULES START',         '# HTTPS RULES END'),
+    ('# METRICS RULES START',       '# METRICS RULES END'),
     ('# ROUTER FORWARD RULES START', '# ROUTER FORWARD RULES END'),
     ('# ROUTER NAT RULES START',    '# ROUTER NAT RULES END'),
     ('# ROUTER MSS RULES START',    '# ROUTER MSS RULES END'),
@@ -223,7 +228,7 @@ def filter_block(ctx):
                 ]
         lines += ['# P2S DNS RULES END']
 
-    # The web service and the Router Manager choose their interfaces themselves, so their ports are open on both.
+    # The web service, the Router Manager and the metrics service choose their interfaces themselves, so their ports are open on both.
     interfaces = [name for name in (ctx['int_iface'], ctx['wan_iface']) if name]
     if ctx['web_enabled'] and interfaces:
         lines += [
@@ -250,10 +255,19 @@ def filter_block(ctx):
             '# MANAGER RULES END',
         ]
 
+    if vpnrouter.metrics_ready(ctx) and interfaces:
+        lines += [
+            '',
+            '# METRICS RULES START',
+            *(f'-A ufw-before-input -i {name} -p tcp --dport {ctx["metrics_port"]} -j ACCEPT' for name in interfaces),
+            '# METRICS RULES END',
+        ]
+
     # Let's Encrypt reaches the HTTP challenge on the external interface; the HTTPS ports are open on both.
     if vpnrouter.https_ready(ctx) and interfaces:
         ports = [port for wanted, port in ((ctx['web_enabled'], ctx['https_web_port']),
-                                           (MANAGER.exists(), ctx['https_manager_port'])) if wanted]
+                                           (MANAGER.exists(), ctx['https_manager_port']),
+                                           (vpnrouter.metrics_ready(ctx), ctx['metrics_https_port'])) if wanted]
         lines += [
             '',
             '# HTTPS RULES START',
@@ -439,6 +453,57 @@ def manage_admin():
     return ''
 
 
+def manage_metrics_account(ctx):
+    """Keep the service account metrics.service_account names, whose password is in METRICS_SECRET; the problem, or ''.
+
+    While metrics are enabled, setup owns the account: it adds it as a service account with a random password when
+    it is missing, and gives it a new random password, ending any lockout, when METRICS_SECRET is missing or does not
+    hold its password. A disabled account is left as it is. An account of that name that is not a service account is
+    left alone and is a problem.
+    """
+    if not vpnrouter.metrics_ready(ctx):
+        return ''
+    name = ctx['metrics_service_account']
+    try:
+        vpnrouter_users.check_name(name)
+        try:
+            password = METRICS_SECRET.read_text(encoding='utf-8').strip()
+        except OSError:
+            password = ''
+        with vpnrouter_users.session() as connection:
+            try:
+                current = vpnrouter_users.entry(connection, name)
+            except vpnrouter_users.Failure as failure:
+                if failure.status != 3:
+                    raise
+                current = None
+            if current is not None and not vpnrouter_users.service(current):
+                return f'metrics.service_account: {name} is a user, not a service account'
+            # An administrator who disabled the account meant it; its password is left as it is.
+            if current is not None and not vpnrouter_users.summary(current)['enabled']:
+                return ''
+            valid = False
+            if current is not None and password:
+                try:
+                    vpnrouter_users.login(name, password, None)
+                    valid = True
+                except vpnrouter_users.Failure:
+                    pass
+            if not valid:
+                password = secrets.token_urlsafe(32)
+                METRICS_SECRET.touch(mode=0o600)
+                METRICS_SECRET.chmod(0o600)
+                METRICS_SECRET.write_text(password + '\n', encoding='utf-8')
+                if current is None:
+                    vpnrouter_users.add(connection, name, password, {}, service=True)
+                else:
+                    vpnrouter_users.passwd(connection, name, password)
+                    vpnrouter_users.unlock(connection, name)
+    except (OSError, ldap.LDAPError, vpnrouter_users.Failure) as exc:
+        return f'the metrics service account cannot be kept: {exc}'
+    return ''
+
+
 def manage_openvpn(ctx, changed):
     """Run the OpenVPN server when it is configured, and stop it when it is not."""
     unit = 'openvpn-server@vpn-router'
@@ -455,11 +520,12 @@ def manage_openvpn(ctx, changed):
 
 
 def manage_caddy(ctx):
-    """Run Caddy, the HTTPS proxy of the web service and the Router Manager, while HTTPS is ready, and stop it when not.
+    """Run Caddy, the HTTPS proxy of the web service, the Router Manager and the metrics service, while HTTPS is ready, and stop it when not.
 
-    CADDYFILE serves the web service on https.web_port and, while the manager is installed, the
-    manager on https.manager_port, both for wan.local_fqdn with a certificate from Let's Encrypt,
-    and both calling the service on its loopback address.
+    CADDYFILE serves the web service on https.web_port, while the manager is installed the
+    manager on https.manager_port, and while metrics are enabled the metrics service on
+    metrics.https_port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all
+    calling the service on its loopback address.
     """
     if not vpnrouter.https_ready(ctx):
         vpnrouter.remove(CADDYFILE)
@@ -480,6 +546,20 @@ def manage_caddy(ctx):
         run(['systemctl', 'reload', CADDY_UNIT], check=False)
 
 
+def manage_metrics(ctx):
+    """Run the metrics service while metrics are enabled, and stop it when not.
+
+    The service reads its settings itself, so it is restarted only to run the code of a new package version, which
+    every run of setup after an upgrade does.
+    """
+    unit = 'vpn-router-metrics.service'
+    if not vpnrouter.metrics_ready(ctx):
+        run(['systemctl', 'disable', '--now', unit], check=False)
+        return
+    run(['systemctl', 'enable', unit], check=False)
+    run(['systemctl', 'restart', unit], check=False)
+
+
 def run_platform_setup(cfg, ctx):
     """Let the platform module apply its own state, if it defines any."""
     module = vpnrouter.platform_module(cfg)
@@ -621,6 +701,10 @@ def main():
         failed_step = 'Router Manager admin'
         progress('running', failed_step)
         problem = manage_admin()
+    if not problem:
+        failed_step = 'metrics service account'
+        progress('running', failed_step)
+        problem = manage_metrics_account(ctx)
     progress('running', 'OpenVPN')
     manage_openvpn(ctx, changed)
     progress('running', 'HTTPS proxy')
@@ -636,6 +720,8 @@ def main():
     refresh_docker_rules()
     progress('running', 'web service state')
     refresh_web(cfg, ctx)
+    progress('running', 'metrics service')
+    manage_metrics(ctx)
     if problem:
         progress('failed', failed_step, problem)
         print(f'vpn-router: error: {problem}', file=sys.stderr)

debian-package/src/usr/lib/vpn-router/vpnrouter.py

@@ -12,6 +12,7 @@ import re
 import shutil
 import subprocess
 import sys
+import urllib.parse
 
 import jinja2
 
@@ -57,8 +58,18 @@ SCHEMA = {
     'web':        {'enabled': True, 'listen': DEFAULT_LISTEN, 'port': 7081, 'database': '', 'trusted_proxies': [],
                    'status_content': 'basic'},
     'https':      {'enabled': True, 'email': '', 'web_port': 7444, 'manager_port': 7443},
+    'metrics':    {'enabled': True, 'listen': 'internal', 'port': 9464, 'https_port': 9465, 'service_account': 'metrics',
+                   'otlp_endpoint': '', 'otlp_username': '', 'otlp_password': '',
+                   'keep_details': '24 hours', 'keep_hourly': '1 month', 'keep_daily': '3 months',
+                   'keep_monthly': '1 year', 'keep_yearly': '10 years'},
 }
 
+#: The retention settings of the metrics, from the details to the yearly summaries.
+METRICS_KEEP = ('keep_details', 'keep_hourly', 'keep_daily', 'keep_monthly', 'keep_yearly')
+
+#: A retention period as PostgreSQL reads an interval: a count and a unit.
+METRICS_INTERVAL = re.compile(r'^[1-9]\d{0,4} (hour|day|month|year)s?$')
+
 #: The port of Let's Encrypt's HTTP challenge, which Caddy answers while HTTPS is on.
 ACME_HTTP_PORT = 80
 
@@ -291,6 +302,33 @@ def validate(cfg):
     for item in _get_list(cfg, 'web', 'trusted_proxies'):
         _check_address(item, 'web.trusted_proxies')
 
+    taken = {web_port: 'web.port', ACME_HTTP_PORT: 'the HTTP challenge',
+             https_ports[0]: 'https.web_port', https_ports[1]: 'https.manager_port'}
+    for option in ('port', 'https_port'):
+        port = _get(cfg, 'metrics', option)
+        if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
+            raise ConfigError(f'metrics.{option}: {port!r} is not a valid port')
+        if port in taken:
+            raise ConfigError(f'metrics.{option}: {port} is the port of {taken[port]}')
+        taken[port] = f'metrics.{option}'
+
+    metrics_listen = _get(cfg, 'metrics', 'listen')
+    if metrics_listen not in LISTEN_CHOICES:
+        raise ConfigError(
+            f'metrics.listen: {metrics_listen!r} is not one of {", ".join(LISTEN_CHOICES)}')
+
+    endpoint = _get(cfg, 'metrics', 'otlp_endpoint')
+    if endpoint and urllib.parse.urlsplit(endpoint).scheme not in ('http', 'https'):
+        raise ConfigError(f'metrics.otlp_endpoint: {endpoint!r} is not an http or https URL')
+    if bool(_get(cfg, 'metrics', 'otlp_username')) != bool(_get(cfg, 'metrics', 'otlp_password')):
+        raise ConfigError('metrics.otlp_username and metrics.otlp_password are given together')
+
+    for option in METRICS_KEEP:
+        keep = _get(cfg, 'metrics', option)
+        if not METRICS_INTERVAL.match(keep):
+            raise ConfigError(
+                f'metrics.{option}: {keep!r} is not a count and a unit, such as "24 hours" or "3 months"')
+
     docker_mode = _get(cfg, 'docker', 'mode')
     if docker_mode not in DOCKER_MODES:
         raise ConfigError(
@@ -483,6 +521,10 @@ def build_context(cfg, local_id_override=None):
         'https_email':     _get(cfg, 'https', 'email'),
         'https_web_port':  _get(cfg, 'https', 'web_port'),
         'https_manager_port': _get(cfg, 'https', 'manager_port'),
+        'metrics_enabled': _get(cfg, 'metrics', 'enabled'),
+        'metrics_port':    _get(cfg, 'metrics', 'port'),
+        'metrics_https_port': _get(cfg, 'metrics', 'https_port'),
+        'metrics_service_account': _get(cfg, 'metrics', 'service_account'),
         'loopback':        LOOPBACK,
     }
 
@@ -518,6 +560,11 @@ def https_ready(ctx):
     return bool(ctx['https_enabled'] and ctx['local_fqdn'])
 
 
+def metrics_ready(ctx):
+    """The metrics service collects, keeps and serves the metrics while metrics.enabled is true."""
+    return bool(ctx['metrics_enabled'])
+
+
 def _jinja_env():
     return jinja2.Environment(
         loader=jinja2.FileSystemLoader(str(TEMPLATE_DIR)),

debian-package/src/usr/lib/vpn-router/vpnrouter_ldap.py

@@ -53,7 +53,7 @@ OVERLAYS = ('ppolicy', 'otp')
 #: The package's own schema, under an OID of the UUID arc 2.25 (ITU-T X.667): attribute types
 #: under .1 and object classes under .2. vpnRouterAccount, auxiliary, adds properties of the
 #: package to a user of either kind: mail, for a Manager-only account, and vpnRouterScope,
-#: authorization scopes that nothing reads yet.
+#: authorization scopes, of which service marks a service account.
 SCHEMA_OID = '2.25.98477352595735834532404256531620013001'
 SCHEMA_NAME = 'vpn-router'
 SCHEMA_ATTRIBUTE_TYPES = (

debian-package/src/usr/lib/vpn-router/vpnrouter_metrics.py

@@ -0,0 +1,173 @@
+"""The metrics of the router in PostgreSQL, for the metrics service and the web service.
+
+The metrics service writes one detail row per minute to the three detail tables, of TABLES:
+metrics_system (CPU, memory, disk and load averages), metrics_network (bytes per second
+received and sent per interface) and metrics_vpn (clients per kind). Each detail table has four
+summary tables, of LEVELS: <table>_hourly from the details, <table>_daily from the hourly,
+<table>_monthly from the daily and <table>_yearly from the monthly rows. A summary row has the
+start of its period, in UTC, and the average and the maximum of each value over the period: of
+the details for the hourly rows, else the average of the averages and the maximum of the maxima
+below. A period is summarized once it is complete; summarize() catches up every complete period
+after the last summarized one, so a stopped service loses no summary of what was written.
+
+collect_garbage() deletes the rows older than the retention of their level, metrics.keep_*;
+the service runs it on start and once a day. history() answers the views of the Metrics tab.
+
+The tables are in the database of the refresh tokens (vpnrouter_auth.connection_string) and are
+created when missing. Every failure of the database raises DatabaseError.
+"""
+
+import contextlib
+
+import psycopg
+
+import vpnrouter_auth
+
+#: The detail tables: their key columns besides time, and their value columns.
+TABLES = {
+    'metrics_system': ((), ('cpu', 'memory_used', 'memory_total', 'disk_free', 'disk_total', 'load1', 'load5', 'load15')),
+    'metrics_network': (('interface',), ('rx', 'tx')),
+    'metrics_vpn': (('kind',), ('clients',)),
+}
+
+#: The summary levels, from the finest: the suffix of the tables, the period of date_trunc and the retention setting.
+LEVELS = (
+    ('hourly', 'hour', 'keep_hourly'),
+    ('daily', 'day', 'keep_daily'),
+    ('monthly', 'month', 'keep_monthly'),
+    ('yearly', 'year', 'keep_yearly'),
+)
+
+#: The kinds of VPN clients of metrics_vpn; site_to_site is 1 while the tunnel is up.
+VPN_KINDS = ('p2s', 'openvpn', 'wireguard', 'site_to_site')
+
+#: The views of history(): the summary level read (None: the details), the bucket and the span.
+RANGES = {
+    'hour': (None, '1 minute', '1 hour'),
+    'day': (None, '10 minutes', '1 day'),
+    'month': ('hourly', '6 hours', '1 month'),
+    'year': ('monthly', '1 month', '1 year'),
+}
+
+
+class DatabaseError(Exception):
+    """The metrics database failed."""
+
+
+def schema():
+    """The statements that create the detail and the summary tables."""
+    statements = []
+    for table, (keys, values) in TABLES.items():
+        key_columns = ''.join(f', {key} text NOT NULL' for key in keys)
+        primary = ', '.join(('time', *keys))
+        detail = ''.join(f', {value} double precision' for value in values)
+        statements.append(f'CREATE TABLE IF NOT EXISTS {table} (time timestamptz NOT NULL{key_columns}{detail}, '
+                          f'PRIMARY KEY ({primary}))')
+        summary = ''.join(f', {value}_avg double precision, {value}_max double precision' for value in values)
+        for suffix, _, _ in LEVELS:
+            statements.append(f'CREATE TABLE IF NOT EXISTS {table}_{suffix} (time timestamptz NOT NULL{key_columns}{summary}, '
+                              f'PRIMARY KEY ({primary}))')
+    return statements
+
+
+@contextlib.contextmanager
+def database(cfg):
+    """A connection to the metrics database, with its tables, closed at the end; DatabaseError on any failure."""
+    try:
+        with psycopg.connect(vpnrouter_auth.connection_string(cfg), autocommit=True) as connection:
+            for statement in schema():
+                connection.execute(statement)
+            yield connection
+    except (psycopg.Error, OSError) as exc:
+        raise DatabaseError(f'the metrics database failed: {exc}') from exc
+
+
+def write(connection, system, network, vpn):
+    """Write a sample as the detail rows of the current minute.
+
+    system maps the values of metrics_system to numbers; network maps an interface to its rx and
+    tx; vpn maps a kind of VPN_KINDS to its clients. A value that is None is written as NULL.
+    """
+    with connection.transaction():
+        minute = connection.execute("SELECT date_trunc('minute', now(), 'UTC')").fetchone()[0]
+        names = TABLES['metrics_system'][1]
+        connection.execute(f"INSERT INTO metrics_system (time, {', '.join(names)}) "
+                           f"VALUES (%s{', %s' * len(names)}) ON CONFLICT DO NOTHING",
+                           (minute, *(system.get(name) for name in names)))
+        for interface, (rx, tx) in network.items():
+            connection.execute('INSERT INTO metrics_network (time, interface, rx, tx) VALUES (%s, %s, %s, %s) '
+                               'ON CONFLICT DO NOTHING', (minute, interface, rx, tx))
+        for kind, clients in vpn.items():
+            connection.execute('INSERT INTO metrics_vpn (time, kind, clients) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING',
+                               (minute, kind, clients))
+
+
+def summarize(connection, level):
+    """Summarize the complete periods of a level of LEVELS that are not summarized yet, for every detail table."""
+    index = [suffix for suffix, _, _ in LEVELS].index(level)
+    _, period, _ = LEVELS[index]
+    for table, (keys, values) in TABLES.items():
+        target = f'{table}_{level}'
+        source = table if index == 0 else f'{table}_{LEVELS[index - 1][0]}'
+        if index == 0:
+            aggregates = ', '.join(f'avg({value}), max({value})' for value in values)
+        else:
+            aggregates = ', '.join(f'avg({value}_avg), max({value}_max)' for value in values)
+        columns = ', '.join(('time', *keys, *(f'{value}_{kind}' for value in values for kind in ('avg', 'max'))))
+        groups = ''.join(f', {key}' for key in keys)
+        connection.execute(
+            f"INSERT INTO {target} ({columns}) "
+            f"SELECT date_trunc('{period}', time, 'UTC') AS period{groups}, {aggregates} FROM {source} "
+            f"WHERE time < date_trunc('{period}', now(), 'UTC') "
+            f"AND time >= coalesce((SELECT max(time) FROM {target}) + interval '1 {period}', '-infinity') "
+            f"GROUP BY period{groups} ON CONFLICT DO NOTHING")
+
+
+def collect_garbage(connection, cfg):
+    """Delete the rows older than the retention of their level: metrics.keep_details, keep_hourly and so on."""
+    metrics = cfg['metrics']
+    for table in TABLES:
+        connection.execute(f'DELETE FROM {table} WHERE time < now() - %s::interval', (metrics['keep_details'],))
+        for suffix, _, keep in LEVELS:
+            connection.execute(f'DELETE FROM {table}_{suffix} WHERE time < now() - %s::interval', (metrics[keep],))
+
+
+def history(connection, view):
+    """The points of a view of RANGES, oldest first, each with the start of its bucket in Unix seconds.
+
+    A point has the average and the maximum of each value of metrics_system, network, which maps
+    each interface to its rx and tx, and clients, which maps each kind of VPN client to its
+    clients; a value without data is None.
+    """
+    level, bucket, span = RANGES[view]
+
+    def rows(table):
+        keys, values = TABLES[table]
+        source = table if level is None else f'{table}_{level}'
+        if level is None:
+            aggregates = ', '.join(f'avg({value}), max({value})' for value in values)
+        else:
+            aggregates = ', '.join(f'avg({value}_avg), max({value}_max)' for value in values)
+        start = (f"date_bin('{bucket}', time, timestamptz '2000-01-01 00:00:00+00')" if view != 'year'
+                 else "date_trunc('month', time, 'UTC')")
+        groups = ''.join(f', {key}' for key in keys)
+        return connection.execute(
+            f"SELECT extract(epoch FROM {start})::bigint AS bucket{groups}, {aggregates} FROM {source} "
+            f"WHERE time >= now() - interval '{span}' GROUP BY bucket{groups} ORDER BY bucket").fetchall()
+
+    def pairs(row, values):
+        return {value: {'avg': row[2 * index], 'max': row[2 * index + 1]} for index, value in enumerate(values)}
+
+    points = {}
+
+    def point(start):
+        return points.setdefault(start, {'time': start, 'network': {}, 'clients': {}})
+
+    for row in rows('metrics_system'):
+        point(row[0]).update(pairs(row[1:], TABLES['metrics_system'][1]))
+    for row in rows('metrics_network'):
+        point(row[0])['network'][row[1]] = pairs(row[2:], TABLES['metrics_network'][1])
+    for row in rows('metrics_vpn'):
+        point(row[0])['clients'][row[1]] = pairs(row[2:], TABLES['metrics_vpn'][1])['clients']
+    empty = {value: {'avg': None, 'max': None} for value in TABLES['metrics_system'][1]}
+    return {'range': view, 'bucket': bucket, 'points': [{**empty, **points[start]} for start in sorted(points)]}

debian-package/src/usr/lib/vpn-router/vpnrouter_status.py

@@ -0,0 +1,111 @@
+"""What the router's VPN systems report, for the web service and the metrics service.
+
+Each reader asks the running system once and answers None when it cannot be asked: swanctl for
+the IKE SAs, wg for the WireGuard peers and the OpenVPN server's status file for its clients.
+"""
+
+import subprocess
+import time
+
+COMMAND_TIMEOUT = 5
+
+OPENVPN_STATUS = '/run/openvpn-server/status-vpn-router.log'
+
+#: Seconds since its last handshake within which a WireGuard peer counts as connected.
+WIREGUARD_CONNECTED_SECONDS = 180
+
+
+def run(*args):
+    """The finished process of a command, or None when it cannot be run."""
+    try:
+        return subprocess.run(args, capture_output=True, text=True, timeout=COMMAND_TIMEOUT)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+
+
+def swanctl(command):
+    """The entries of a swanctl list command, or None when swanctl cannot be asked.
+
+    The --pretty output has one item per line: "name {" opens a section, "key = [" a list
+    whose items follow one per line, "}" and "]" close them, and "key = value" is a pair
+    whose value may contain spaces. Every "... event {" section at the top holds one
+    connection or SA; each becomes an entry with its name under "name". The closing
+    "... reply {" section is left out. Values stay strings, as swanctl prints them.
+    Output with more closing than opening lines gives None.
+    """
+    result = run('swanctl', command, '--pretty')
+    if result is None or result.returncode != 0:
+        return None
+    events = []
+    stack = []
+    for line in result.stdout.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        if line in ('}', ']'):
+            if not stack:
+                return None
+            stack.pop()
+        elif not stack:
+            section = {}
+            if line.endswith(' event {'):
+                events.append(section)
+            stack.append(section)
+        elif isinstance(stack[-1], list):
+            stack[-1].append(line)
+        elif line.endswith(' {'):
+            stack[-1][line[:-2]] = section = {}
+            stack.append(section)
+        elif line.endswith(' = ['):
+            stack[-1][line[:-4]] = items = []
+            stack.append(items)
+        else:
+            key, _, value = line.partition(' = ')
+            stack[-1][key] = value
+    return [dict(body, name=name) for event in events for name, body in event.items()]
+
+
+def established(connection):
+    """The number of established IKE SAs of the connection, or None when swanctl cannot be asked."""
+    sas = swanctl('--list-sas')
+    if sas is None:
+        return None
+    return sum(1 for sa in sas if sa['name'] == connection and sa.get('state') == 'ESTABLISHED')
+
+
+def wireguard_peers():
+    """One entry per WireGuard peer, with its handshake age and traffic, or None when wg0 is down."""
+    result = run('wg', 'show', 'wg0', 'dump')
+    if result is None or result.returncode != 0:
+        return None
+    now = int(time.time())
+    peers = []
+    for line in result.stdout.splitlines()[1:]:
+        fields = line.split('\t')
+        if len(fields) < 7:
+            continue
+        handshake = int(fields[4])
+        peers.append({
+            'handshake_seconds_ago': now - handshake if handshake else None,
+            'received_bytes': int(fields[5]),
+            'sent_bytes': int(fields[6]),
+        })
+    return peers
+
+
+def wireguard_connected():
+    """The number of WireGuard peers with a handshake in the last WIREGUARD_CONNECTED_SECONDS, or None when wg0 is down."""
+    peers = wireguard_peers()
+    if peers is None:
+        return None
+    return sum(1 for peer in peers
+               if peer['handshake_seconds_ago'] is not None and peer['handshake_seconds_ago'] < WIREGUARD_CONNECTED_SECONDS)
+
+
+def openvpn_clients():
+    """The number of connected OpenVPN clients, from the server's status file, or None when it cannot be read."""
+    try:
+        with open(OPENVPN_STATUS, encoding='utf-8') as handle:
+            return sum(1 for line in handle if line.startswith('CLIENT_LIST,'))
+    except OSError:
+        return None

debian-package/src/usr/lib/vpn-router/vpnrouter_users.py

@@ -6,8 +6,10 @@ password OpenVPN checks, and a client certificate of the router CA with the user
 common name, which OpenVPN and IKEv2 accept. A Manager-only account is an account,
 simpleSecurityObject and vpnRouterAccount (the package's schema) with a name, a password and
 optionally an e-mail address, no names, no certificate and so no VPN access; it is added when
-neither the first nor the last name is given, and can sign in to the web service. Users exist
-whether or not a P2S system is enabled.
+neither the first nor the last name is given, and can sign in to the web service. A service
+account is a Manager-only account with the scope SERVICE_SCOPE in vpnRouterScope, which
+vpn-router-setup adds and keeps for the metrics service, as metrics.service_account names it; it
+cannot sign in to the web service. Users exist whether or not a P2S system is enabled.
 
 A group is a groupOfNames in ou=groups with a name, which follows the rules of user names, an
 optional description and at least one member; a group cannot be empty, so the last member
@@ -62,7 +64,9 @@ EMAIL_MAX = 254
 #: The keys of the person attributes and their LDAP attributes.
 PERSON = (('first_name', 'givenName'), ('last_name', 'sn'), ('email', 'mail'))
 #: The attributes read for a user.
-ATTRIBUTES = ['uid', 'objectClass', 'pwdAccountLockedTime', 'givenName', 'sn', 'mail', 'oathTOTPToken']
+ATTRIBUTES = ['uid', 'objectClass', 'pwdAccountLockedTime', 'givenName', 'sn', 'mail', 'oathTOTPToken', 'vpnRouterScope']
+#: The vpnRouterScope of a service account.
+SERVICE_SCOPE = 'service'
 #: The object classes of a VPN user and of a Manager-only account.
 VPN_USER_CLASSES = [b'inetOrgPerson']
 MANAGER_ONLY_CLASSES = [b'account', b'simpleSecurityObject', b'vpnRouterAccount']
@@ -183,6 +187,11 @@ def manager_only(attributes):
     return b'account' in (value.lower() for value in attributes.get('objectClass', []))
 
 
+def service(attributes):
+    """Whether the entry is a service account."""
+    return SERVICE_SCOPE.encode() in (value.lower() for value in attributes.get('vpnRouterScope', []))
+
+
 def summary(attributes):
     """A user as the answers show it; a person attribute the entry does not have is None."""
     name = attributes['uid'][0].decode()
@@ -190,6 +199,7 @@ def summary(attributes):
     user = {
         'name': name,
         'manager_only': manager_only(attributes),
+        'service': service(attributes),
         'enabled': lock != LOCKED,
         'locked': lock not in (None, LOCKED) and locked_since(lock) > time.time() - vpnrouter_ldap.LOCKOUT_SECONDS,
         'certificate': certificate(name)[0].is_file(),
@@ -213,12 +223,15 @@ def list_users(connection):
     return {'users': sorted((summary(attributes) for _, attributes in found), key=lambda user: user['name'])}
 
 
-def add(connection, name, password, values):
+def add(connection, name, password, values, service=False):
     """Add a user: a VPN user with its certificate when values gives all three person attributes, a Manager-only account
-    with at most the e-mail address when it gives neither name."""
+    with at most the e-mail address when it gives neither name, and, with service, which only vpn-router-setup uses, a
+    service account, with at most the e-mail address too."""
     check_name(name)
     details = person(values, required=False)
     vpn_user = 'givenName' in details or 'sn' in details
+    if service and vpn_user:
+        raise Failure('a service account has no first or last name', 2)
     if vpn_user and len(details) != len(PERSON):
         raise Failure('give first_name, last_name and email for a VPN user, or at most email for a Manager-only account', 2)
     check_password(password)
@@ -229,6 +242,8 @@ def add(connection, name, password, values):
     else:
         # simpleSecurityObject requires a userPassword; passwd_s sets the real one at once.
         record = {'objectClass': MANAGER_ONLY_CLASSES, 'uid': [name.encode()], 'userPassword': [b'{CRYPT}!']}
+        if service:
+            record['vpnRouterScope'] = [SERVICE_SCOPE.encode()]
     record.update({attribute: [value.encode()] for attribute, value in details.items()})
     try:
         connection.add_s(dn_of(name), ldap.modlist.addModlist(record))

debian-package/src/usr/lib/vpn-router/web

@@ -18,8 +18,11 @@ Pages:
   /account the account of the signed-in user: its details, its password and its TOTP
            enrolment (see account())
   /auth    sign-in with a user's password and TOTP code, refresh and sign-out (see auth())
+  /metrics/history
+           the metrics the metrics service keeps, for a view: hour, day, month or year (see
+           metrics_history())
 
-/config, /users, /groups and /account need an HS256 JWT in an Authorization: Bearer header signed
+/config, /metrics/history, /users, /groups and /account need an HS256 JWT in an Authorization: Bearer header signed
 with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub;
 /account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A
 token of vpn-router-token has the admin claim and names no user; it may call every path but
@@ -57,10 +60,10 @@ import collections
 import json
 import os
 import signal
-import subprocess
 import sys
 import threading
 import time
+import urllib.parse
 from html import escape
 from http.server import BaseHTTPRequestHandler
 
@@ -70,9 +73,11 @@ import ldap
 
 import vpnrouter
 import vpnrouter_auth
+import vpnrouter_metrics
 import vpnrouter_net
 import vpnrouter_serve
 import vpnrouter_users
+from vpnrouter_status import established, openvpn_clients, run, swanctl, wireguard_peers
 
 STATE_FILE = '/run/vpn-router/web.json'
 CONFIG_FILE = '/etc/vpn-router/vpn-router.json'
@@ -85,7 +90,6 @@ DEFAULT_PORT = 7081
 
 #: Seconds between two discoveries of where the service should listen.
 DISCOVERY_SECONDS = 30
-COMMAND_TIMEOUT = 5
 CACHE_SECONDS = 2
 
 #: The calls of /auth allowed per client address in AUTH_WINDOW seconds.
@@ -97,8 +101,6 @@ TAILWIND = 'https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4'
 #: Seconds after which the status page in a browser reloads itself.
 REFRESH_SECONDS = 60
 
-OPENVPN_STATUS = '/run/openvpn-server/status-vpn-router.log'
-
 #: The largest request body accepted, in bytes.
 BODY_LIMIT = 4096
 
@@ -138,14 +140,6 @@ def current():
     return dict(read_json(STATE_FILE), key=vpnrouter_auth.key())
 
 
-def run(*args):
-    """The finished process of a command, or None when it cannot be run."""
-    try:
-        return subprocess.run(args, capture_output=True, text=True, timeout=COMMAND_TIMEOUT)
-    except (OSError, subprocess.TimeoutExpired):
-        return None
-
-
 def unit_state(unit):
     result = run('systemctl', 'is-active', unit)
     return result.stdout.strip() if result and result.stdout.strip() else 'unknown'
@@ -169,85 +163,6 @@ def docker_status():
     }
 
 
-def swanctl(command):
-    """The entries of a swanctl list command, or None when swanctl cannot be asked.
-
-    The --pretty output has one item per line: "name {" opens a section, "key = [" a list
-    whose items follow one per line, "}" and "]" close them, and "key = value" is a pair
-    whose value may contain spaces. Every "... event {" section at the top holds one
-    connection or SA; each becomes an entry with its name under "name". The closing
-    "... reply {" section is left out. Values stay strings, as swanctl prints them.
-    Output with more closing than opening lines gives None.
-    """
-    result = run('swanctl', command, '--pretty')
-    if result is None or result.returncode != 0:
-        return None
-    events = []
-    stack = []
-    for line in result.stdout.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        if line in ('}', ']'):
-            if not stack:
-                return None
-            stack.pop()
-        elif not stack:
-            section = {}
-            if line.endswith(' event {'):
-                events.append(section)
-            stack.append(section)
-        elif isinstance(stack[-1], list):
-            stack[-1].append(line)
-        elif line.endswith(' {'):
-            stack[-1][line[:-2]] = section = {}
-            stack.append(section)
-        elif line.endswith(' = ['):
-            stack[-1][line[:-4]] = items = []
-            stack.append(items)
-        else:
-            key, _, value = line.partition(' = ')
-            stack[-1][key] = value
-    return [dict(body, name=name) for event in events for name, body in event.items()]
-
-
-def established(connection):
-    """The number of established IKE SAs of the connection, or None when swanctl cannot be asked."""
-    sas = swanctl('--list-sas')
-    if sas is None:
-        return None
-    return sum(1 for sa in sas if sa['name'] == connection and sa.get('state') == 'ESTABLISHED')
-
-
-def wireguard_peers():
-    """One entry per WireGuard peer, with its handshake age and traffic, or None when wg0 is down."""
-    result = run('wg', 'show', 'wg0', 'dump')
-    if result is None or result.returncode != 0:
-        return None
-    now = int(time.time())
-    peers = []
-    for line in result.stdout.splitlines()[1:]:
-        fields = line.split('\t')
-        if len(fields) < 7:
-            continue
-        handshake = int(fields[4])
-        peers.append({
-            'handshake_seconds_ago': now - handshake if handshake else None,
-            'received_bytes': int(fields[5]),
-            'sent_bytes': int(fields[6]),
-        })
-    return peers
-
-
-def openvpn_clients():
-    """The number of connected OpenVPN clients, from the server's status file, or None when it cannot be read."""
-    try:
-        with open(OPENVPN_STATUS, encoding='utf-8') as handle:
-            return sum(1 for line in handle if line.startswith('CLIENT_LIST,'))
-    except OSError:
-        return None
-
-
 def postgresql_clusters():
     """The state of each PostgreSQL cluster unit (postgresql@<version>-<name>), or {'postgresql': 'no cluster'} without one.
 
@@ -453,7 +368,7 @@ def users(state, method, path, body):
 
       GET    /users                   the users
       POST   /users                   add a user: {"name", "password", "first_name", "last_name", "email"};
-                                      without the last three a Manager-only account
+                                      without the names a Manager-only account
       PUT    /users/<name>            change any of {"first_name", "last_name", "email"}
       PUT    /users/<name>/password   set the password: {"password": ...}
       POST   /users/<name>/disable    lock the user out of OpenVPN
@@ -593,6 +508,22 @@ def account(name, method, path, body):
     return 200, document
 
 
+def metrics_history(view):
+    """Answer GET /metrics/history?range=<view> with vpnrouter_metrics.history: the HTTP status and the JSON document.
+
+    view is hour, the default, day, month or year (vpnrouter_metrics.RANGES).
+    """
+    if view not in vpnrouter_metrics.RANGES:
+        return 400, {'error': f'range must be one of {", ".join(vpnrouter_metrics.RANGES)}'}
+    try:
+        with vpnrouter_metrics.database(vpnrouter.load_config()) as connection:
+            return 200, vpnrouter_metrics.history(connection, view)
+    except vpnrouter_metrics.DatabaseError as exc:
+        return 500, {'error': str(exc)}
+    except SystemExit as exc:
+        return 500, {'error': str(exc.code)}
+
+
 def changes_admins(connection, method, parts, body):
     """Whether a /users or /groups request creates, renames to, changes or removes ADMIN_GROUP, or removes a member of it."""
     admins = vpnrouter_users.ADMIN_GROUP
@@ -638,6 +569,9 @@ def auth(path, body):
         cfg = vpnrouter.load_config()
         if path == '/auth':
             vpnrouter_users.login(body.get('name'), body.get('password'), body.get('code'))
+            with vpnrouter_users.session() as connection:
+                if vpnrouter_users.service(vpnrouter_users.entry(connection, body['name'])):
+                    raise vpnrouter_users.Failure('the name, the password or the code is wrong', 5)
             return 200, vpnrouter_auth.issue(cfg, body['name'])
         name = vpnrouter_auth.take(cfg, body.get('refresh_token'))
         if path == '/auth/logout':
@@ -714,7 +648,7 @@ class Handler(BaseHTTPRequestHandler):
         if self.command not in ('GET', 'HEAD'):
             self.close_connection = True
         directory = next((name for name in ('users', 'groups', 'account') if path == f'/{name}' or path.startswith(f'/{name}/')), None)
-        protected = path == '/config' or directory is not None
+        protected = path in ('/config', '/metrics/history') or directory is not None
         signing = path in ('/auth', '/auth/refresh', '/auth/logout')
         state = current() if protected or signing else {}
         claims = self.claims(state['key']) if protected and state['key'] else None
@@ -741,10 +675,14 @@ class Handler(BaseHTTPRequestHandler):
             body = json.dumps({'error': refused[1]}, indent=2).encode() + b'\n'
         elif path == '/config' and self.command in ('GET', 'HEAD'):
             body, kind, code = json.dumps(details(state), indent=2).encode() + b'\n', 'application/json', 200
+        elif path == '/metrics/history' and self.command in ('GET', 'HEAD'):
+            query = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)
+            code, document = metrics_history(query.get('range', ['hour'])[0])
+            body, kind = json.dumps(document).encode() + b'\n', 'application/json'
         elif directory == 'account':
             code, document = account(claims['sub'], self.command, path, payload)
             body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
-        elif protected and path != '/config':
+        elif directory is not None:
             code, document = (users if directory == 'users' else groups)(state, self.command, path, payload)
             body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
         elif self.command not in ('GET', 'HEAD'):

debian-package/src/usr/sbin/vpn-router-user

@@ -115,7 +115,7 @@ def text_of(answer):
     if not users:
         return 'no users'
     head = ('USER', 'KIND', 'STATE', 'CERTIFICATE', 'OTP', 'FIRST NAME', 'LAST NAME', 'E-MAIL')
-    rows = [(user['name'], 'manager' if user['manager_only'] else 'vpn',
+    rows = [(user['name'], 'service' if user['service'] else 'manager' if user['manager_only'] else 'vpn',
              'disabled' if not user['enabled'] else 'locked' if user['locked'] else 'enabled',
              'issued' if user['certificate'] else 'missing',
              'enrolled' if user['otp'] else '-', user['first_name'] or '-', user['last_name'] or '-', user['email'] or '-')

debian-package/src/usr/share/doc/vpn-router/vpn-router.json.example

@@ -56,5 +56,20 @@
     "email": "",
     "web_port": 7444,
     "manager_port": 7443
+  },
+  "metrics": {
+    "enabled": true,
+    "listen": "internal",
+    "port": 9464,
+    "https_port": 9465,
+    "service_account": "metrics",
+    "otlp_endpoint": "",
+    "otlp_username": "",
+    "otlp_password": "",
+    "keep_details": "24 hours",
+    "keep_hourly": "1 month",
+    "keep_daily": "3 months",
+    "keep_monthly": "1 year",
+    "keep_yearly": "10 years"
   }
 }

debian-package/src/usr/share/vpn-router/templates/Caddyfile.j2

@@ -15,3 +15,8 @@
 	reverse_proxy {{ loopback }}:{{ manager_port }}
 }
 {% endif %}
+{%- if metrics_enabled %}
+{{ local_fqdn }}:{{ metrics_https_port }} {
+	reverse_proxy {{ loopback }}:{{ metrics_port }}
+}
+{% endif %}