feature: Caddy serves static web sites from folders on the router, defined in their own database, with files uploaded through the web service or WebDAV, and an optional PowerDNS name server answers for a zone of the sites.
Release / release (push) Successful in 21s

This commit is contained in:
slawek committed 2026-10-26 15:27:28 +01:00
1 parent 77bdbcb058
commit ea8ca97417
27 files changed
+2234 -134

No files matched your search

+255 -9
View File
@@ -49,6 +49,11 @@ Pages:
metrics_history())
/logs the last lines of the journal of the router's units and of the files web.log_files lists
(see logs())
/sites the web sites Caddy serves: the settings of sites, list, add, change and remove (see sites()),
and the files of each: list, download, upload, extract a zip archive, create a folder, rename and
remove (see Handler.site_files()); a member of a site's editors group may use the site's files
/dns the name server of dns.zone: its settings, the records it serves, and the manual records: add and
remove (see dns())
/postgresql
the databases of each PostgreSQL cluster on the router with their sizes, as JSON (see
postgresql_databases()); /config has the clusters' settings
@@ -57,7 +62,7 @@ Pages:
loopback address, where Caddy passes the requests of the HTTPS site; POST /oidc/login counts
towards the limit of /auth calls
/status/full, /config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed
/status/full, /config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /sites, /dns, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed
with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub;
/account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A
token of vpn-router-token has the admin claim and names no user; it may call every path but
@@ -78,7 +83,7 @@ whether each service runs, whether Docker is installed, how many of the site-to-
and WireGuard. No configuration data, key or connection name is shown, and no address but the client's.
The service first initializes, without listening: it makes the signing key, the OpenID Provider's key
(vpnrouter_oidc) and the refresh tokens' database (vpnrouter_auth), retrying every INIT_RETRY_SECONDS until all are ready and
(vpnrouter_oidc), the refresh tokens' database (vpnrouter_auth) and the web sites' database (vpnrouter_sites), retrying every INIT_RETRY_SECONDS until all are ready and
logging each failure. Then it listens. Its unit starts it only once wait-for-libpq has found libpq
in the linker cache, where psycopg looks it up.
@@ -100,8 +105,10 @@ import fcntl
import hashlib
import ipaddress
import json
import mimetypes
import os
import pwd
import re
import secrets
import select
import shutil
@@ -124,12 +131,14 @@ import ldap
import vpnrouter
import vpnrouter_auth
import vpnrouter_backup
import vpnrouter_dns
import vpnrouter_metrics
import vpnrouter_net
import vpnrouter_oidc
import vpnrouter_passkeys
import vpnrouter_pki
import vpnrouter_serve
import vpnrouter_sites
import vpnrouter_users
from vpnrouter_status import established, established_each, openvpn_clients, run, swanctl, wireguard_peers
@@ -184,6 +193,15 @@ USER_STATUS = {2: 400, 3: 404, 4: 409, 5: 401}
#: The HTTP status for each status of a vpnrouter_pki.Failure; any other is 500.
PKI_STATUS = {2: 400, 3: 404, 4: 409}
#: The HTTP status for each status of a vpnrouter_sites.Failure; any other is 500.
SITE_STATUS = {2: 400, 3: 404, 4: 409}
#: A path of a site's files: the site and the path in its folder.
SITE_FILES = re.compile(r'^/sites/([^/]+)/files(?:/(.*))?$')
#: The bytes in a MiB, the unit of sites.upload_max_mb.
MIB = 1024 * 1024
#: A file /pki/ serves: the certificate or the CRL of a CA, and its content type.
PUBLISHED = {'.crt': 'application/pkix-cert', '.crl': 'application/pkix-crl'}
@@ -1128,6 +1146,149 @@ def logs(method, path, query):
return 404, {'error': f'no route {method} {path}'}
def section_change(name, body):
"""Change any of the settings of the configuration's section as /backup changes its own; the section, or the HTTP
status and the error document as a tuple."""
settings = {option: body[option] for option in vpnrouter.SCHEMA[name] if option in body}
try:
with vpnrouter.config_lock():
data = vpnrouter.read_settings()
current = data.get(name)
data[name] = {**(current if isinstance(current, dict) else {}), **settings}
cfg = vpnrouter.with_defaults(data)
vpnrouter.validate(cfg)
vpnrouter.save_config(data)
except vpnrouter.ConfigError as exc:
return 400, {'error': str(exc)}
except OSError as exc:
return 500, {'error': f'the configuration cannot be written: {exc}'}
except SystemExit as exc:
return 500, {'error': str(exc.code)}
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
return cfg[name]
def sites(method, path, query, body):
"""Answer a /sites request but one of a site's files with vpnrouter_sites: the HTTP status and the JSON document.
GET /sites {"settings", "sites"}: the sites section of the configuration and the sites, each with its
settings but private_key, with private_key_set, modified and its folder
PUT /sites change any of the settings of the sites section
POST /sites add a site: {"name", and any of "enabled", "hosts", "port", "listen", "tls", "ca",
"certificate", "private_key", "index_file", "spa", "editors"}; an existing name is 409
GET /sites/<name> the site
PUT /sites/<name> change any of the settings of POST but name; an empty or absent private_key keeps the key
DELETE /sites/<name> remove the site, with content=true also its folder
A change is applied by restarting vpn-router-setup without waiting for it.
"""
if body == 'invalid' or (body is not None and not isinstance(body, dict)):
return 400, {'error': 'the body must be a JSON object'}
body = body or {}
parts = path.strip('/').split('/')
route = (method, len(parts))
try:
cfg = vpnrouter.load_config()
if route == ('GET', 1):
return 200, {'settings': cfg['sites'], 'sites': vpnrouter_sites.listing()}
if route == ('PUT', 1):
changed = section_change('sites', body)
return changed if isinstance(changed, tuple) else (200, {'settings': changed})
if route == ('POST', 1):
name = body.get('name')
document = vpnrouter_sites.save(cfg, name, {key: value for key, value in body.items() if key != 'name'},
True)
code = 201
elif route == ('GET', 2):
with vpnrouter_sites.database() as connection:
return 200, vpnrouter_sites.shown(vpnrouter_sites.get(connection, parts[1]))
elif route == ('PUT', 2):
document, code = vpnrouter_sites.save(cfg, parts[1], body, False), 200
elif route == ('DELETE', 2):
document = vpnrouter_sites.remove(parts[1], query.get('content', [''])[0] == 'true')
code = 200
else:
return 404, {'error': f'no route {method} {path}'}
except vpnrouter_sites.Failure as failure:
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
except vpnrouter_sites.DatabaseError as exc:
return 500, {'error': str(exc)}
except (ldap.LDAPError, OSError) as exc:
return 500, {'error': f'the site cannot be changed: {exc}'}
except SystemExit as exc:
return 500, {'error': str(exc.code)}
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
return code, document
def site_file_action(name, target, body):
"""Answer a POST to a site's file or folder: {"folder": true} creates the folder, {"rename": "<path>"} moves the file
or folder to the path in the site's folder; the HTTP status and the JSON document."""
if not isinstance(body, dict):
return 400, {'error': 'the body must be a JSON object'}
try:
if body.get('folder') is True:
vpnrouter_sites.make_folder(target)
return 201, {'path': vpnrouter_sites.relative_of(name, target), 'folder': True}
if isinstance(body.get('rename'), str):
moved = vpnrouter_sites.rename(name, target, body['rename'])
return 200, {'path': vpnrouter_sites.relative_of(name, moved)}
except vpnrouter_sites.Failure as failure:
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
except OSError as exc:
return 500, {'error': f'{exc.filename or target.name}: {exc.strerror}'}
return 400, {'error': 'give {"folder": true} or {"rename": "<path>"}'}
def dns(method, path, query, body):
"""Answer a /dns request with vpnrouter_dns: the HTTP status and the JSON document.
GET /dns {"settings", "installed", "running", "records", "manual", "problem"}: the dns section of the
configuration, whether PowerDNS is installed and runs, the records the name server serves,
the manual records, and why the records cannot be read, null when they can
PUT /dns change any of the settings of the dns section
POST /dns/records add a manual record: {"name", "type", "content", "ttl"?}; name is relative to dns.zone, @
for the zone itself, or absolute with a final dot; ttl is dns.ttl without it
DELETE /dns/records remove the manual records of name= and type=, with content= only that one
A change is applied by restarting vpn-router-setup without waiting for it.
"""
if body == 'invalid' or (body is not None and not isinstance(body, dict)):
return 400, {'error': 'the body must be a JSON object'}
body = body or {}
try:
cfg = vpnrouter.load_config()
if (method, path) == ('GET', '/dns'):
served, manual, problem = [], [], None
try:
manual = vpnrouter_dns.manual()
served = vpnrouter_dns.served(cfg['dns']['zone'].strip())
except vpnrouter_dns.DatabaseError as exc:
problem = str(exc)
return 200, {'settings': cfg['dns'], 'installed': vpnrouter_dns.PDNS_SERVER.exists(),
'running': unit_state(vpnrouter_dns.UNIT) == 'active', 'records': served, 'manual': manual,
'problem': problem}
if (method, path) == ('PUT', '/dns'):
changed = section_change('dns', body)
return changed if isinstance(changed, tuple) else (200, {'settings': changed})
if (method, path) == ('POST', '/dns/records'):
document, code = vpnrouter_dns.add(cfg, body), 201
elif (method, path) == ('DELETE', '/dns/records'):
content = query.get('content', [None])[0]
document = vpnrouter_dns.remove(cfg, query.get('name', [''])[0], query.get('type', [''])[0], content)
code = 200
else:
return 404, {'error': f'no route {method} {path}'}
except vpnrouter_dns.Failure as failure:
return (404 if failure.status == 3 else 400), {'error': str(failure)}
except vpnrouter_dns.DatabaseError as exc:
return 500, {'error': str(exc)}
except SystemExit as exc:
return 500, {'error': str(exc.code)}
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
return code, document
def metrics_history(view):
"""Answer GET /metrics/history?range=<view> with vpnrouter_metrics.history: the HTTP status and the JSON document.
@@ -1160,7 +1321,10 @@ def changes_admins(connection, method, parts, body):
def refusal(claims, method, path, body):
"""Why the holder of the claims may not make the request to a protected path, or '' when it may."""
"""Why the holder of the claims may not make the request to a protected path, or '' when it may.
The files of a site may also be read and changed by a member of the site's editors group.
"""
name = claims.get('sub')
if path == '/account' or path.startswith('/account/'):
return '' if isinstance(name, str) else 'the token names no user'
@@ -1171,6 +1335,9 @@ def refusal(claims, method, path, body):
admins = vpnrouter_users.ADMIN_GROUP
with vpnrouter_users.session() as connection:
if not vpnrouter_users.is_admin(connection, name):
files = SITE_FILES.match(path)
if files and files.group(1) in vpnrouter_sites.editable(vpnrouter_users.groups_of(connection, name)):
return ''
return f'user {name} is not a member of {admins}'
parts = path.strip('/').split('/')
if changes_admins(connection, method, parts, body if isinstance(body, dict) else {}):
@@ -1358,8 +1525,66 @@ class Handler(BaseHTTPRequestHandler):
reason = refusal(claims, self.command, path, payload)
except (ldap.LDAPError, OSError) as exc:
return 500, f'the user directory failed: {exc}'
except vpnrouter_sites.DatabaseError as exc:
return 500, str(exc)
return (403, reason) if reason else None
def site_files(self, name, relative, query, payload, send_body):
"""Answer a request of a site's files: the HTTP status and the JSON document, or None once a file was sent.
GET /sites/<name>/files/<path> a folder: {"path", "entries": [{name, folder, size, modified}]}; a file:
the file itself, as an attachment
PUT /sites/<name>/files/<path> write the body to the file, its folders created; with unzip=true the body
is a zip archive extracted into the folder of the path: {"path", "files"}
POST /sites/<name>/files/<path> {"folder": true} creates the folder, {"rename": "<path>"} moves it
DELETE /sites/<name>/files/<path> remove the file or folder
The body of a PUT needs a Content-Length of at most sites.upload_max_mb MiB.
"""
try:
target = vpnrouter_sites.resolve(name, urllib.parse.unquote(relative))
if self.command in ('GET', 'HEAD'):
if target.is_dir():
return 200, vpnrouter_sites.entries(name, target)
if not target.is_file():
return 404, {'error': f'{relative} does not exist'}
with open(target, 'rb') as handle:
size = os.fstat(handle.fileno()).st_size
self.send_response(200)
self.send_header('Content-Type', mimetypes.guess_type(target.name)[0] or 'application/octet-stream')
self.send_header('Content-Length', str(size))
self.send_header('Content-Disposition',
f"attachment; filename*=UTF-8''{urllib.parse.quote(target.name)}")
self.send_header('Cache-Control', 'no-store')
self.end_headers()
if send_body:
shutil.copyfileobj(handle, self.wfile, vpnrouter_sites.CHUNK)
return None
if self.command == 'PUT':
length = self.headers.get('Content-Length')
if not length or not length.isdigit():
return 411, {'error': 'the body needs a Content-Length'}
limit = vpnrouter.load_config()['sites']['upload_max_mb'] * MIB
if query.get('unzip', [''])[0] == 'true':
count = vpnrouter_sites.extract(self.rfile, int(length), target, limit)
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'files': count}
vpnrouter_sites.receive(self.rfile, int(length), target, limit)
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'files': 1}
if self.command == 'POST':
return site_file_action(name, target, payload)
if self.command == 'DELETE':
if not target.exists() and not target.is_symlink():
return 404, {'error': f'{relative} does not exist'}
vpnrouter_sites.delete(name, target)
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'removed': True}
except vpnrouter_sites.Failure as failure:
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
except OSError as exc:
return 500, {'error': f'{exc.filename or relative}: {exc.strerror}'}
except SystemExit as exc:
return 500, {'error': str(exc.code)}
return 405, {'error': f'{self.command} is not allowed on a site\'s files'}
def exact(self, size):
"""The next size bytes from the client; ConnectionError when it closes first."""
data = self.rfile.read(size)
@@ -1546,15 +1771,19 @@ class Handler(BaseHTTPRequestHandler):
if self.command not in ('GET', 'HEAD'):
self.close_connection = True
directory = next((name for name in ('users', 'service-accounts', 'groups', 'account', 'site-to-site', 'backup',
'ca', 'shell', 'remote-access', 'logs')
'ca', 'shell', 'remote-access', 'logs', 'sites', 'dns')
if path == f'/{name}' or path.startswith(f'/{name}/')), None)
protected = path in ('/status/full', '/config', '/metrics/history', '/postgresql') or directory is not None
signing = path in ('/auth', '/auth/passkey', '/auth/passkey/complete', '/auth/refresh', '/auth/logout')
oidc = (path == '/.well-known/openid-configuration' or path.startswith('/oidc/')) and \
self.server.server_address[0] == vpnrouter_net.LOOPBACK
files = SITE_FILES.match(path) if directory == 'sites' else None
query = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)
state = current() if protected or signing else {}
claims = self.claims(state['key']) if protected and state['key'] else None
payload = self.request_body() if (protected or signing) and self.command not in ('GET', 'HEAD') else None
# The body of a file upload is read as a stream, after the request is allowed.
payload = self.request_body() if (protected or signing) and self.command not in ('GET', 'HEAD') and \
not (files and self.command == 'PUT') else None
if claims is not None:
self.principal = 'admin' if claims.get('admin') is True else claims.get('sub')
if (protected or signing) and not state['key']:
@@ -1610,8 +1839,20 @@ class Handler(BaseHTTPRequestHandler):
code, document = ca(self.command, path, payload)
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
elif directory == 'logs':
code, document = logs(self.command, path, urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query))
code, document = logs(self.command, path, query)
body, kind = json.dumps(document).encode() + b'\n', 'application/json'
elif files:
answered = self.site_files(files.group(1), files.group(2) or '', query, payload, send_body)
if answered is None:
return
code, document = answered
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
elif directory == 'sites':
code, document = sites(self.command, path, query, payload)
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
elif directory == 'dns':
code, document = dns(self.command, path, query, payload)
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
elif directory == 'service-accounts':
code, document = service_accounts(self.command, path, payload)
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
@@ -1700,8 +1941,8 @@ def log(message):
def initialize():
"""Make the signing key, the OpenID Provider's key and the refresh tokens' database, retrying every
INIT_RETRY_SECONDS until all are ready."""
"""Make the signing key, the OpenID Provider's key, the refresh tokens' database and the web sites' database,
retrying every INIT_RETRY_SECONDS until all are ready."""
reported = None
while True:
phase = 'reading the configuration'
@@ -1716,7 +1957,12 @@ def initialize():
phase = 'connecting to the token database'
with vpnrouter_auth.database(cfg):
pass
except (OSError, SystemExit, vpnrouter_auth.DatabaseError) as exc:
phase = 'creating the database of the web sites'
vpnrouter_sites.provision()
phase = 'connecting to the database of the web sites'
with vpnrouter_sites.database():
pass
except (OSError, SystemExit, vpnrouter_auth.DatabaseError, vpnrouter_sites.DatabaseError) as exc:
error = f'{phase} failed: {exc.code if isinstance(exc, SystemExit) else exc}'
if error != reported:
log(f'{error}; retrying every {INIT_RETRY_SECONDS} seconds')