feature: Caddy serves static web sites from folders on the router, defined in their own database, with files uploaded through the web service or WebDAV, and an optional PowerDNS name server answers for a zone of the sites.
Release / release (push) Successful in 21s
Release / release (push) Successful in 21s
This commit is contained in:
1 parent
77bdbcb058
commit
ea8ca97417
27 files changed
+2234
-134
No files matched your search
@@ -49,6 +49,11 @@ Pages:
|
||||
metrics_history())
|
||||
/logs the last lines of the journal of the router's units and of the files web.log_files lists
|
||||
(see logs())
|
||||
/sites the web sites Caddy serves: the settings of sites, list, add, change and remove (see sites()),
|
||||
and the files of each: list, download, upload, extract a zip archive, create a folder, rename and
|
||||
remove (see Handler.site_files()); a member of a site's editors group may use the site's files
|
||||
/dns the name server of dns.zone: its settings, the records it serves, and the manual records: add and
|
||||
remove (see dns())
|
||||
/postgresql
|
||||
the databases of each PostgreSQL cluster on the router with their sizes, as JSON (see
|
||||
postgresql_databases()); /config has the clusters' settings
|
||||
@@ -57,7 +62,7 @@ Pages:
|
||||
loopback address, where Caddy passes the requests of the HTTPS site; POST /oidc/login counts
|
||||
towards the limit of /auth calls
|
||||
|
||||
/status/full, /config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed
|
||||
/status/full, /config, /metrics/history, /postgresql, /users, /service-accounts, /groups, /site-to-site, /backup, /remote-access, /ca, /logs, /sites, /dns, /shell and /account need an HS256 JWT in an Authorization: Bearer header signed
|
||||
with vpnrouter_auth.WEB_KEY (see vpnrouter_auth.verify). A token of /auth names a user in sub;
|
||||
/account serves that user, and the other paths need a member of vpnrouter_users.ADMIN_GROUP. A
|
||||
token of vpn-router-token has the admin claim and names no user; it may call every path but
|
||||
@@ -78,7 +83,7 @@ whether each service runs, whether Docker is installed, how many of the site-to-
|
||||
and WireGuard. No configuration data, key or connection name is shown, and no address but the client's.
|
||||
|
||||
The service first initializes, without listening: it makes the signing key, the OpenID Provider's key
|
||||
(vpnrouter_oidc) and the refresh tokens' database (vpnrouter_auth), retrying every INIT_RETRY_SECONDS until all are ready and
|
||||
(vpnrouter_oidc), the refresh tokens' database (vpnrouter_auth) and the web sites' database (vpnrouter_sites), retrying every INIT_RETRY_SECONDS until all are ready and
|
||||
logging each failure. Then it listens. Its unit starts it only once wait-for-libpq has found libpq
|
||||
in the linker cache, where psycopg looks it up.
|
||||
|
||||
@@ -100,8 +105,10 @@ import fcntl
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import secrets
|
||||
import select
|
||||
import shutil
|
||||
@@ -124,12 +131,14 @@ import ldap
|
||||
import vpnrouter
|
||||
import vpnrouter_auth
|
||||
import vpnrouter_backup
|
||||
import vpnrouter_dns
|
||||
import vpnrouter_metrics
|
||||
import vpnrouter_net
|
||||
import vpnrouter_oidc
|
||||
import vpnrouter_passkeys
|
||||
import vpnrouter_pki
|
||||
import vpnrouter_serve
|
||||
import vpnrouter_sites
|
||||
import vpnrouter_users
|
||||
from vpnrouter_status import established, established_each, openvpn_clients, run, swanctl, wireguard_peers
|
||||
|
||||
@@ -184,6 +193,15 @@ USER_STATUS = {2: 400, 3: 404, 4: 409, 5: 401}
|
||||
#: The HTTP status for each status of a vpnrouter_pki.Failure; any other is 500.
|
||||
PKI_STATUS = {2: 400, 3: 404, 4: 409}
|
||||
|
||||
#: The HTTP status for each status of a vpnrouter_sites.Failure; any other is 500.
|
||||
SITE_STATUS = {2: 400, 3: 404, 4: 409}
|
||||
|
||||
#: A path of a site's files: the site and the path in its folder.
|
||||
SITE_FILES = re.compile(r'^/sites/([^/]+)/files(?:/(.*))?$')
|
||||
|
||||
#: The bytes in a MiB, the unit of sites.upload_max_mb.
|
||||
MIB = 1024 * 1024
|
||||
|
||||
#: A file /pki/ serves: the certificate or the CRL of a CA, and its content type.
|
||||
PUBLISHED = {'.crt': 'application/pkix-cert', '.crl': 'application/pkix-crl'}
|
||||
|
||||
@@ -1128,6 +1146,149 @@ def logs(method, path, query):
|
||||
return 404, {'error': f'no route {method} {path}'}
|
||||
|
||||
|
||||
def section_change(name, body):
|
||||
"""Change any of the settings of the configuration's section as /backup changes its own; the section, or the HTTP
|
||||
status and the error document as a tuple."""
|
||||
settings = {option: body[option] for option in vpnrouter.SCHEMA[name] if option in body}
|
||||
try:
|
||||
with vpnrouter.config_lock():
|
||||
data = vpnrouter.read_settings()
|
||||
current = data.get(name)
|
||||
data[name] = {**(current if isinstance(current, dict) else {}), **settings}
|
||||
cfg = vpnrouter.with_defaults(data)
|
||||
vpnrouter.validate(cfg)
|
||||
vpnrouter.save_config(data)
|
||||
except vpnrouter.ConfigError as exc:
|
||||
return 400, {'error': str(exc)}
|
||||
except OSError as exc:
|
||||
return 500, {'error': f'the configuration cannot be written: {exc}'}
|
||||
except SystemExit as exc:
|
||||
return 500, {'error': str(exc.code)}
|
||||
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
|
||||
return cfg[name]
|
||||
|
||||
|
||||
def sites(method, path, query, body):
|
||||
"""Answer a /sites request but one of a site's files with vpnrouter_sites: the HTTP status and the JSON document.
|
||||
|
||||
GET /sites {"settings", "sites"}: the sites section of the configuration and the sites, each with its
|
||||
settings but private_key, with private_key_set, modified and its folder
|
||||
PUT /sites change any of the settings of the sites section
|
||||
POST /sites add a site: {"name", and any of "enabled", "hosts", "port", "listen", "tls", "ca",
|
||||
"certificate", "private_key", "index_file", "spa", "editors"}; an existing name is 409
|
||||
GET /sites/<name> the site
|
||||
PUT /sites/<name> change any of the settings of POST but name; an empty or absent private_key keeps the key
|
||||
DELETE /sites/<name> remove the site, with content=true also its folder
|
||||
|
||||
A change is applied by restarting vpn-router-setup without waiting for it.
|
||||
"""
|
||||
if body == 'invalid' or (body is not None and not isinstance(body, dict)):
|
||||
return 400, {'error': 'the body must be a JSON object'}
|
||||
body = body or {}
|
||||
parts = path.strip('/').split('/')
|
||||
route = (method, len(parts))
|
||||
try:
|
||||
cfg = vpnrouter.load_config()
|
||||
if route == ('GET', 1):
|
||||
return 200, {'settings': cfg['sites'], 'sites': vpnrouter_sites.listing()}
|
||||
if route == ('PUT', 1):
|
||||
changed = section_change('sites', body)
|
||||
return changed if isinstance(changed, tuple) else (200, {'settings': changed})
|
||||
if route == ('POST', 1):
|
||||
name = body.get('name')
|
||||
document = vpnrouter_sites.save(cfg, name, {key: value for key, value in body.items() if key != 'name'},
|
||||
True)
|
||||
code = 201
|
||||
elif route == ('GET', 2):
|
||||
with vpnrouter_sites.database() as connection:
|
||||
return 200, vpnrouter_sites.shown(vpnrouter_sites.get(connection, parts[1]))
|
||||
elif route == ('PUT', 2):
|
||||
document, code = vpnrouter_sites.save(cfg, parts[1], body, False), 200
|
||||
elif route == ('DELETE', 2):
|
||||
document = vpnrouter_sites.remove(parts[1], query.get('content', [''])[0] == 'true')
|
||||
code = 200
|
||||
else:
|
||||
return 404, {'error': f'no route {method} {path}'}
|
||||
except vpnrouter_sites.Failure as failure:
|
||||
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
|
||||
except vpnrouter_sites.DatabaseError as exc:
|
||||
return 500, {'error': str(exc)}
|
||||
except (ldap.LDAPError, OSError) as exc:
|
||||
return 500, {'error': f'the site cannot be changed: {exc}'}
|
||||
except SystemExit as exc:
|
||||
return 500, {'error': str(exc.code)}
|
||||
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
|
||||
return code, document
|
||||
|
||||
|
||||
def site_file_action(name, target, body):
|
||||
"""Answer a POST to a site's file or folder: {"folder": true} creates the folder, {"rename": "<path>"} moves the file
|
||||
or folder to the path in the site's folder; the HTTP status and the JSON document."""
|
||||
if not isinstance(body, dict):
|
||||
return 400, {'error': 'the body must be a JSON object'}
|
||||
try:
|
||||
if body.get('folder') is True:
|
||||
vpnrouter_sites.make_folder(target)
|
||||
return 201, {'path': vpnrouter_sites.relative_of(name, target), 'folder': True}
|
||||
if isinstance(body.get('rename'), str):
|
||||
moved = vpnrouter_sites.rename(name, target, body['rename'])
|
||||
return 200, {'path': vpnrouter_sites.relative_of(name, moved)}
|
||||
except vpnrouter_sites.Failure as failure:
|
||||
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
|
||||
except OSError as exc:
|
||||
return 500, {'error': f'{exc.filename or target.name}: {exc.strerror}'}
|
||||
return 400, {'error': 'give {"folder": true} or {"rename": "<path>"}'}
|
||||
|
||||
|
||||
def dns(method, path, query, body):
|
||||
"""Answer a /dns request with vpnrouter_dns: the HTTP status and the JSON document.
|
||||
|
||||
GET /dns {"settings", "installed", "running", "records", "manual", "problem"}: the dns section of the
|
||||
configuration, whether PowerDNS is installed and runs, the records the name server serves,
|
||||
the manual records, and why the records cannot be read, null when they can
|
||||
PUT /dns change any of the settings of the dns section
|
||||
POST /dns/records add a manual record: {"name", "type", "content", "ttl"?}; name is relative to dns.zone, @
|
||||
for the zone itself, or absolute with a final dot; ttl is dns.ttl without it
|
||||
DELETE /dns/records remove the manual records of name= and type=, with content= only that one
|
||||
|
||||
A change is applied by restarting vpn-router-setup without waiting for it.
|
||||
"""
|
||||
if body == 'invalid' or (body is not None and not isinstance(body, dict)):
|
||||
return 400, {'error': 'the body must be a JSON object'}
|
||||
body = body or {}
|
||||
try:
|
||||
cfg = vpnrouter.load_config()
|
||||
if (method, path) == ('GET', '/dns'):
|
||||
served, manual, problem = [], [], None
|
||||
try:
|
||||
manual = vpnrouter_dns.manual()
|
||||
served = vpnrouter_dns.served(cfg['dns']['zone'].strip())
|
||||
except vpnrouter_dns.DatabaseError as exc:
|
||||
problem = str(exc)
|
||||
return 200, {'settings': cfg['dns'], 'installed': vpnrouter_dns.PDNS_SERVER.exists(),
|
||||
'running': unit_state(vpnrouter_dns.UNIT) == 'active', 'records': served, 'manual': manual,
|
||||
'problem': problem}
|
||||
if (method, path) == ('PUT', '/dns'):
|
||||
changed = section_change('dns', body)
|
||||
return changed if isinstance(changed, tuple) else (200, {'settings': changed})
|
||||
if (method, path) == ('POST', '/dns/records'):
|
||||
document, code = vpnrouter_dns.add(cfg, body), 201
|
||||
elif (method, path) == ('DELETE', '/dns/records'):
|
||||
content = query.get('content', [None])[0]
|
||||
document = vpnrouter_dns.remove(cfg, query.get('name', [''])[0], query.get('type', [''])[0], content)
|
||||
code = 200
|
||||
else:
|
||||
return 404, {'error': f'no route {method} {path}'}
|
||||
except vpnrouter_dns.Failure as failure:
|
||||
return (404 if failure.status == 3 else 400), {'error': str(failure)}
|
||||
except vpnrouter_dns.DatabaseError as exc:
|
||||
return 500, {'error': str(exc)}
|
||||
except SystemExit as exc:
|
||||
return 500, {'error': str(exc.code)}
|
||||
run('systemctl', 'restart', '--no-block', 'vpn-router-setup.service')
|
||||
return code, document
|
||||
|
||||
|
||||
def metrics_history(view):
|
||||
"""Answer GET /metrics/history?range=<view> with vpnrouter_metrics.history: the HTTP status and the JSON document.
|
||||
|
||||
@@ -1160,7 +1321,10 @@ def changes_admins(connection, method, parts, body):
|
||||
|
||||
|
||||
def refusal(claims, method, path, body):
|
||||
"""Why the holder of the claims may not make the request to a protected path, or '' when it may."""
|
||||
"""Why the holder of the claims may not make the request to a protected path, or '' when it may.
|
||||
|
||||
The files of a site may also be read and changed by a member of the site's editors group.
|
||||
"""
|
||||
name = claims.get('sub')
|
||||
if path == '/account' or path.startswith('/account/'):
|
||||
return '' if isinstance(name, str) else 'the token names no user'
|
||||
@@ -1171,6 +1335,9 @@ def refusal(claims, method, path, body):
|
||||
admins = vpnrouter_users.ADMIN_GROUP
|
||||
with vpnrouter_users.session() as connection:
|
||||
if not vpnrouter_users.is_admin(connection, name):
|
||||
files = SITE_FILES.match(path)
|
||||
if files and files.group(1) in vpnrouter_sites.editable(vpnrouter_users.groups_of(connection, name)):
|
||||
return ''
|
||||
return f'user {name} is not a member of {admins}'
|
||||
parts = path.strip('/').split('/')
|
||||
if changes_admins(connection, method, parts, body if isinstance(body, dict) else {}):
|
||||
@@ -1358,8 +1525,66 @@ class Handler(BaseHTTPRequestHandler):
|
||||
reason = refusal(claims, self.command, path, payload)
|
||||
except (ldap.LDAPError, OSError) as exc:
|
||||
return 500, f'the user directory failed: {exc}'
|
||||
except vpnrouter_sites.DatabaseError as exc:
|
||||
return 500, str(exc)
|
||||
return (403, reason) if reason else None
|
||||
|
||||
def site_files(self, name, relative, query, payload, send_body):
|
||||
"""Answer a request of a site's files: the HTTP status and the JSON document, or None once a file was sent.
|
||||
|
||||
GET /sites/<name>/files/<path> a folder: {"path", "entries": [{name, folder, size, modified}]}; a file:
|
||||
the file itself, as an attachment
|
||||
PUT /sites/<name>/files/<path> write the body to the file, its folders created; with unzip=true the body
|
||||
is a zip archive extracted into the folder of the path: {"path", "files"}
|
||||
POST /sites/<name>/files/<path> {"folder": true} creates the folder, {"rename": "<path>"} moves it
|
||||
DELETE /sites/<name>/files/<path> remove the file or folder
|
||||
|
||||
The body of a PUT needs a Content-Length of at most sites.upload_max_mb MiB.
|
||||
"""
|
||||
try:
|
||||
target = vpnrouter_sites.resolve(name, urllib.parse.unquote(relative))
|
||||
if self.command in ('GET', 'HEAD'):
|
||||
if target.is_dir():
|
||||
return 200, vpnrouter_sites.entries(name, target)
|
||||
if not target.is_file():
|
||||
return 404, {'error': f'{relative} does not exist'}
|
||||
with open(target, 'rb') as handle:
|
||||
size = os.fstat(handle.fileno()).st_size
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', mimetypes.guess_type(target.name)[0] or 'application/octet-stream')
|
||||
self.send_header('Content-Length', str(size))
|
||||
self.send_header('Content-Disposition',
|
||||
f"attachment; filename*=UTF-8''{urllib.parse.quote(target.name)}")
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
self.end_headers()
|
||||
if send_body:
|
||||
shutil.copyfileobj(handle, self.wfile, vpnrouter_sites.CHUNK)
|
||||
return None
|
||||
if self.command == 'PUT':
|
||||
length = self.headers.get('Content-Length')
|
||||
if not length or not length.isdigit():
|
||||
return 411, {'error': 'the body needs a Content-Length'}
|
||||
limit = vpnrouter.load_config()['sites']['upload_max_mb'] * MIB
|
||||
if query.get('unzip', [''])[0] == 'true':
|
||||
count = vpnrouter_sites.extract(self.rfile, int(length), target, limit)
|
||||
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'files': count}
|
||||
vpnrouter_sites.receive(self.rfile, int(length), target, limit)
|
||||
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'files': 1}
|
||||
if self.command == 'POST':
|
||||
return site_file_action(name, target, payload)
|
||||
if self.command == 'DELETE':
|
||||
if not target.exists() and not target.is_symlink():
|
||||
return 404, {'error': f'{relative} does not exist'}
|
||||
vpnrouter_sites.delete(name, target)
|
||||
return 200, {'path': vpnrouter_sites.relative_of(name, target), 'removed': True}
|
||||
except vpnrouter_sites.Failure as failure:
|
||||
return SITE_STATUS.get(failure.status, 500), {'error': str(failure)}
|
||||
except OSError as exc:
|
||||
return 500, {'error': f'{exc.filename or relative}: {exc.strerror}'}
|
||||
except SystemExit as exc:
|
||||
return 500, {'error': str(exc.code)}
|
||||
return 405, {'error': f'{self.command} is not allowed on a site\'s files'}
|
||||
|
||||
def exact(self, size):
|
||||
"""The next size bytes from the client; ConnectionError when it closes first."""
|
||||
data = self.rfile.read(size)
|
||||
@@ -1546,15 +1771,19 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if self.command not in ('GET', 'HEAD'):
|
||||
self.close_connection = True
|
||||
directory = next((name for name in ('users', 'service-accounts', 'groups', 'account', 'site-to-site', 'backup',
|
||||
'ca', 'shell', 'remote-access', 'logs')
|
||||
'ca', 'shell', 'remote-access', 'logs', 'sites', 'dns')
|
||||
if path == f'/{name}' or path.startswith(f'/{name}/')), None)
|
||||
protected = path in ('/status/full', '/config', '/metrics/history', '/postgresql') or directory is not None
|
||||
signing = path in ('/auth', '/auth/passkey', '/auth/passkey/complete', '/auth/refresh', '/auth/logout')
|
||||
oidc = (path == '/.well-known/openid-configuration' or path.startswith('/oidc/')) and \
|
||||
self.server.server_address[0] == vpnrouter_net.LOOPBACK
|
||||
files = SITE_FILES.match(path) if directory == 'sites' else None
|
||||
query = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)
|
||||
state = current() if protected or signing else {}
|
||||
claims = self.claims(state['key']) if protected and state['key'] else None
|
||||
payload = self.request_body() if (protected or signing) and self.command not in ('GET', 'HEAD') else None
|
||||
# The body of a file upload is read as a stream, after the request is allowed.
|
||||
payload = self.request_body() if (protected or signing) and self.command not in ('GET', 'HEAD') and \
|
||||
not (files and self.command == 'PUT') else None
|
||||
if claims is not None:
|
||||
self.principal = 'admin' if claims.get('admin') is True else claims.get('sub')
|
||||
if (protected or signing) and not state['key']:
|
||||
@@ -1610,8 +1839,20 @@ class Handler(BaseHTTPRequestHandler):
|
||||
code, document = ca(self.command, path, payload)
|
||||
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
|
||||
elif directory == 'logs':
|
||||
code, document = logs(self.command, path, urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query))
|
||||
code, document = logs(self.command, path, query)
|
||||
body, kind = json.dumps(document).encode() + b'\n', 'application/json'
|
||||
elif files:
|
||||
answered = self.site_files(files.group(1), files.group(2) or '', query, payload, send_body)
|
||||
if answered is None:
|
||||
return
|
||||
code, document = answered
|
||||
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
|
||||
elif directory == 'sites':
|
||||
code, document = sites(self.command, path, query, payload)
|
||||
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
|
||||
elif directory == 'dns':
|
||||
code, document = dns(self.command, path, query, payload)
|
||||
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
|
||||
elif directory == 'service-accounts':
|
||||
code, document = service_accounts(self.command, path, payload)
|
||||
body, kind = json.dumps(document, indent=2).encode() + b'\n', 'application/json'
|
||||
@@ -1700,8 +1941,8 @@ def log(message):
|
||||
|
||||
|
||||
def initialize():
|
||||
"""Make the signing key, the OpenID Provider's key and the refresh tokens' database, retrying every
|
||||
INIT_RETRY_SECONDS until all are ready."""
|
||||
"""Make the signing key, the OpenID Provider's key, the refresh tokens' database and the web sites' database,
|
||||
retrying every INIT_RETRY_SECONDS until all are ready."""
|
||||
reported = None
|
||||
while True:
|
||||
phase = 'reading the configuration'
|
||||
@@ -1716,7 +1957,12 @@ def initialize():
|
||||
phase = 'connecting to the token database'
|
||||
with vpnrouter_auth.database(cfg):
|
||||
pass
|
||||
except (OSError, SystemExit, vpnrouter_auth.DatabaseError) as exc:
|
||||
phase = 'creating the database of the web sites'
|
||||
vpnrouter_sites.provision()
|
||||
phase = 'connecting to the database of the web sites'
|
||||
with vpnrouter_sites.database():
|
||||
pass
|
||||
except (OSError, SystemExit, vpnrouter_auth.DatabaseError, vpnrouter_sites.DatabaseError) as exc:
|
||||
error = f'{phase} failed: {exc.code if isinstance(exc, SystemExit) else exc}'
|
||||
if error != reported:
|
||||
log(f'{error}; retrying every {INIT_RETRY_SECONDS} seconds')
|
||||
|
||||
Reference in new issue
Block a user