slawek/linux-cloud-router
3
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: Caddy serves static web sites from folders on the router, defined in their own database, with files uploaded through the web service or WebDAV, and an optional PowerDNS name server answers for a zone of the sites.
Release / release (push) Successful in 21s
Details

Browse Source
This commit is contained in:
slawek committed 2026-10-26 15:27:28 +01:00
1 parent 77bdbcb058
commit ea8ca97417
27 files changed
+2234 -134
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

debian-package/src/usr/lib/vpn-router/setup
+197 -42
View File
@@ -7,7 +7,7 @@ and service reloads. Idempotent - a second run changes nothing.
Until a run has succeeded (SETUP_DONE), every run starts the bootstrap
services, BOOTSTRAP_UNITS, which show the progress and the failure recorded in
SETUP_STATE on BOOTSTRAP_PORTS, and the firewall opens those ports on the
SETUP_STATE on vpnrouter.BOOTSTRAP_PORTS, and the firewall opens those ports on the
internal interface. The first successful run writes SETUP_DONE, closes the
ports and stops the bootstrap services; a failed run leaves them running.
@@ -33,8 +33,11 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import ldap
import vpnrouter
import vpnrouter_auth
import vpnrouter_dns
import vpnrouter_ldap
import vpnrouter_pki
import vpnrouter_sites
import vpnrouter_users
from vpnrouter import fail, warn
@@ -49,18 +52,12 @@ SETUP_STATE = pathlib.Path('/run/vpn-router/setup.json')
#: Written once a run has succeeded; until then setup runs the bootstrap services and opens their ports.
SETUP_DONE = pathlib.Path('/var/lib/vpn-router/setup.done')
#: The units of the bootstrap services, and their ports: the Router Manager's and the router's.
#: The units of the bootstrap services, whose ports are vpnrouter.BOOTSTRAP_PORTS.
BOOTSTRAP_UNITS = ('vpn-router-bootstrap.service', 'vpn-router-manager-bootstrap.service')
BOOTSTRAP_PORTS = (17080, 17081)
#: The Router Manager, vpn-router-manager, when it is installed, and its settings.
MANAGER = pathlib.Path('/usr/lib/vpn-router-manager/manager')
MANAGER_CONFIG = pathlib.Path('/etc/vpn-router-manager/manager.json')
#: The default admin of the Router Manager, {"username", "password"}, which its package writes from
#: its debconf answers; setup creates the account and removes the file.
MANAGER_ADMIN = pathlib.Path('/etc/vpn-router-manager/admin.json')
MANAGER_PORT = 7080
#: The password of the metrics service account, which setup makes and keeps (root only).
METRICS_SECRET = pathlib.Path('/etc/vpn-router/metrics.secret')
@@ -77,9 +74,19 @@ CADDY_GROUP = 'caddy'
#: The unit that installs Caddy from https.caddy_source with vpn-router-caddy, outside setup.
CADDY_INSTALL_UNIT = 'vpn-router-caddy.service'
#: The certificates and keys of the issuing CAs behind Caddy's ACME server, <name>/cert.pem and key.pem.
#: The certificates and keys of the issuing CAs behind Caddy's ACME server and the sites of tls ca, <name>/cert.pem and
#: key.pem.
CADDY_PKI_DIR = pathlib.Path('/etc/caddy/vpn-router-pki')
#: The certificates and keys of the sites of tls file, <site>/cert.pem and key.pem.
CADDY_SITES_DIR = pathlib.Path('/etc/caddy/vpn-router-sites')
#: The unit of rclone's WebDAV server of the sites' folders.
WEBDAV_UNIT = 'vpn-router-webdav.service'
#: The unit that installs the packages of a feature with install-packages, outside setup; {} is the feature.
INSTALL_UNIT = 'vpn-router-install@{}.service'
#: The timer of vpn-router-ca maintain, which keeps the CRLs fresh.
PKI_TIMER = 'vpn-router-pki.timer'
@@ -107,6 +114,8 @@ BLOCKS = (
('# MANAGER RULES START', '# MANAGER RULES END'),
('# HTTPS RULES START', '# HTTPS RULES END'),
('# METRICS RULES START', '# METRICS RULES END'),
('# SITES RULES START', '# SITES RULES END'),
('# DNS RULES START', '# DNS RULES END'),
('# ROUTER FORWARD RULES START', '# ROUTER FORWARD RULES END'),
('# ROUTER NAT RULES START', '# ROUTER NAT RULES END'),
('# ROUTER MSS RULES START', '# ROUTER MSS RULES END'),
@@ -194,15 +203,6 @@ def is_configured(ctx):
vpnrouter.openvpn_ready(ctx)))
def manager_port():
"""The port of the Router Manager: port in its settings, or MANAGER_PORT when it is not set or not a port."""
try:
port = json.loads(MANAGER_CONFIG.read_text(encoding='utf-8')).get('port')
except (OSError, ValueError, AttributeError):
return MANAGER_PORT
return port if isinstance(port, int) and not isinstance(port, bool) and 1 <= port <= 65535 else MANAGER_PORT
def filter_block(ctx):
lines = [
'',
@@ -265,12 +265,13 @@ def filter_block(ctx):
lines += [
'',
'# BOOTSTRAP RULES START',
*(f'-A ufw-before-input -i {ctx["int_iface"]} -p tcp --dport {port} -j ACCEPT' for port in BOOTSTRAP_PORTS),
*(f'-A ufw-before-input -i {ctx["int_iface"]} -p tcp --dport {port} -j ACCEPT'
for port in vpnrouter.BOOTSTRAP_PORTS),
'# BOOTSTRAP RULES END',
]
if MANAGER.exists() and interfaces:
port = manager_port()
if vpnrouter.MANAGER.exists() and interfaces:
port = vpnrouter.manager_port()
lines += [
'',
'# MANAGER RULES START',
@@ -292,8 +293,9 @@ def filter_block(ctx):
publish = ctx['pki_publish']
ports = [port for wanted, port in ((ctx['web_enabled'] or 'https' in publish or ctx['pki_acme'],
ctx['https_web_port']),
(MANAGER.exists(), ctx['https_manager_port']),
(vpnrouter.MANAGER.exists(), ctx['https_manager_port']),
(vpnrouter.metrics_ready(ctx), ctx['metrics_https_port']),
(vpnrouter.webdav_ready(ctx), ctx['webdav_port']),
('http-80' in publish, vpnrouter.ACME_HTTP_PORT),
('https-default' in publish, vpnrouter.HTTPS_DEFAULT_PORT)) if wanted]
lines += [
@@ -305,6 +307,32 @@ def filter_block(ctx):
'# HTTPS RULES END',
]
# Each site's port is open on the interfaces of its listen; a site with a certificate of Let's Encrypt needs the HTTP
# challenge on the external interface.
by_listen = {'internal': [ctx['int_iface']], 'external': [ctx['wan_iface']],
'both': [ctx['int_iface'], ctx['wan_iface']]}
sites = ctx['sites'] or []
rules = sorted({(name, site['port']) for site in sites for name in by_listen[site['listen']] if name})
if ctx['wan_iface'] and any(site['tls'] == 'acme' for site in sites):
rules = sorted(set(rules) | {(ctx['wan_iface'], vpnrouter.ACME_HTTP_PORT)})
if rules:
lines += [
'',
'# SITES RULES START',
*(f'-A ufw-before-input -i {name} -p tcp --dport {port} -j ACCEPT' for name, port in rules),
'# SITES RULES END',
]
dns_interfaces = [name for name in by_listen[ctx['dns_listen']] if name] if vpnrouter.dns_ready(ctx) else []
if dns_interfaces:
lines += [
'',
'# DNS RULES START',
*(f'-A ufw-before-input -i {name} -p {protocol} --dport 53 -j ACCEPT'
for name in dns_interfaces for protocol in ('udp', 'tcp')),
'# DNS RULES END',
]
if vpnrouter.routing_ready(ctx):
lines += ['', '# ROUTER FORWARD RULES START']
for subnet in ctx['local_cidr_list']:
@@ -640,22 +668,37 @@ def manage_openvpn(ctx, changed):
run(['systemctl', 'restart', unit], check=False)
def manage_caddy(cfg, ctx):
"""Run Caddy, the HTTPS proxy of the web service, the Router Manager and the metrics service, while HTTPS is ready, and stop it when not.
def caddy_address(host, port, tls):
"""The address of a site block of the Caddyfile: the host, an IPv6 address in brackets, and the port, with http://
for a plain HTTP site; no host answers any name."""
host = f'[{host}]' if ':' in host else host
return f'{"http://" if tls == "none" else ""}{host}:{port}'
CADDYFILE serves the web service on https.web_port, while the manager is installed the
manager on https.manager_port, and while metrics are enabled the metrics service on
metrics.https_port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all
calling the service on its loopback address. It also serves the CA files of
vpnrouter_pki.PUBLISH_DIR at /pki/ on the endpoints of pki.publish that are Caddy's, and an
ACME server at /acme/<name>/ on https.web_port for each issuing CA with acme and a
certificate, whose certificate and key are copied to CADDY_PKI_DIR.
def manage_caddy(cfg, ctx):
"""Run Caddy, the HTTPS proxy of the router's services and the server of the web sites, while either is wanted, and
stop it when not.
While HTTPS is ready, CADDYFILE serves the web service on https.web_port, while the manager is installed the
manager on https.manager_port, while metrics are enabled the metrics service on metrics.https_port, and while
WebDAV is ready WebDAV on sites.webdav_port, all for wan.local_fqdn with a certificate from Let's Encrypt, and all
calling the service on its loopback address. It also serves the CA files of vpnrouter_pki.PUBLISH_DIR at /pki/ on
the endpoints of pki.publish that are Caddy's, and an ACME server at /acme/<name>/ on https.web_port for each
issuing CA with acme and a certificate.
It serves each site of ctx's sites from its folder, with the TLS of the site: an issuing CA of tls ca issues its
certificates, and the certificate and key of tls file are written to CADDY_SITES_DIR/<name>. The certificates and
keys of the issuing CAs of both are copied to CADDY_PKI_DIR. While the sites cannot be read (ctx's sites is None),
the Caddyfile is left as it is.
Caddy is installed by CADDY_INSTALL_UNIT, outside setup, when it is missing or when
https.caddy_source is upstream and Caddy's repository is not configured; the unit restarts
setup, whose next run serves with it.
"""
if not vpnrouter.https_ready(ctx):
sites = ctx['sites']
if not vpnrouter.https_ready(ctx) and not sites:
if sites is None:
return
vpnrouter.remove(CADDYFILE)
if unit_is_active(CADDY_UNIT):
run(['systemctl', 'disable', '--now', CADDY_UNIT], check=False)
@@ -669,21 +712,33 @@ def manage_caddy(cfg, ctx):
if source == 'ubuntu' and vpnrouter.CADDY_APT_SOURCE.is_file():
warn("https.caddy_source ubuntu is ignored: Caddy's repository is configured, and Caddy stays upstream's")
if sites is None:
warn('the web sites cannot be read; the Caddyfile is left as it is')
if not unit_is_active(CADDY_UNIT) and CADDYFILE.exists():
run(['systemctl', 'enable', '--now', CADDY_UNIT], check=False)
return
try:
group = grp.getgrnam(CADDY_GROUP).gr_gid
except KeyError:
fail(f'the group {CADDY_GROUP} does not exist; is the caddy package installed?')
https = vpnrouter.https_ready(ctx)
wanted = list(dict.fromkeys([*(ctx['pki_acme'] if https else []),
*(site['ca'] for site in sites if site['tls'] == 'ca')]))
try:
credentials = {name: vpnrouter_pki.authority_files(name) for name in ctx['pki_acme']}
credentials = {name: vpnrouter_pki.authority_files(name) for name in wanted}
except ldap.LDAPError as exc:
warn(f'the ACME servers are left out: the PKI cannot be read: {exc}')
warn(f'the ACME servers and the sites of issuing CAs are left out: the PKI cannot be read: {exc}')
credentials = {}
acme = [name for name, files in credentials.items() if files is not None]
authorities = [name for name, files in credentials.items() if files is not None]
for site in sites:
if site['tls'] == 'ca' and site['ca'] not in authorities:
warn(f'site {site["name"]} is left out: issuing CA {site["ca"]} has no certificate')
sites = [site for site in sites if site['tls'] != 'ca' or site['ca'] in authorities]
copied = False
for folder in (CADDY_PKI_DIR.iterdir() if CADDY_PKI_DIR.is_dir() else []):
if folder.name not in acme:
if folder.name not in authorities:
shutil.rmtree(folder)
for name in acme:
for name in authorities:
folder = CADDY_PKI_DIR / name
for path in (CADDY_PKI_DIR, folder):
path.mkdir(mode=0o750, exist_ok=True)
@@ -691,12 +746,33 @@ def manage_caddy(cfg, ctx):
for file, data in zip(('cert.pem', 'key.pem'), credentials[name]):
copied |= vpnrouter_pki.write(folder / file, data, 0o640)
os.chown(folder / file, 0, group)
authorities = vpnrouter.issuing_cas(cfg)
caddy_ctx = dict(ctx, manager_port=manager_port() if MANAGER.exists() else 0,
files = {site['name']: site for site in sites if site['tls'] == 'file'}
for folder in (CADDY_SITES_DIR.iterdir() if CADDY_SITES_DIR.is_dir() else []):
if folder.name not in files:
shutil.rmtree(folder)
for name, site in files.items():
folder = CADDY_SITES_DIR / name
for path in (CADDY_SITES_DIR, folder):
path.mkdir(mode=0o750, exist_ok=True)
os.chown(path, 0, group)
for file, data in (('cert.pem', site['certificate']), ('key.pem', site['private_key'])):
copied |= vpnrouter_pki.write(folder / file, data.strip().encode() + b'\n', 0o640)
os.chown(folder / file, 0, group)
vpnrouter_sites.ensure_folders(site['name'] for site in sites)
names = vpnrouter.issuing_cas(cfg)
caddy_ctx = dict(ctx, https=https, manager_port=vpnrouter.manager_port() if vpnrouter.MANAGER.exists() else 0,
pki_publish_dir=str(vpnrouter_pki.PUBLISH_DIR),
acme_cas=[{'id': name, 'name': authorities[name]['name'].strip(),
'cert': str(CADDY_PKI_DIR / name / 'cert.pem'),
'key': str(CADDY_PKI_DIR / name / 'key.pem')} for name in acme])
pki_cas=[{'id': name, 'name': names[name]['name'].strip(),
'cert': str(CADDY_PKI_DIR / name / 'cert.pem'),
'key': str(CADDY_PKI_DIR / name / 'key.pem')} for name in authorities],
acme_cas=[name for name in authorities if https and name in ctx['pki_acme']],
webdav=vpnrouter.webdav_ready(ctx), webdav_loopback_port=vpnrouter.WEBDAV_LOOPBACK_PORT,
sites=[{'name': site['name'], 'tls': site['tls'], 'ca': site['ca'], 'spa': site['spa'],
'index': site['index_file'], 'root': str(vpnrouter_sites.SITES_DIR / site['name']),
'addresses': [caddy_address(host, site['port'], site['tls']) for host in site['hosts']]
or [caddy_address('', site['port'], site['tls'])],
'cert': str(CADDY_SITES_DIR / site['name'] / 'cert.pem'),
'key': str(CADDY_SITES_DIR / site['name'] / 'key.pem')} for site in sites])
changed = vpnrouter.render(vpnrouter._jinja_env(), caddy_ctx, 'Caddyfile.j2', CADDYFILE, 0o640)
os.chown(CADDYFILE, 0, group)
if not unit_is_active(CADDY_UNIT):
@@ -705,6 +781,71 @@ def manage_caddy(cfg, ctx):
run(['systemctl', 'reload', CADDY_UNIT], check=False)
def install_packages(feature):
"""Start the installation of the packages of a feature by INSTALL_UNIT, which runs setup again, and say so."""
unit = INSTALL_UNIT.format(feature)
run(['systemctl', 'start', '--no-block', unit], check=False)
warn(f'the packages of {feature} are installed by {unit}, which runs setup again')
def manage_webdav(ctx):
"""Run rclone's WebDAV server of the sites' folders while WebDAV is ready, and stop it when not; rclone is installed
when it is missing."""
if not vpnrouter.webdav_ready(ctx):
if unit_is_active(WEBDAV_UNIT):
run(['systemctl', 'disable', '--now', WEBDAV_UNIT], check=False)
return
if shutil.which('rclone') is None:
install_packages('webdav')
return
vpnrouter_sites.ensure_folders([])
run(['systemctl', 'enable', '--now', WEBDAV_UNIT], check=False)
def manage_dns(cfg, ctx):
"""Run PowerDNS, the name server of dns.zone, on the addresses of dns.listen while DNS is enabled, and stop it when
not; its packages are installed when they are missing.
The settings, vpnrouter_dns.PDNS_CONF, are written before the packages are installed, so the server their
installation starts answers on those addresses only, beside systemd-resolved's stub on 127.0.0.53. The zone gets
the host names of the enabled sites, and keeps those it has while the sites cannot be read.
"""
if not vpnrouter.dns_ready(ctx):
if unit_is_active(vpnrouter_dns.UNIT):
run(['systemctl', 'disable', '--now', vpnrouter_dns.UNIT], check=False)
vpnrouter.remove(vpnrouter_dns.PDNS_CONF)
if vpnrouter_dns.DATABASE_SECRET.is_file():
try:
vpnrouter_dns.apply(cfg, [])
except vpnrouter_dns.DatabaseError as exc:
warn(str(exc))
return
addresses = {'internal': [ctx['int_addr']], 'external': ctx['local_addr_list'],
'both': [ctx['int_addr'], *ctx['local_addr_list']]}[ctx['dns_listen']]
addresses = [address for address in addresses if address]
if not addresses:
warn(f'the name server does not run: the interfaces of dns.listen {ctx["dns_listen"]} have no address')
return
try:
vpnrouter_auth.provision_database(vpnrouter_dns.DATABASE, vpnrouter_dns.DATABASE_SECRET)
vpnrouter_dns.PDNS_CONF.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
changed = write_if_changed(vpnrouter_dns.PDNS_CONF, vpnrouter_dns.config_text(addresses), 0o640)
if not vpnrouter_dns.PDNS_SERVER.exists() or not vpnrouter_dns.PDNS_SCHEMA.exists():
install_packages('dns')
return
os.chown(vpnrouter_dns.PDNS_CONF, 0, grp.getgrnam(vpnrouter_dns.GROUP).gr_gid)
vpnrouter_dns.provision()
if ctx['sites'] is not None:
vpnrouter_dns.apply(cfg, [host for site in ctx['sites'] for host in site['hosts']])
except (OSError, KeyError, vpnrouter_dns.DatabaseError) as exc:
warn(f'the name server cannot be set up: {exc}')
return
if not unit_is_active(vpnrouter_dns.UNIT):
run(['systemctl', 'enable', '--now', vpnrouter_dns.UNIT], check=False)
elif changed:
run(['systemctl', 'restart', vpnrouter_dns.UNIT], check=False)
def manage_metrics(ctx):
"""Run the metrics service while metrics are enabled, and stop it when not.
@@ -863,6 +1004,16 @@ def main():
ctx, more = vpnrouter.configure(cfg, public_ip=address)
changed |= more
# The enabled sites while sites are on, None while they cannot be read.
ctx['sites'] = []
if cfg['sites']['enabled']:
progress('running', 'web sites')
try:
ctx['sites'] = vpnrouter_sites.served()
except vpnrouter_sites.DatabaseError as exc:
warn(str(exc))
ctx['sites'] = None
progress('running', 'WireGuard keys')
setup_wireguard(ctx)
@@ -900,6 +1051,10 @@ def main():
manage_openvpn(ctx, changed)
progress('running', 'HTTPS proxy')
manage_caddy(cfg, ctx)
progress('running', 'WebDAV')
manage_webdav(ctx)
progress('running', 'name server')
manage_dns(cfg, ctx)
progress('running', 'PKI timer')
run(['systemctl', 'enable', '--now', PKI_TIMER], check=False)