440 lines
20 KiB
Python
440 lines
20 KiB
Python
"""The static web sites Caddy serves, for the web service, vpn-router-site, vpn-router-setup and webdav-auth.
|
|
|
|
A site is a row of the sites table in the vpn-router-sites database, which the web service creates on its first run
|
|
(provision) with its role and the password in DATABASE_SECRET; SITES_SQL defines the table and its procedures, and the
|
|
first connection of a process loads it. A site has a name, which also names its folder SITES_DIR/<name>, the host names
|
|
Caddy answers on its port (hosts, the names TLS clients send as SNI; none answers any name), the interfaces whose
|
|
firewall opens the port (listen), and TLS: none (plain HTTP), acme (a certificate of Let's Encrypt), ca (certificates
|
|
Caddy issues with one of pki.issuing_cas) or file (a certificate and key given with the site). Caddy serves the files of
|
|
the folder, index_file for a folder, and with spa index_file of the site for any path without a file. editors names
|
|
the group whose members change the site's files besides the admins, through the web service and WebDAV.
|
|
|
|
The sites are served while sites.enabled is true; vpn-router-setup writes Caddy's configuration and opens the ports. A
|
|
site is changed only through this module, which checks it (check), and its callers then restart vpn-router-setup.
|
|
|
|
The files of a site are read and written by path relative to its folder; resolve() refuses a path that leaves it.
|
|
receive() writes a file from a stream, extract() a zip archive into a folder; both stop at sites.upload_max_mb.
|
|
|
|
Every failure of the database raises DatabaseError; a request that cannot be done raises Failure, whose status says
|
|
why: 2 invalid, 3 no such site, 4 the site exists, 1 anything else.
|
|
"""
|
|
|
|
import contextlib
|
|
import datetime
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import tempfile
|
|
import zipfile
|
|
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from cryptography import x509
|
|
from cryptography.hazmat.primitives import serialization
|
|
|
|
import vpnrouter
|
|
import vpnrouter_auth
|
|
import vpnrouter_pki
|
|
import vpnrouter_users
|
|
|
|
DATABASE = 'vpn-router-sites'
|
|
DATABASE_SECRET = pathlib.Path('/etc/vpn-router/sites-database.secret')
|
|
SITES_SQL = pathlib.Path('/usr/share/vpn-router/sql/sites.sql')
|
|
|
|
#: The folders of the sites, SITES_DIR/<name>.
|
|
SITES_DIR = pathlib.Path('/var/lib/vpn-router/sites')
|
|
|
|
#: The name of a site, which also names its folder.
|
|
NAME = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
|
|
|
#: A file name of index_file.
|
|
INDEX = re.compile(r'^[A-Za-z0-9_-][A-Za-z0-9._-]{0,63}$')
|
|
|
|
LISTEN = vpnrouter.LISTEN_CHOICES
|
|
TLS = ('none', 'acme', 'ca', 'file')
|
|
|
|
#: The most host names of one site.
|
|
HOSTS_MAX = 16
|
|
|
|
#: The most members a zip archive extract() takes.
|
|
ARCHIVE_MEMBERS_MAX = 10000
|
|
|
|
#: The bytes copied at once from a stream.
|
|
CHUNK = 65536
|
|
|
|
#: The settings of a site and their defaults; the type of a default is the required type.
|
|
SETTINGS = {'enabled': True, 'hosts': [], 'port': 443, 'listen': 'external', 'tls': 'acme', 'ca': '', 'certificate': '',
|
|
'private_key': '', 'index_file': 'index.html', 'spa': False, 'editors': ''}
|
|
|
|
#: Whether this process has loaded SITES_SQL.
|
|
_loaded = False
|
|
|
|
|
|
class DatabaseError(Exception):
|
|
"""The database of the sites failed."""
|
|
|
|
|
|
class Failure(Exception):
|
|
"""A request that cannot be done; status is 2 invalid, 3 no such site, 4 the site exists, 1 other."""
|
|
|
|
def __init__(self, message, status=1):
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
def provision():
|
|
"""Create the role, its password file and the database of the sites when they are missing."""
|
|
vpnrouter_auth.provision_database(DATABASE, DATABASE_SECRET)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def database():
|
|
"""A connection to the database of the sites, with its table and procedures, closed at the end; DatabaseError on any
|
|
failure, also before the web service has created the database."""
|
|
global _loaded
|
|
try:
|
|
schema = None if _loaded else SITES_SQL.read_text(encoding='utf-8')
|
|
target = vpnrouter_auth.local_connection(DATABASE, DATABASE_SECRET)
|
|
except OSError as exc:
|
|
raise DatabaseError(f'the database of the sites is not ready: {exc}') from exc
|
|
try:
|
|
with psycopg.connect(target, autocommit=True, row_factory=dict_row) as connection:
|
|
if schema is not None:
|
|
connection.execute(schema)
|
|
_loaded = True
|
|
yield connection
|
|
except psycopg.Error as exc:
|
|
raise DatabaseError(f'the database of the sites failed: {exc}') from exc
|
|
|
|
|
|
def shown(row):
|
|
"""A site as the answers show it: its settings without the private key, with private_key_set and its folder."""
|
|
site = {key: value for key, value in row.items() if key != 'private_key'}
|
|
site['private_key_set'] = bool(row.get('private_key'))
|
|
site['modified'] = row['modified'].isoformat(timespec='seconds')
|
|
site['directory'] = str(SITES_DIR / row['name'])
|
|
return site
|
|
|
|
|
|
def listing():
|
|
"""Every site, as shown()."""
|
|
with database() as connection:
|
|
return [shown(row) for row in connection.execute('SELECT * FROM sites_list()').fetchall()]
|
|
|
|
|
|
def served():
|
|
"""The enabled sites with their private keys."""
|
|
with database() as connection:
|
|
return connection.execute('SELECT * FROM sites_served()').fetchall()
|
|
|
|
|
|
def get(connection, name):
|
|
"""The site with its private key; Failure with status 3 when there is none."""
|
|
if not isinstance(name, str) or not NAME.match(name):
|
|
raise Failure(f'no site {name}', 3)
|
|
row = connection.execute('SELECT * FROM site_get(%s)', (name,)).fetchone()
|
|
if row is None:
|
|
raise Failure(f'no site {name}', 3)
|
|
return row
|
|
|
|
|
|
def editable(groups):
|
|
"""The names of the sites whose files the members of the groups may change."""
|
|
with database() as connection:
|
|
return [row['name'] for row in connection.execute('SELECT * FROM sites_editable(%s)', (list(groups),))]
|
|
|
|
|
|
def sites_of_group(group):
|
|
"""The names of the sites whose editors are the group; [] before the web service has created the database."""
|
|
if not DATABASE_SECRET.is_file():
|
|
return []
|
|
with database() as connection:
|
|
return [row['name'] for row in connection.execute('SELECT * FROM sites_of_group(%s)', (group,))]
|
|
|
|
|
|
def rename_group(old, new):
|
|
"""Give the sites whose editors are the group old its new name."""
|
|
if DATABASE_SECRET.is_file():
|
|
with database() as connection:
|
|
connection.execute('CALL sites_rename_group(%s, %s)', (old, new))
|
|
|
|
|
|
def taken_ports(cfg):
|
|
"""The TCP ports of the router's own services, port -> what has it, which no site may take."""
|
|
taken = {22: 'SSH', cfg['web']['port']: 'web.port', cfg['https']['web_port']: 'https.web_port',
|
|
cfg['https']['manager_port']: 'https.manager_port', cfg['metrics']['port']: 'metrics.port',
|
|
cfg['metrics']['https_port']: 'metrics.https_port', vpnrouter.manager_port(): 'the Router Manager',
|
|
cfg['sites']['webdav_port']: 'sites.webdav_port', vpnrouter.WEBDAV_LOOPBACK_PORT: 'rclone\'s WebDAV',
|
|
53: 'DNS'}
|
|
for port in vpnrouter.BOOTSTRAP_PORTS:
|
|
taken[port] = 'the bootstrap services'
|
|
if cfg['openvpn']['protocol'] == 'tcp':
|
|
taken[cfg['openvpn']['port']] = 'openvpn.port'
|
|
for item in cfg['docker']['exposed_ports']:
|
|
first, _, last = item.strip().partition(':')
|
|
if first.isdigit() and (not last or last.isdigit()):
|
|
for port in range(int(first), int(last or first) + 1):
|
|
taken[port] = 'docker.exposed_ports'
|
|
return taken
|
|
|
|
|
|
def check_certificate(certificate, private_key, hosts):
|
|
"""Refuse a PEM certificate chain whose first certificate is not the key's or names none of the hosts."""
|
|
try:
|
|
chain = x509.load_pem_x509_certificates(certificate.encode())
|
|
except ValueError:
|
|
raise Failure('certificate must be PEM certificates, the site\'s first', 2)
|
|
try:
|
|
key = serialization.load_pem_private_key(private_key.encode(), password=None)
|
|
except (ValueError, TypeError):
|
|
raise Failure('private_key must be an unencrypted PEM private key', 2)
|
|
if not vpnrouter_pki.same_key(chain[0].public_key(), key.public_key()):
|
|
raise Failure('private_key is not the key of the certificate', 2)
|
|
names = {value.partition(':')[2] for value in vpnrouter_pki.san_values(chain[0]) if not value.startswith('email:')}
|
|
missing = [host for host in hosts if host not in names and f'*.{host.partition(".")[2]}' not in names]
|
|
if missing:
|
|
raise Failure(f'the certificate does not name {", ".join(missing)}', 2)
|
|
|
|
|
|
def check(cfg, sites, site):
|
|
"""Refuse a site, a dict of SETTINGS with its name, that is invalid or collides with the other sites (sites, rows of
|
|
the table) or with the router's own ports."""
|
|
name = site['name']
|
|
if not isinstance(name, str) or not NAME.match(name):
|
|
raise Failure('a site name starts with a lowercase letter and has at most 31 lowercase letters, digits and '
|
|
'hyphens', 2)
|
|
for option, default in SETTINGS.items():
|
|
value = site[option]
|
|
if type(value) is not type(default) or (isinstance(value, list) and not all(isinstance(item, str) for item in value)):
|
|
raise Failure(f'{option} must be {vpnrouter.TYPE_NAMES[type(default)]}', 2)
|
|
hosts = site['hosts']
|
|
if len(hosts) > HOSTS_MAX or len(set(hosts)) != len(hosts):
|
|
raise Failure(f'hosts must be at most {HOSTS_MAX} different host names', 2)
|
|
for host in hosts:
|
|
if not (vpnrouter_pki.DNS_NAME.match(host) or vpnrouter_pki.is_address(host)):
|
|
raise Failure(f'hosts: {host!r} is not a host name in lowercase or an IP address', 2)
|
|
if host == cfg['wan']['local_fqdn'].strip():
|
|
raise Failure(f'hosts: {host} is wan.local_fqdn, the name of the router\'s own services', 2)
|
|
port = site['port']
|
|
if not 1 <= port <= 65535:
|
|
raise Failure(f'port: {port} is not a port', 2)
|
|
taken = taken_ports(cfg)
|
|
if port in taken:
|
|
raise Failure(f'port: {port} is the port of {taken[port]}', 2)
|
|
if site['listen'] not in LISTEN:
|
|
raise Failure(f'listen: {site["listen"]!r} is not one of {", ".join(LISTEN)}', 2)
|
|
tls = site['tls']
|
|
if tls not in TLS:
|
|
raise Failure(f'tls: {tls!r} is not one of {", ".join(TLS)}', 2)
|
|
if tls != 'none' and not hosts:
|
|
raise Failure(f'tls {tls} needs hosts, the names of the certificate', 2)
|
|
if tls == 'acme' and any(host.startswith('*.') or vpnrouter_pki.is_address(host) for host in hosts):
|
|
raise Failure('tls acme takes host names only, without wildcards', 2)
|
|
if tls == 'ca':
|
|
authorities = {key: value for key, value in vpnrouter.issuing_cas(cfg).items() if not value['builtin']}
|
|
if site['ca'] not in authorities:
|
|
raise Failure(f'ca: {site["ca"]!r} is not one of pki.issuing_cas', 2)
|
|
if tls == 'file':
|
|
check_certificate(site['certificate'], site['private_key'], hosts)
|
|
if not INDEX.match(site['index_file']):
|
|
raise Failure('index_file must be a file name', 2)
|
|
if site['editors']:
|
|
with vpnrouter_users.session() as connection:
|
|
try:
|
|
vpnrouter_users.group_entry(connection, site['editors'])
|
|
except vpnrouter_users.Failure:
|
|
raise Failure(f'editors: group {site["editors"]} does not exist', 2)
|
|
if not site['enabled']:
|
|
return
|
|
for other in sites:
|
|
if other['name'] == name or not other['enabled'] or other['port'] != port:
|
|
continue
|
|
prefix = f'port {port} is also the port of site {other["name"]}'
|
|
if (other['tls'] == 'none') != (tls == 'none'):
|
|
raise Failure(f'{prefix}, which is {"plain HTTP" if other["tls"] == "none" else "HTTPS"}', 2)
|
|
if other['listen'] != site['listen']:
|
|
raise Failure(f'{prefix}, which listens on {other["listen"]}', 2)
|
|
if not hosts or not other['hosts']:
|
|
raise Failure(f'{prefix}; sites sharing a port need host names', 2)
|
|
common = sorted(set(hosts) & set(other['hosts']))
|
|
if common:
|
|
raise Failure(f'{prefix}, which has {", ".join(common)} too', 2)
|
|
|
|
|
|
def save(cfg, name, values, create):
|
|
"""Add the site with values, a dict of some of SETTINGS (create), or change those of an existing one; the site as
|
|
shown(). An absent or empty private_key keeps the current one."""
|
|
unknown = sorted(set(values) - set(SETTINGS))
|
|
if unknown:
|
|
raise Failure(f'unknown settings: {", ".join(unknown)}', 2)
|
|
with database() as connection:
|
|
rows = connection.execute('SELECT * FROM sites_list()').fetchall()
|
|
current = next((row for row in rows if row['name'] == name), None)
|
|
if create and current is not None:
|
|
raise Failure(f'site {name} exists', 4)
|
|
if not create:
|
|
current = get(connection, name)
|
|
site = {**SETTINGS, **({key: current[key] for key in SETTINGS} if current else {}), 'name': name}
|
|
site.update({key: value for key, value in values.items() if key != 'private_key' or value})
|
|
if isinstance(site['hosts'], list):
|
|
site['hosts'] = [host.strip().lower() for host in site['hosts'] if isinstance(host, str) and host.strip()]
|
|
if site['tls'] != 'file':
|
|
site['certificate'] = site['private_key'] = ''
|
|
if site['tls'] != 'ca':
|
|
site['ca'] = ''
|
|
check(cfg, rows, site)
|
|
connection.execute('CALL site_put(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',
|
|
tuple(site[key] for key in ('name', *SETTINGS)))
|
|
ensure_folders([name])
|
|
return shown(get(connection, name))
|
|
|
|
|
|
def remove(name, content):
|
|
"""Remove the site, and with content its folder."""
|
|
with database() as connection:
|
|
get(connection, name)
|
|
connection.execute('CALL site_remove(%s)', (name,))
|
|
if content:
|
|
shutil.rmtree(SITES_DIR / name, ignore_errors=True)
|
|
return {'name': name, 'removed': True}
|
|
|
|
|
|
def ensure_folders(names):
|
|
"""Create the folders of the sites that miss one, readable by all."""
|
|
SITES_DIR.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
for name in names:
|
|
(SITES_DIR / name).mkdir(mode=0o755, exist_ok=True)
|
|
|
|
|
|
def resolve(name, relative):
|
|
"""The path of the site's file or folder relative to its folder; Failure with status 2 for a path that leaves it,
|
|
status 3 when the site has no folder."""
|
|
if not isinstance(name, str) or not NAME.match(name):
|
|
raise Failure(f'no site {name}', 3)
|
|
base = SITES_DIR / name
|
|
if not base.is_dir():
|
|
raise Failure(f'site {name} has no folder', 3)
|
|
if '\0' in relative or any(part in ('.', '..') for part in relative.split('/')):
|
|
raise Failure(f'{relative!r} is not a path in the site\'s folder', 2)
|
|
target = pathlib.Path(os.path.realpath(base / relative.strip('/')))
|
|
if target != base and base not in target.parents:
|
|
raise Failure(f'{relative!r} is not a path in the site\'s folder', 2)
|
|
return target
|
|
|
|
|
|
def relative_of(name, path):
|
|
return str(path.relative_to(SITES_DIR / name))
|
|
|
|
|
|
def entries(name, folder):
|
|
"""The entries of a folder of the site: [{name, folder, size, modified}], folders first, then by name."""
|
|
found = []
|
|
for item in os.scandir(folder):
|
|
try:
|
|
info = item.stat(follow_symlinks=False)
|
|
except OSError:
|
|
continue
|
|
found.append({'name': item.name, 'folder': stat.S_ISDIR(info.st_mode),
|
|
'size': info.st_size if stat.S_ISREG(info.st_mode) else 0,
|
|
'modified': datetime.datetime.fromtimestamp(info.st_mtime, datetime.timezone.utc)
|
|
.isoformat(timespec='seconds')})
|
|
return {'path': relative_of(name, folder) if folder != SITES_DIR / name else '',
|
|
'entries': sorted(found, key=lambda entry: (not entry['folder'], entry['name']))}
|
|
|
|
|
|
def receive(stream, length, target, limit):
|
|
"""Write length bytes of the stream to the file target, replacing it in one step; Failure with status 2 over limit
|
|
bytes. Its folders are created."""
|
|
if length > limit:
|
|
raise Failure(f'the file has {length} bytes, more than sites.upload_max_mb allows', 2)
|
|
if target.is_dir():
|
|
raise Failure(f'{target.name} is a folder', 2)
|
|
target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
descriptor, name = tempfile.mkstemp(dir=target.parent, prefix=f'.{target.name}.')
|
|
try:
|
|
with os.fdopen(descriptor, 'wb') as handle:
|
|
left = length
|
|
while left:
|
|
data = stream.read(min(CHUNK, left))
|
|
if not data:
|
|
raise Failure('the client sent less than its Content-Length', 2)
|
|
handle.write(data)
|
|
left -= len(data)
|
|
os.chmod(name, 0o644)
|
|
os.replace(name, target)
|
|
except BaseException:
|
|
os.unlink(name)
|
|
raise
|
|
|
|
|
|
def extract(stream, length, folder, limit):
|
|
"""Extract the zip archive of length bytes of the stream into the folder, created when missing; the number of files.
|
|
|
|
A member with an absolute path, a .. or a link, more than ARCHIVE_MEMBERS_MAX members or more than limit bytes
|
|
unpacked refuse the whole archive, before anything is written.
|
|
"""
|
|
if length > limit:
|
|
raise Failure(f'the archive has {length} bytes, more than sites.upload_max_mb allows', 2)
|
|
folder.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
with tempfile.TemporaryFile(dir=folder) as spool:
|
|
left = length
|
|
while left:
|
|
data = stream.read(min(CHUNK, left))
|
|
if not data:
|
|
raise Failure('the client sent less than its Content-Length', 2)
|
|
spool.write(data)
|
|
left -= len(data)
|
|
spool.seek(0)
|
|
try:
|
|
archive = zipfile.ZipFile(spool)
|
|
except zipfile.BadZipFile:
|
|
raise Failure('the file is not a zip archive', 2)
|
|
with archive:
|
|
members = archive.infolist()
|
|
if len(members) > ARCHIVE_MEMBERS_MAX:
|
|
raise Failure(f'the archive has more than {ARCHIVE_MEMBERS_MAX} members', 2)
|
|
if sum(member.file_size for member in members) > limit:
|
|
raise Failure('the archive unpacks to more than sites.upload_max_mb allows', 2)
|
|
for member in members:
|
|
parts = member.filename.replace('\\', '/').split('/')
|
|
if member.filename.startswith('/') or '..' in parts or '\0' in member.filename or \
|
|
stat.S_ISLNK(member.external_attr >> 16):
|
|
raise Failure(f'the archive member {member.filename!r} is a link or leaves the folder', 2)
|
|
count = 0
|
|
for member in members:
|
|
target = folder / member.filename.replace('\\', '/').strip('/')
|
|
if member.is_dir():
|
|
target.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
continue
|
|
target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
with archive.open(member) as source:
|
|
receive(source, member.file_size, target, limit)
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def make_folder(target):
|
|
target.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
|
|
|
|
def rename(name, source, relative):
|
|
"""Move the file or folder source of the site to the path relative to its folder; Failure when that exists."""
|
|
target = resolve(name, relative)
|
|
if target.exists():
|
|
raise Failure(f'{relative} exists', 4)
|
|
target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
os.replace(source, target)
|
|
return target
|
|
|
|
|
|
def delete(name, target):
|
|
"""Remove the file or folder of the site, but not its folder."""
|
|
if target == SITES_DIR / name:
|
|
raise Failure('the site\'s folder itself cannot be removed', 2)
|
|
if target.is_dir() and not target.is_symlink():
|
|
shutil.rmtree(target)
|
|
else:
|
|
target.unlink()
|