361 lines
18 KiB
Python
Executable File
361 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run the Linux Manager test scenarios on an LXD remote: run.py core|nat|s2s... or run.py all.
|
|
|
|
The remote and its storage pool are required, as --remote and --pool or LXD_REMOTE and
|
|
LXD_POOL. The instances live in a project of their own, which must not exist and is removed
|
|
with its networks after each scenario unless --keep is given. The package is the newest
|
|
out/linux-manager_*_all.deb. Each check prints PASS, FAIL or SKIP; the run exits 1 when a
|
|
check failed.
|
|
"""
|
|
|
|
import argparse
|
|
import importlib
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
ROOT = HERE.parent.parent
|
|
TMP = ROOT / 'tmp' / 'remote-test'
|
|
SCENARIOS = sorted(path.stem for path in (HERE / 'scenarios').glob('*.py') if path.stem != '__init__')
|
|
|
|
|
|
class Lab:
|
|
"""The instances and networks of a run on one LXD remote and project, and the results of its checks.
|
|
|
|
A NIC is a (network, mac, address/prefix[, gateway]) tuple; an address of None is taken from
|
|
DHCP.
|
|
"""
|
|
|
|
def __init__(self, remote, project, pool, image, keep, deb, admin_password, psk):
|
|
self.remote = remote
|
|
self.project = project
|
|
self.pool = pool
|
|
self.image = image
|
|
self.keep = keep
|
|
self.deb = deb
|
|
self.admin_password = admin_password
|
|
self.psk = psk
|
|
self.passed = self.failed = self.skipped = 0
|
|
self.networks = []
|
|
self.macs = {}
|
|
self.addresses = {}
|
|
|
|
def lxc(self, *args, check=True):
|
|
"""Run lxc with the arguments and the project; raise CalledProcessError when it fails, unless check is false."""
|
|
return subprocess.run(['lxc', *args, '--project', self.project], capture_output=True, text=True, check=check)
|
|
|
|
def on(self, name, *command, input=None):
|
|
"""Run the command in the instance with the input, and return its CompletedProcess."""
|
|
return subprocess.run(
|
|
['lxc', 'exec', f'{self.remote}:{name}', '--project', self.project, '-T', '--', *command],
|
|
input=input, capture_output=True, text=True)
|
|
|
|
def push(self, source, name, path, mode='0644'):
|
|
"""Copy the local file into the instance; True when it was copied."""
|
|
result = self.lxc('file', 'push', str(source), f'{self.remote}:{name}{path}', '--mode', mode, check=False)
|
|
return result.returncode == 0
|
|
|
|
def push_text(self, text, name, path, mode='0644'):
|
|
"""Write the text to a local file readable by its owner only, copy it into the instance and remove it."""
|
|
local = TMP / f'{name}{path.replace("/", "_")}'
|
|
local.touch(mode=0o600)
|
|
local.chmod(0o600)
|
|
local.write_text(text, encoding='utf-8')
|
|
try:
|
|
return self.push(local, name, path, mode)
|
|
finally:
|
|
local.unlink()
|
|
|
|
def json_of(self, name, *command):
|
|
"""The JSON output of the command in the instance, or None when the command fails or its output is no JSON."""
|
|
result = self.on(name, *command)
|
|
try:
|
|
return json.loads(result.stdout) if result.returncode == 0 else None
|
|
except ValueError:
|
|
return None
|
|
|
|
def result(self, description, ok):
|
|
"""Count and print the check as PASS when ok, else FAIL."""
|
|
if ok:
|
|
self.passed += 1
|
|
print(f'PASS {description}')
|
|
else:
|
|
self.failed += 1
|
|
print(f'FAIL {description}')
|
|
|
|
def skip(self, description):
|
|
"""Count and print the check as SKIP."""
|
|
self.skipped += 1
|
|
print(f'SKIP {description}')
|
|
|
|
def check(self, description, name, *command):
|
|
"""Pass when the command succeeds in the instance."""
|
|
self.result(description, self.on(name, *command).returncode == 0)
|
|
|
|
def expect(self, description, pattern, name, *command):
|
|
"""Pass when a line of the command's output, standard error included, matches the regular expression."""
|
|
result = self.on(name, *command)
|
|
self.result(description, re.search(pattern, result.stdout + result.stderr, re.M) is not None)
|
|
|
|
def refuse(self, description, pattern, name, *command):
|
|
"""Pass when no line of the command's output, standard error included, matches the regular expression."""
|
|
result = self.on(name, *command)
|
|
self.result(description, re.search(pattern, result.stdout + result.stderr, re.M) is None)
|
|
|
|
def network(self, name, address):
|
|
"""Create a bridge with outbound NAT at the gateway address GATEWAY/PREFIX, or without a host address when None.
|
|
|
|
A bridge without a host address is reachable only through the instances on it.
|
|
"""
|
|
if address is None:
|
|
self.lxc('network', 'create', f'{self.remote}:{name}', 'ipv4.address=none', 'ipv6.address=none')
|
|
else:
|
|
self.lxc('network', 'create', f'{self.remote}:{name}', f'ipv4.address={address}', 'ipv4.nat=true',
|
|
'ipv4.dhcp=false', 'ipv6.address=none')
|
|
self.networks.append(name)
|
|
|
|
def instance(self, name, kind, user_data, *nics):
|
|
"""Create and start a vm or container with the NICs, whose static addresses are set through cloud-init.
|
|
|
|
A container's NICs are eth0 and on, named by LXD, as a veth has no permanent MAC address for
|
|
netplan to match; a VM's NICs keep their kernel names, which the initramfs sets before
|
|
netplan could rename them, and are matched by MAC address (nic_of).
|
|
"""
|
|
self.macs[name] = [nic[1] for nic in nics]
|
|
self.addresses[name] = nics[0][2]
|
|
netcfg = 'version: 2\nethernets:\n'
|
|
for index, (_, mac, address, *gateway) in enumerate(nics):
|
|
if kind == 'vm':
|
|
netcfg += f' nic{index}:\n match:\n macaddress: "{mac}"\n'
|
|
else:
|
|
netcfg += f' eth{index}:\n'
|
|
netcfg += f' addresses: [{address}]\n' if address else ' dhcp4: true\n'
|
|
if gateway:
|
|
netcfg += (f' routes:\n - to: default\n via: {gateway[0]}\n'
|
|
f' nameservers:\n addresses: [{gateway[0]}]\n')
|
|
limits = ['-c', 'limits.cpu=2', '-c', 'limits.memory=4GiB'] if kind == 'vm' else \
|
|
['-c', 'limits.cpu=1', '-c', 'limits.memory=1GiB']
|
|
self.lxc('init', self.image, f'{self.remote}:{name}', *(['--vm'] if kind == 'vm' else []), *limits,
|
|
'-c', 'cloud-init.user-data=' + (HERE / 'cloud-init' / user_data).read_text(encoding='utf-8'),
|
|
'-c', 'cloud-init.network-config=' + netcfg)
|
|
if kind != 'vm':
|
|
self.lxc('config', 'device', 'add', f'{self.remote}:{name}', 'tun', 'unix-char', 'path=/dev/net/tun')
|
|
for index, (network, mac, *_) in enumerate(nics):
|
|
self.lxc('config', 'device', 'add', f'{self.remote}:{name}', f'eth{index}', 'nic', f'network={network}',
|
|
f'hwaddr={mac}', *([] if kind == 'vm' else [f'name=eth{index}']))
|
|
self.lxc('start', f'{self.remote}:{name}')
|
|
|
|
def ready(self, *names):
|
|
"""Wait for each instance's agent and cloud-init, then for its first address; False when one is not ready.
|
|
|
|
cloud-init's status 2, done with recoverable errors, counts as booted. An instance whose first
|
|
NIC takes its address from DHCP waits for any IPv4 address outside lo.
|
|
"""
|
|
for name in names:
|
|
for _ in range(120):
|
|
if self.on(name, 'cloud-init', 'status', '--wait').returncode in (0, 2):
|
|
break
|
|
time.sleep(5)
|
|
else:
|
|
print(f'{name} did not boot', file=sys.stderr)
|
|
return False
|
|
address = self.addresses[name]
|
|
pattern = f' inet {re.escape(address.split("/")[0])}/' if address else r'^\d+: (?!lo\b)\S+ +inet '
|
|
for _ in range(24):
|
|
if re.search(pattern, self.on(name, 'ip', '-4', '-o', 'addr', 'show').stdout, re.M):
|
|
break
|
|
time.sleep(5)
|
|
else:
|
|
print(f'{name} has no address {address or "from DHCP"}', file=sys.stderr)
|
|
return False
|
|
return True
|
|
|
|
def nic_of(self, name, index):
|
|
"""The name of the instance's NIC index, counted from 0, or None when it has no such NIC."""
|
|
if index >= len(self.macs[name]):
|
|
return None
|
|
marker = f'link/ether {self.macs[name][index]} '
|
|
for line in self.on(name, 'ip', '-o', 'link', 'show').stdout.splitlines():
|
|
if marker in line:
|
|
return line.split()[1].rstrip(':').split('@')[0]
|
|
return None
|
|
|
|
def address(self, name):
|
|
"""The IPv4 interface address of the instance's first NIC: the static one, else the one it holds."""
|
|
if self.addresses.get(name):
|
|
return ipaddress.IPv4Interface(self.addresses[name])
|
|
nic = self.nic_of(name, 0)
|
|
match = re.search(r' inet (\S+) ', self.on(name, 'ip', '-4', '-o', 'addr', 'show', 'dev', nic).stdout) \
|
|
if nic else None
|
|
if not match:
|
|
raise RuntimeError(f'{name} has no IPv4 address on its first NIC')
|
|
return ipaddress.IPv4Interface(match.group(1))
|
|
|
|
def client(self, name):
|
|
"""Install the tools of a VPN client; True when they are installed."""
|
|
return (self.on(name, 'apt-get', 'update', '-q').returncode == 0 and
|
|
self.on(name, 'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-q',
|
|
'wireguard-tools', 'openvpn', 'curl').returncode == 0)
|
|
|
|
def setup_run(self, name):
|
|
"""Run linux-manager-setup and wait for it; True when it succeeded."""
|
|
return self.on(name, 'systemctl', 'restart', 'linux-manager-setup').returncode == 0
|
|
|
|
def server(self, name, config):
|
|
"""Install the package with the test administrator, then apply the configuration; True when both succeeded."""
|
|
if not self.push(self.deb, name, '/tmp/linux-manager.deb'):
|
|
return False
|
|
preseed = (f'linux-manager linux-manager/admin_username string admin\n'
|
|
f'linux-manager linux-manager/admin_password password {self.admin_password}\n')
|
|
if self.on(name, 'debconf-set-selections', input=preseed).returncode != 0:
|
|
return False
|
|
result = self.on(name, 'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y',
|
|
'/tmp/linux-manager.deb')
|
|
(TMP / f'{name}-install.log').write_text(result.stdout + result.stderr, encoding='utf-8')
|
|
return result.returncode == 0 and self.configure(name, config)
|
|
|
|
def render(self, name, config):
|
|
"""Replace the server's configuration, with its placeholders filled in.
|
|
|
|
@PSK@ is the pre-shared key, @ETH0@ and @ETH1@ the names of the server's NICs and @CIDR0@ the
|
|
network of its first address.
|
|
"""
|
|
text = (HERE / 'configs' / config).read_text(encoding='utf-8').replace('@PSK@', self.psk)
|
|
for index in (0, 1):
|
|
placeholder = f'@ETH{index}@'
|
|
if placeholder in text:
|
|
nic = self.nic_of(name, index)
|
|
if nic is None:
|
|
raise RuntimeError(f'{name} has no NIC {index} for {placeholder} of {config}')
|
|
text = text.replace(placeholder, nic)
|
|
if '@CIDR0@' in text:
|
|
text = text.replace('@CIDR0@', str(self.address(name).network))
|
|
return self.push_text(text, name, '/etc/linux-manager/linux-manager.json', '0600')
|
|
|
|
def configure(self, name, config):
|
|
"""Render the configuration and apply it twice, as the server's certificates follow the first run."""
|
|
return self.render(name, config) and self.setup_run(name) and self.setup_run(name)
|
|
|
|
def echo_server(self, name):
|
|
"""Answer HTTP on port 8080 of the instance with the address of the client."""
|
|
return (self.push(HERE / 'echo.py', name, '/root/echo.py') and
|
|
self.on(name, 'systemd-run', '--unit=lm-test-echo', 'python3', '/root/echo.py').returncode == 0)
|
|
|
|
def vpn_user(self, name):
|
|
"""Add the VPN user alice, whose password is the administrator's; True when it was added."""
|
|
return self.on(name, 'linux-manager-user', '--json', 'add', 'alice', '--first-name', 'Alice',
|
|
'--last-name', 'Test', '--email', 'alice@example.org',
|
|
input=self.admin_password + '\n').returncode == 0
|
|
|
|
def alice(self, name):
|
|
"""User alice from the server's user list, or None."""
|
|
listing = self.json_of(name, 'linux-manager-user', '--json', 'list') or {}
|
|
return next((user for user in listing.get('users', []) if user.get('name') == 'alice'), None)
|
|
|
|
def wireguard_client(self, client, server, endpoint, address, allowed):
|
|
"""Give alice a peer of a new client key and bring wg0 up on the client towards the server, for allowed only."""
|
|
private_key = self.on(client, 'wg', 'genkey').stdout.strip()
|
|
public_key = self.on(client, 'wg', 'pubkey', input=private_key + '\n').stdout.strip()
|
|
if not private_key or not public_key:
|
|
return False
|
|
if not self.push(HERE / 'wireguard_peer.py', server, '/root/wireguard_peer.py'):
|
|
return False
|
|
if self.on(server, 'python3', '/root/wireguard_peer.py', public_key, address).returncode != 0:
|
|
return False
|
|
if not self.setup_run(server):
|
|
return False
|
|
server_key = self.on(server, 'cat', '/etc/wireguard/wg0.pub').stdout.strip()
|
|
conf = (f'[Interface]\nPrivateKey = {private_key}\nAddress = {address}/32\n\n'
|
|
f'[Peer]\nPublicKey = {server_key}\nEndpoint = {endpoint}:51820\nAllowedIPs = {allowed}\n')
|
|
return (self.push_text(conf, client, '/etc/wireguard/wg0.conf', '0600') and
|
|
self.on(client, 'bash', '/usr/bin/wg-quick', 'up', 'wg0').returncode == 0)
|
|
|
|
def wireguard_module(self, name):
|
|
"""True when the instance can create a WireGuard link, that is the host kernel has the module."""
|
|
if self.on(name, 'ip', 'link', 'add', 'lm-probe', 'type', 'wireguard').returncode != 0:
|
|
return False
|
|
self.on(name, 'ip', 'link', 'delete', 'lm-probe')
|
|
return True
|
|
|
|
def cleanup(self):
|
|
"""Delete the project's instances, the networks and the project, or with keep say what is left."""
|
|
if self.keep:
|
|
print(f'--keep: project {self.project} and networks {" ".join(self.networks)} are left on {self.remote}')
|
|
return
|
|
listing = self.lxc('list', f'{self.remote}:', '-c', 'n', '-f', 'csv', check=False)
|
|
for name in listing.stdout.split():
|
|
self.lxc('delete', '--force', f'{self.remote}:{name}', check=False)
|
|
for name in self.networks:
|
|
subprocess.run(['lxc', 'network', 'delete', f'{self.remote}:{name}'], capture_output=True)
|
|
self.networks = []
|
|
subprocess.run(['lxc', 'project', 'delete', f'{self.remote}:{self.project}'], capture_output=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument('scenarios', nargs='+', metavar='scenario', help=f'{", ".join(SCENARIOS)} or all')
|
|
parser.add_argument('--remote', default=os.environ.get('LXD_REMOTE'), help='the lxc remote (LXD_REMOTE)')
|
|
parser.add_argument('--pool', default=os.environ.get('LXD_POOL'),
|
|
help='the storage pool of the project (LXD_POOL)')
|
|
parser.add_argument('--project', default=os.environ.get('LXD_PROJECT', 'lm-test'),
|
|
help='the project the instances live in (LXD_PROJECT, default lm-test)')
|
|
parser.add_argument('--image', default=os.environ.get('LXD_IMAGE', 'ubuntu:26.04'),
|
|
help='the image of every instance (LXD_IMAGE, default ubuntu:26.04)')
|
|
parser.add_argument('--keep', action='store_true', help='leave the project and the networks in place')
|
|
args = parser.parse_args()
|
|
if not args.remote:
|
|
parser.error('--remote or LXD_REMOTE is required')
|
|
if not args.pool:
|
|
parser.error('--pool or LXD_POOL is required')
|
|
scenarios = SCENARIOS if args.scenarios == ['all'] else args.scenarios
|
|
unknown = [name for name in scenarios if name not in SCENARIOS]
|
|
if unknown:
|
|
parser.error(f'no scenario {", ".join(unknown)}')
|
|
|
|
debs = sorted((ROOT / 'out').glob('linux-manager_*_all.deb'))
|
|
if not debs:
|
|
sys.exit('no out/linux-manager_*_all.deb; run ./build.sh first')
|
|
if subprocess.run(['lxc', 'project', 'show', f'{args.remote}:{args.project}'], capture_output=True).returncode == 0:
|
|
sys.exit(f'project {args.project} exists on {args.remote}; remove it or choose another with --project')
|
|
|
|
TMP.mkdir(parents=True, exist_ok=True)
|
|
lab = Lab(args.remote, args.project, args.pool, args.image, args.keep, debs[-1],
|
|
secrets.token_hex(16), secrets.token_hex(24))
|
|
env = ROOT / 'tmp' / 'remote-test.env'
|
|
env.touch(mode=0o600)
|
|
env.chmod(0o600)
|
|
env.write_text(f'ADMIN_USERNAME=admin\nADMIN_PASSWORD={lab.admin_password}\nPSK={lab.psk}\n', encoding='utf-8')
|
|
print(f'package: {debs[-1].name}; remote: {args.remote}; project: {args.project}; image: {args.image}')
|
|
|
|
sys.path.insert(0, str(HERE))
|
|
try:
|
|
for name in scenarios:
|
|
subprocess.run(['lxc', 'project', 'create', f'{args.remote}:{args.project}', '-s', args.pool,
|
|
'-c', 'features.images=false'], capture_output=True, check=True)
|
|
print(f'== {name}')
|
|
try:
|
|
importlib.import_module(f'scenarios.{name}').run(lab)
|
|
except (subprocess.CalledProcessError, RuntimeError) as error:
|
|
detail = error.stderr.strip() if isinstance(error, subprocess.CalledProcessError) else str(error)
|
|
lab.result(f'{name}: scenario ran to its end ({detail})', False)
|
|
finally:
|
|
lab.cleanup()
|
|
except KeyboardInterrupt:
|
|
sys.exit(130)
|
|
|
|
print(f'== passed {lab.passed}, failed {lab.failed}, skipped {lab.skipped}')
|
|
print("Not covered: IKEv2 client access, HTTPS with Let's Encrypt, the OpenID Provider and passkeys,")
|
|
print('PowerDNS answering from outside, the Azure and GCP platform modules.')
|
|
sys.exit(1 if lab.failed else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|