Files
linux-cloud-router/tests/remote/run.py
T

361 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
"""Run the Linux Manager test scenarios on an LXD remote: run.py core|nat|s2s... or run.py all.
The remote and its storage pool are required, as --remote and --pool or LXD_REMOTE and
LXD_POOL. The instances live in a project of their own, which must not exist and is removed
with its networks after each scenario unless --keep is given. The package is the newest
out/linux-manager_*_all.deb. Each check prints PASS, FAIL or SKIP; the run exits 1 when a
check failed.
"""
import argparse
import importlib
import ipaddress
import json
import os
import pathlib
import re
import secrets
import subprocess
import sys
import time
HERE = pathlib.Path(__file__).resolve().parent
ROOT = HERE.parent.parent
TMP = ROOT / 'tmp' / 'remote-test'
SCENARIOS = sorted(path.stem for path in (HERE / 'scenarios').glob('*.py') if path.stem != '__init__')
class Lab:
"""The instances and networks of a run on one LXD remote and project, and the results of its checks.
A NIC is a (network, mac, address/prefix[, gateway]) tuple; an address of None is taken from
DHCP.
"""
def __init__(self, remote, project, pool, image, keep, deb, admin_password, psk):
self.remote = remote
self.project = project
self.pool = pool
self.image = image
self.keep = keep
self.deb = deb
self.admin_password = admin_password
self.psk = psk
self.passed = self.failed = self.skipped = 0
self.networks = []
self.macs = {}
self.addresses = {}
def lxc(self, *args, check=True):
"""Run lxc with the arguments and the project; raise CalledProcessError when it fails, unless check is false."""
return subprocess.run(['lxc', *args, '--project', self.project], capture_output=True, text=True, check=check)
def on(self, name, *command, input=None):
"""Run the command in the instance with the input, and return its CompletedProcess."""
return subprocess.run(
['lxc', 'exec', f'{self.remote}:{name}', '--project', self.project, '-T', '--', *command],
input=input, capture_output=True, text=True)
def push(self, source, name, path, mode='0644'):
"""Copy the local file into the instance; True when it was copied."""
result = self.lxc('file', 'push', str(source), f'{self.remote}:{name}{path}', '--mode', mode, check=False)
return result.returncode == 0
def push_text(self, text, name, path, mode='0644'):
"""Write the text to a local file readable by its owner only, copy it into the instance and remove it."""
local = TMP / f'{name}{path.replace("/", "_")}'
local.touch(mode=0o600)
local.chmod(0o600)
local.write_text(text, encoding='utf-8')
try:
return self.push(local, name, path, mode)
finally:
local.unlink()
def json_of(self, name, *command):
"""The JSON output of the command in the instance, or None when the command fails or its output is no JSON."""
result = self.on(name, *command)
try:
return json.loads(result.stdout) if result.returncode == 0 else None
except ValueError:
return None
def result(self, description, ok):
"""Count and print the check as PASS when ok, else FAIL."""
if ok:
self.passed += 1
print(f'PASS {description}')
else:
self.failed += 1
print(f'FAIL {description}')
def skip(self, description):
"""Count and print the check as SKIP."""
self.skipped += 1
print(f'SKIP {description}')
def check(self, description, name, *command):
"""Pass when the command succeeds in the instance."""
self.result(description, self.on(name, *command).returncode == 0)
def expect(self, description, pattern, name, *command):
"""Pass when a line of the command's output, standard error included, matches the regular expression."""
result = self.on(name, *command)
self.result(description, re.search(pattern, result.stdout + result.stderr, re.M) is not None)
def refuse(self, description, pattern, name, *command):
"""Pass when no line of the command's output, standard error included, matches the regular expression."""
result = self.on(name, *command)
self.result(description, re.search(pattern, result.stdout + result.stderr, re.M) is None)
def network(self, name, address):
"""Create a bridge with outbound NAT at the gateway address GATEWAY/PREFIX, or without a host address when None.
A bridge without a host address is reachable only through the instances on it.
"""
if address is None:
self.lxc('network', 'create', f'{self.remote}:{name}', 'ipv4.address=none', 'ipv6.address=none')
else:
self.lxc('network', 'create', f'{self.remote}:{name}', f'ipv4.address={address}', 'ipv4.nat=true',
'ipv4.dhcp=false', 'ipv6.address=none')
self.networks.append(name)
def instance(self, name, kind, user_data, *nics):
"""Create and start a vm or container with the NICs, whose static addresses are set through cloud-init.
A container's NICs are eth0 and on, named by LXD, as a veth has no permanent MAC address for
netplan to match; a VM's NICs keep their kernel names, which the initramfs sets before
netplan could rename them, and are matched by MAC address (nic_of).
"""
self.macs[name] = [nic[1] for nic in nics]
self.addresses[name] = nics[0][2]
netcfg = 'version: 2\nethernets:\n'
for index, (_, mac, address, *gateway) in enumerate(nics):
if kind == 'vm':
netcfg += f' nic{index}:\n match:\n macaddress: "{mac}"\n'
else:
netcfg += f' eth{index}:\n'
netcfg += f' addresses: [{address}]\n' if address else ' dhcp4: true\n'
if gateway:
netcfg += (f' routes:\n - to: default\n via: {gateway[0]}\n'
f' nameservers:\n addresses: [{gateway[0]}]\n')
limits = ['-c', 'limits.cpu=2', '-c', 'limits.memory=4GiB'] if kind == 'vm' else \
['-c', 'limits.cpu=1', '-c', 'limits.memory=1GiB']
self.lxc('init', self.image, f'{self.remote}:{name}', *(['--vm'] if kind == 'vm' else []), *limits,
'-c', 'cloud-init.user-data=' + (HERE / 'cloud-init' / user_data).read_text(encoding='utf-8'),
'-c', 'cloud-init.network-config=' + netcfg)
if kind != 'vm':
self.lxc('config', 'device', 'add', f'{self.remote}:{name}', 'tun', 'unix-char', 'path=/dev/net/tun')
for index, (network, mac, *_) in enumerate(nics):
self.lxc('config', 'device', 'add', f'{self.remote}:{name}', f'eth{index}', 'nic', f'network={network}',
f'hwaddr={mac}', *([] if kind == 'vm' else [f'name=eth{index}']))
self.lxc('start', f'{self.remote}:{name}')
def ready(self, *names):
"""Wait for each instance's agent and cloud-init, then for its first address; False when one is not ready.
cloud-init's status 2, done with recoverable errors, counts as booted. An instance whose first
NIC takes its address from DHCP waits for any IPv4 address outside lo.
"""
for name in names:
for _ in range(120):
if self.on(name, 'cloud-init', 'status', '--wait').returncode in (0, 2):
break
time.sleep(5)
else:
print(f'{name} did not boot', file=sys.stderr)
return False
address = self.addresses[name]
pattern = f' inet {re.escape(address.split("/")[0])}/' if address else r'^\d+: (?!lo\b)\S+ +inet '
for _ in range(24):
if re.search(pattern, self.on(name, 'ip', '-4', '-o', 'addr', 'show').stdout, re.M):
break
time.sleep(5)
else:
print(f'{name} has no address {address or "from DHCP"}', file=sys.stderr)
return False
return True
def nic_of(self, name, index):
"""The name of the instance's NIC index, counted from 0, or None when it has no such NIC."""
if index >= len(self.macs[name]):
return None
marker = f'link/ether {self.macs[name][index]} '
for line in self.on(name, 'ip', '-o', 'link', 'show').stdout.splitlines():
if marker in line:
return line.split()[1].rstrip(':').split('@')[0]
return None
def address(self, name):
"""The IPv4 interface address of the instance's first NIC: the static one, else the one it holds."""
if self.addresses.get(name):
return ipaddress.IPv4Interface(self.addresses[name])
nic = self.nic_of(name, 0)
match = re.search(r' inet (\S+) ', self.on(name, 'ip', '-4', '-o', 'addr', 'show', 'dev', nic).stdout) \
if nic else None
if not match:
raise RuntimeError(f'{name} has no IPv4 address on its first NIC')
return ipaddress.IPv4Interface(match.group(1))
def client(self, name):
"""Install the tools of a VPN client; True when they are installed."""
return (self.on(name, 'apt-get', 'update', '-q').returncode == 0 and
self.on(name, 'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-q',
'wireguard-tools', 'openvpn', 'curl').returncode == 0)
def setup_run(self, name):
"""Run linux-manager-setup and wait for it; True when it succeeded."""
return self.on(name, 'systemctl', 'restart', 'linux-manager-setup').returncode == 0
def server(self, name, config):
"""Install the package with the test administrator, then apply the configuration; True when both succeeded."""
if not self.push(self.deb, name, '/tmp/linux-manager.deb'):
return False
preseed = (f'linux-manager linux-manager/admin_username string admin\n'
f'linux-manager linux-manager/admin_password password {self.admin_password}\n')
if self.on(name, 'debconf-set-selections', input=preseed).returncode != 0:
return False
result = self.on(name, 'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y',
'/tmp/linux-manager.deb')
(TMP / f'{name}-install.log').write_text(result.stdout + result.stderr, encoding='utf-8')
return result.returncode == 0 and self.configure(name, config)
def render(self, name, config):
"""Replace the server's configuration, with its placeholders filled in.
@PSK@ is the pre-shared key, @ETH0@ and @ETH1@ the names of the server's NICs and @CIDR0@ the
network of its first address.
"""
text = (HERE / 'configs' / config).read_text(encoding='utf-8').replace('@PSK@', self.psk)
for index in (0, 1):
placeholder = f'@ETH{index}@'
if placeholder in text:
nic = self.nic_of(name, index)
if nic is None:
raise RuntimeError(f'{name} has no NIC {index} for {placeholder} of {config}')
text = text.replace(placeholder, nic)
if '@CIDR0@' in text:
text = text.replace('@CIDR0@', str(self.address(name).network))
return self.push_text(text, name, '/etc/linux-manager/linux-manager.json', '0600')
def configure(self, name, config):
"""Render the configuration and apply it twice, as the server's certificates follow the first run."""
return self.render(name, config) and self.setup_run(name) and self.setup_run(name)
def echo_server(self, name):
"""Answer HTTP on port 8080 of the instance with the address of the client."""
return (self.push(HERE / 'echo.py', name, '/root/echo.py') and
self.on(name, 'systemd-run', '--unit=lm-test-echo', 'python3', '/root/echo.py').returncode == 0)
def vpn_user(self, name):
"""Add the VPN user alice, whose password is the administrator's; True when it was added."""
return self.on(name, 'linux-manager-user', '--json', 'add', 'alice', '--first-name', 'Alice',
'--last-name', 'Test', '--email', 'alice@example.org',
input=self.admin_password + '\n').returncode == 0
def alice(self, name):
"""User alice from the server's user list, or None."""
listing = self.json_of(name, 'linux-manager-user', '--json', 'list') or {}
return next((user for user in listing.get('users', []) if user.get('name') == 'alice'), None)
def wireguard_client(self, client, server, endpoint, address, allowed):
"""Give alice a peer of a new client key and bring wg0 up on the client towards the server, for allowed only."""
private_key = self.on(client, 'wg', 'genkey').stdout.strip()
public_key = self.on(client, 'wg', 'pubkey', input=private_key + '\n').stdout.strip()
if not private_key or not public_key:
return False
if not self.push(HERE / 'wireguard_peer.py', server, '/root/wireguard_peer.py'):
return False
if self.on(server, 'python3', '/root/wireguard_peer.py', public_key, address).returncode != 0:
return False
if not self.setup_run(server):
return False
server_key = self.on(server, 'cat', '/etc/wireguard/wg0.pub').stdout.strip()
conf = (f'[Interface]\nPrivateKey = {private_key}\nAddress = {address}/32\n\n'
f'[Peer]\nPublicKey = {server_key}\nEndpoint = {endpoint}:51820\nAllowedIPs = {allowed}\n')
return (self.push_text(conf, client, '/etc/wireguard/wg0.conf', '0600') and
self.on(client, 'bash', '/usr/bin/wg-quick', 'up', 'wg0').returncode == 0)
def wireguard_module(self, name):
"""True when the instance can create a WireGuard link, that is the host kernel has the module."""
if self.on(name, 'ip', 'link', 'add', 'lm-probe', 'type', 'wireguard').returncode != 0:
return False
self.on(name, 'ip', 'link', 'delete', 'lm-probe')
return True
def cleanup(self):
"""Delete the project's instances, the networks and the project, or with keep say what is left."""
if self.keep:
print(f'--keep: project {self.project} and networks {" ".join(self.networks)} are left on {self.remote}')
return
listing = self.lxc('list', f'{self.remote}:', '-c', 'n', '-f', 'csv', check=False)
for name in listing.stdout.split():
self.lxc('delete', '--force', f'{self.remote}:{name}', check=False)
for name in self.networks:
subprocess.run(['lxc', 'network', 'delete', f'{self.remote}:{name}'], capture_output=True)
self.networks = []
subprocess.run(['lxc', 'project', 'delete', f'{self.remote}:{self.project}'], capture_output=True)
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument('scenarios', nargs='+', metavar='scenario', help=f'{", ".join(SCENARIOS)} or all')
parser.add_argument('--remote', default=os.environ.get('LXD_REMOTE'), help='the lxc remote (LXD_REMOTE)')
parser.add_argument('--pool', default=os.environ.get('LXD_POOL'),
help='the storage pool of the project (LXD_POOL)')
parser.add_argument('--project', default=os.environ.get('LXD_PROJECT', 'lm-test'),
help='the project the instances live in (LXD_PROJECT, default lm-test)')
parser.add_argument('--image', default=os.environ.get('LXD_IMAGE', 'ubuntu:26.04'),
help='the image of every instance (LXD_IMAGE, default ubuntu:26.04)')
parser.add_argument('--keep', action='store_true', help='leave the project and the networks in place')
args = parser.parse_args()
if not args.remote:
parser.error('--remote or LXD_REMOTE is required')
if not args.pool:
parser.error('--pool or LXD_POOL is required')
scenarios = SCENARIOS if args.scenarios == ['all'] else args.scenarios
unknown = [name for name in scenarios if name not in SCENARIOS]
if unknown:
parser.error(f'no scenario {", ".join(unknown)}')
debs = sorted((ROOT / 'out').glob('linux-manager_*_all.deb'))
if not debs:
sys.exit('no out/linux-manager_*_all.deb; run ./build.sh first')
if subprocess.run(['lxc', 'project', 'show', f'{args.remote}:{args.project}'], capture_output=True).returncode == 0:
sys.exit(f'project {args.project} exists on {args.remote}; remove it or choose another with --project')
TMP.mkdir(parents=True, exist_ok=True)
lab = Lab(args.remote, args.project, args.pool, args.image, args.keep, debs[-1],
secrets.token_hex(16), secrets.token_hex(24))
env = ROOT / 'tmp' / 'remote-test.env'
env.touch(mode=0o600)
env.chmod(0o600)
env.write_text(f'ADMIN_USERNAME=admin\nADMIN_PASSWORD={lab.admin_password}\nPSK={lab.psk}\n', encoding='utf-8')
print(f'package: {debs[-1].name}; remote: {args.remote}; project: {args.project}; image: {args.image}')
sys.path.insert(0, str(HERE))
try:
for name in scenarios:
subprocess.run(['lxc', 'project', 'create', f'{args.remote}:{args.project}', '-s', args.pool,
'-c', 'features.images=false'], capture_output=True, check=True)
print(f'== {name}')
try:
importlib.import_module(f'scenarios.{name}').run(lab)
except (subprocess.CalledProcessError, RuntimeError) as error:
detail = error.stderr.strip() if isinstance(error, subprocess.CalledProcessError) else str(error)
lab.result(f'{name}: scenario ran to its end ({detail})', False)
finally:
lab.cleanup()
except KeyboardInterrupt:
sys.exit(130)
print(f'== passed {lab.passed}, failed {lab.failed}, skipped {lab.skipped}')
print("Not covered: IKEv2 client access, HTTPS with Let's Encrypt, the OpenID Provider and passkeys,")
print('PowerDNS answering from outside, the Azure and GCP platform modules.')
sys.exit(1 if lab.failed else 0)
if __name__ == '__main__':
main()