fix: connectivity panel rearrangement. LXD VM for repeatable tests.

This commit is contained in:
slawek committed 2026-11-22 15:59:40 +01:00
1 parent 85a1ffa21f
commit da889b4865
15 files changed
+363 -47

No files matched your search

+50 -3
View File
@@ -1,6 +1,6 @@
# Remote tests
`run.py` installs the newest `out/linux-manager_*_all.deb` into LXD instances of a remote and
`run.py` installs the newest `out/linux-manager_*_all.deb` into instances of an LXD remote and
checks the router and VPN components in three scenarios:
- `core`: one interface, router and VPN off;
@@ -44,5 +44,52 @@ its end. The administrator password and the pre-shared key are generated per run
`tmp/remote-test.env`. The run prints PASS, FAIL or SKIP per check and exits non-zero on a
failure; WireGuard is skipped when the host kernel has no WireGuard module.
Not covered: IKEv2 client access, HTTPS with Let's Encrypt, the OpenID Provider and passkeys,
PowerDNS answering from outside, and the Azure and GCP platform modules.
The configurations in `configs/` have placeholders that are filled in for each server:
- `@PSK@`: the pre-shared key;
- `@ETH0@` and `@ETH1@`: the names of its interfaces;
- `@CIDR0@`: the network of its first address.
## A VM to work with
`vm.py` launches one server VM, `lm-dev` by default, on the LXD remote and leaves it
running. The VM is created in a project, `default` by default, with that project's default
profile, and takes its address from DHCP. It gets the newest package with the administrator
`admin` and a generated password, written to `tmp/vm-<name>.env`, and a configuration from
`configs/`, `core.json` by default; `@ETH0@` is the interface of the VM's default route.
The leased address is pinned on the VM's NIC, as NAT proxy devices require a static address.
Proxy devices then forward ports of the LXD host address `--listen` to the VM; by default that is
the IPv4 address of the host in the remote's URL (`lxc remote list`), which must be an address of
the LXD host itself:
- `7080` to the Manager;
- `7081` to the status web service;
- `2222` to SSH, port `22`.
The script prints the VM's address, the administrator's password, the proxied URLs, the command
of a shell in it and the command that removes it.
```sh
tests/remote/vm.py
```
```sh
tests/remote/vm.py --remove
```
`--remove` deletes the VM and its env file. Options:
| Option | Variable | Default | Meaning |
|---|---|---|---|
| `--remote` | `LXD_REMOTE` | required | The `lxc` remote |
| `--project` | | `default` | The project of the VM |
| `--name` | | `lm-dev` | The name of the VM; it must not exist |
| `--image` | `LXD_IMAGE` | `ubuntu:26.04` | The image of the VM |
| `--config` | | `core.json` | The configuration from `configs/` |
| `--listen` | `LXD_LISTEN` | the remote URL's host | An address of the LXD host the proxies listen on |
## Scope of the tests
Not covered by `run.py`: IKEv2 client access, HTTPS with Let's Encrypt, the OpenID Provider and
passkeys, PowerDNS answering from outside, and the Azure and GCP platform modules.
+1 -1
View File
@@ -4,7 +4,7 @@
"vpn": {"enabled": false},
"interfaces": {"external": "@ETH0@"},
"wan": {"local_fqdn": "lm-nat.test"},
"local": {"cidrs": ["172.31.11.0/24"]},
"local": {"cidrs": ["@CIDR0@"]},
"wireguard": {"enabled": false, "address": "10.66.0.1/24", "listen_port": 51820},
"openvpn": {"enabled": false, "address_pool": "10.67.0.0/24", "port": 1194, "protocol": "udp"},
"https": {"enabled": false}
+1 -1
View File
@@ -4,7 +4,7 @@
"vpn": {"enabled": true},
"interfaces": {"external": "@ETH0@"},
"wan": {"local_fqdn": "lm-nat.test"},
"local": {"cidrs": ["172.31.11.0/24"]},
"local": {"cidrs": ["@CIDR0@"]},
"wireguard": {"enabled": true, "address": "10.66.0.1/24", "listen_port": 51820},
"openvpn": {"enabled": true, "address_pool": "10.67.0.0/24", "port": 1194, "protocol": "udp"},
"https": {"enabled": false}
+38 -14
View File
@@ -10,6 +10,7 @@ check failed.
import argparse
import importlib
import ipaddress
import json
import os
import pathlib
@@ -26,7 +27,11 @@ SCENARIOS = sorted(path.stem for path in (HERE / 'scenarios').glob('*.py') if pa
class Lab:
"""The instances and networks of a run on one LXD remote and project, and the results of its checks."""
"""The instances and networks of a run on one LXD remote and project, and the results of its checks.
A NIC is a (network, mac, address/prefix[, gateway]) tuple; an address of None is taken from
DHCP.
"""
def __init__(self, remote, project, pool, image, keep, deb, admin_password, psk):
self.remote = remote
@@ -117,22 +122,21 @@ class Lab:
self.networks.append(name)
def instance(self, name, kind, user_data, *nics):
"""Create and start a vm or container whose NICs are (network, mac, address/prefix[, gateway]) tuples.
"""Create and start a vm or container with the NICs, whose static addresses are set through cloud-init.
The addresses are static, set through cloud-init. A container's NICs are eth0 and on, named
by LXD, as a veth has no permanent MAC address for netplan to match; a VM's NICs keep their
kernel names, which the initramfs sets before netplan could rename them, and are matched by
MAC address (nic_of).
A container's NICs are eth0 and on, named by LXD, as a veth has no permanent MAC address for
netplan to match; a VM's NICs keep their kernel names, which the initramfs sets before
netplan could rename them, and are matched by MAC address (nic_of).
"""
self.macs[name] = [nic[1] for nic in nics]
self.addresses[name] = nics[0][2].split('/')[0]
self.addresses[name] = nics[0][2]
netcfg = 'version: 2\nethernets:\n'
for index, (_, mac, address, *gateway) in enumerate(nics):
if kind == 'vm':
netcfg += f' nic{index}:\n match:\n macaddress: "{mac}"\n'
else:
netcfg += f' eth{index}:\n'
netcfg += f' addresses: [{address}]\n'
netcfg += f' addresses: [{address}]\n' if address else ' dhcp4: true\n'
if gateway:
netcfg += (f' routes:\n - to: default\n via: {gateway[0]}\n'
f' nameservers:\n addresses: [{gateway[0]}]\n')
@@ -151,7 +155,8 @@ class Lab:
def ready(self, *names):
"""Wait for each instance's agent and cloud-init, then for its first address; False when one is not ready.
cloud-init's status 2, done with recoverable errors, counts as booted.
cloud-init's status 2, done with recoverable errors, counts as booted. An instance whose first
NIC takes its address from DHCP waits for any IPv4 address outside lo.
"""
for name in names:
for _ in range(120):
@@ -161,13 +166,14 @@ class Lab:
else:
print(f'{name} did not boot', file=sys.stderr)
return False
pattern = f' inet {re.escape(self.addresses[name])}/'
address = self.addresses[name]
pattern = f' inet {re.escape(address.split("/")[0])}/' if address else r'^\d+: (?!lo\b)\S+ +inet '
for _ in range(24):
if re.search(pattern, self.on(name, 'ip', '-4', '-o', 'addr', 'show').stdout):
if re.search(pattern, self.on(name, 'ip', '-4', '-o', 'addr', 'show').stdout, re.M):
break
time.sleep(5)
else:
print(f'{name} has no address {self.addresses[name]}', file=sys.stderr)
print(f'{name} has no address {address or "from DHCP"}', file=sys.stderr)
return False
return True
@@ -181,6 +187,17 @@ class Lab:
return line.split()[1].rstrip(':').split('@')[0]
return None
def address(self, name):
"""The IPv4 interface address of the instance's first NIC: the static one, else the one it holds."""
if self.addresses.get(name):
return ipaddress.IPv4Interface(self.addresses[name])
nic = self.nic_of(name, 0)
match = re.search(r' inet (\S+) ', self.on(name, 'ip', '-4', '-o', 'addr', 'show', 'dev', nic).stdout) \
if nic else None
if not match:
raise RuntimeError(f'{name} has no IPv4 address on its first NIC')
return ipaddress.IPv4Interface(match.group(1))
def client(self, name):
"""Install the tools of a VPN client; True when they are installed."""
return (self.on(name, 'apt-get', 'update', '-q').returncode == 0 and
@@ -205,7 +222,11 @@ class Lab:
return result.returncode == 0 and self.configure(name, config)
def render(self, name, config):
"""Replace the server's configuration, with the pre-shared key for @PSK@ and its NIC names for @ETH0@ and @ETH1@."""
"""Replace the server's configuration, with its placeholders filled in.
@PSK@ is the pre-shared key, @ETH0@ and @ETH1@ the names of the server's NICs and @CIDR0@ the
network of its first address.
"""
text = (HERE / 'configs' / config).read_text(encoding='utf-8').replace('@PSK@', self.psk)
for index in (0, 1):
placeholder = f'@ETH{index}@'
@@ -214,6 +235,8 @@ class Lab:
if nic is None:
raise RuntimeError(f'{name} has no NIC {index} for {placeholder} of {config}')
text = text.replace(placeholder, nic)
if '@CIDR0@' in text:
text = text.replace('@CIDR0@', str(self.address(name).network))
return self.push_text(text, name, '/etc/linux-manager/linux-manager.json', '0600')
def configure(self, name, config):
@@ -251,7 +274,8 @@ class Lab:
server_key = self.on(server, 'cat', '/etc/wireguard/wg0.pub').stdout.strip()
conf = (f'[Interface]\nPrivateKey = {private_key}\nAddress = {address}/32\n\n'
f'[Peer]\nPublicKey = {server_key}\nEndpoint = {endpoint}:51820\nAllowedIPs = {allowed}\n')
return self.push_text(conf, client, '/etc/wireguard/wg0.conf', '0600') and self.on(client, 'bash', '/usr/bin/wg-quick', 'up', 'wg0').returncode == 0
return (self.push_text(conf, client, '/etc/wireguard/wg0.conf', '0600') and
self.on(client, 'bash', '/usr/bin/wg-quick', 'up', 'wg0').returncode == 0)
def wireguard_module(self, name):
"""True when the instance can create a WireGuard link, that is the host kernel has the module."""
+2 -1
View File
@@ -34,4 +34,5 @@ def run(lab):
lab.expect('core: web state has router and vpn false', r'"router": false, "vpn": false',
'lm-core', 'cat', '/run/linux-manager/web.json')
lab.expect('core: status web service answers the client', r'^200$',
'lm-core-client', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', 'http://172.31.10.10:7081/status')
'lm-core-client', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}',
f'http://{lab.address("lm-core").ip}:7081/status')
+10 -7
View File
@@ -18,6 +18,9 @@ def run(lab):
lab.result('nat: package installed and configured', lab.server('lm-nat', 'nat.json'))
lab.echo_server('lm-nat-target')
ext = re.escape(lab.nic_of('lm-nat', 0) or '')
server = str(lab.address('lm-nat').ip)
target = str(lab.address('lm-nat-target').ip)
from_server = f'^{re.escape(server)}$'
lab.expect('nat: setup succeeded', r'^success$',
'lm-nat', 'systemctl', 'show', '-p', 'Result', '--value', 'linux-manager-setup')
@@ -34,9 +37,9 @@ def run(lab):
lab.result('nat: VPN user has a certificate', user is not None and user.get('certificate') is True)
if lab.wireguard_module('lm-nat-client'):
if lab.wireguard_client('lm-nat-client', 'lm-nat', '172.31.11.10', '10.66.0.2', '172.31.11.20/32'):
lab.expect('nat: WireGuard client reaches the target from the server address', r'^172\.31\.11\.10$',
'lm-nat-client', 'curl', '-s', '--max-time', '10', 'http://172.31.11.20:8080/')
if lab.wireguard_client('lm-nat-client', 'lm-nat', server, '10.66.0.2', f'{target}/32'):
lab.expect('nat: WireGuard client reaches the target from the server address', from_server,
'lm-nat-client', 'curl', '-s', '--max-time', '10', f'http://{target}:8080/')
else:
lab.result('nat: WireGuard client set up', False)
lab.on('lm-nat-client', 'bash', '/usr/bin/wg-quick', 'down', 'wg0')
@@ -46,16 +49,16 @@ def run(lab):
profile = (lab.json_of('lm-nat', 'linux-manager-user', '--json', 'profile', 'alice') or {}).get('profile', '')
lab.push_text(profile, 'lm-nat-client', '/root/alice.ovpn', '0600')
lab.push_text(f'alice\n{lab.admin_password}\n', 'lm-nat-client', '/root/alice.pass', '0600')
lab.on('lm-nat-client', 'sh', '-c', 'echo "172.31.11.10 lm-nat.test" >>/etc/hosts')
lab.on('lm-nat-client', 'sh', '-c', f'echo "{server} lm-nat.test" >>/etc/hosts')
lab.on('lm-nat-client', 'openvpn', '--config', '/root/alice.ovpn', '--auth-user-pass', '/root/alice.pass',
'--route-nopull', '--disable-dco', '--route', '172.31.11.20', '255.255.255.255',
'--route-nopull', '--disable-dco', '--route', target, '255.255.255.255',
'--daemon', '--log', '/root/openvpn.log')
for _ in range(12):
if lab.on('lm-nat-client', 'grep', '-q', 'Initialization Sequence Completed', '/root/openvpn.log').returncode == 0:
break
time.sleep(5)
lab.expect('nat: OpenVPN client reaches the target from the server address', r'^172\.31\.11\.10$',
'lm-nat-client', 'curl', '-s', '--max-time', '10', 'http://172.31.11.20:8080/')
lab.expect('nat: OpenVPN client reaches the target from the server address', from_server,
'lm-nat-client', 'curl', '-s', '--max-time', '10', f'http://{target}:8080/')
lab.on('lm-nat-client', 'pkill', 'openvpn')
lab.configure('lm-nat', 'nat-off.json')
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
"""Launch one Linux Manager server VM on an LXD remote to work with, or remove it: vm.py [--remove].
The VM is launched on the remote (--remote or LXD_REMOTE) in --project with that project's
default profile. It gets the newest out/linux-manager_*_all.deb with the administrator admin and
a generated password, and a configuration from configs/ (core.json by default), whose @ETH0@ is
the interface of the VM's default route, @CIDR0@ its network and @PSK@ a generated key. The
password and the key are written to tmp/vm-NAME.env. The VM's leased address is pinned on its
NIC, and NAT proxy devices forward ports 7080 and 7081 of the host address --listen (or LXD_LISTEN,
by default the IPv4 address of the host of the remote's URL) to the same ports of the VM, and port
2222 to its SSH port. --remove deletes the VM and that file.
"""
import argparse
import ipaddress
import json
import os
import pathlib
import re
import secrets
import socket
import subprocess
import sys
import time
import urllib.parse
HERE = pathlib.Path(__file__).resolve().parent
ROOT = HERE.parent.parent
# The host port and the VM port of each proxy: the Manager, the status web service and SSH.
PROXIES = ((7080, 7080), (7081, 7081), (2222, 22))
def inside(args, *command, input=None):
"""Run the command in the VM with the input, and return its CompletedProcess."""
return subprocess.run(['lxc', 'exec', f'{args.remote}:{args.name}', '--project', args.project, '-T', '--',
*command], input=input, capture_output=True, text=True)
def push(args, source, path, mode):
"""Copy the local file into the VM at the path with the mode; True when it was copied."""
return subprocess.run(['lxc', 'file', 'push', str(source), f'{args.remote}:{args.name}{path}',
'--project', args.project, '--mode', mode], capture_output=True).returncode == 0
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument('--remote', default=os.environ.get('LXD_REMOTE'), help='the lxc remote (LXD_REMOTE)')
parser.add_argument('--project', default='default', help='the project of the VM (default default)')
parser.add_argument('--name', default='lm-dev', help='the name of the VM (default lm-dev)')
parser.add_argument('--image', default=os.environ.get('LXD_IMAGE', 'ubuntu:26.04'),
help='the image of the VM (LXD_IMAGE, default ubuntu:26.04)')
parser.add_argument('--config', default='core.json', help='the configuration from configs/ (default core.json)')
parser.add_argument('--listen', default=os.environ.get('LXD_LISTEN'),
help='the LXD host address the proxies listen on (LXD_LISTEN, default the IPv4 address '
'of the remote URL\'s host)')
parser.add_argument('--remove', action='store_true', help='delete the VM and its env file')
args = parser.parse_args()
if not args.remote:
parser.error('--remote or LXD_REMOTE is required')
if not args.remove and not args.listen:
# The default listen address is the IPv4 address of the remote's host in its lxc URL.
remotes = json.loads(subprocess.run(['lxc', 'remote', 'list', '--format', 'json'], capture_output=True,
text=True, check=True).stdout)
host = urllib.parse.urlsplit(remotes.get(args.remote, {}).get('Addr', '')).hostname
try:
args.listen = socket.getaddrinfo(host, None, socket.AF_INET)[0][4][0] if host else None
except socket.gaierror:
pass
if not args.listen:
parser.error(f'no IPv4 address of the host of remote {args.remote}; give --listen or LXD_LISTEN')
where = f'{args.remote}:{args.name} (project {args.project})'
exists = subprocess.run(['lxc', 'info', f'{args.remote}:{args.name}', '--project', args.project],
capture_output=True).returncode == 0
env = ROOT / 'tmp' / f'vm-{args.name}.env'
if args.remove:
if not exists:
sys.exit(f'no VM {where}')
subprocess.run(['lxc', 'delete', '--force', f'{args.remote}:{args.name}', '--project', args.project],
check=True)
env.unlink(missing_ok=True)
print(f'removed {where}')
return
if exists:
sys.exit(f'{where} exists; remove it with --remove or choose another --name')
config = HERE / 'configs' / args.config
if not config.is_file():
parser.error(f'no configs/{args.config}')
debs = sorted((ROOT / 'out').glob('linux-manager_*_all.deb'))
if not debs:
sys.exit('no out/linux-manager_*_all.deb; run ./build.sh first')
password, psk = secrets.token_hex(16), secrets.token_hex(24)
env.parent.mkdir(parents=True, exist_ok=True)
env.touch(mode=0o600)
env.chmod(0o600)
env.write_text(f'ADMIN_USERNAME=admin\nADMIN_PASSWORD={password}\nPSK={psk}\n', encoding='utf-8')
print(f'launching {where} from {args.image} with {debs[-1].name}; proxies listen on {args.listen}')
subprocess.run(['lxc', 'launch', args.image, f'{args.remote}:{args.name}', '--vm', '-c', 'limits.cpu=2',
'-c', 'limits.memory=4GiB',
'-c', 'cloud-init.user-data=' + (HERE / 'cloud-init' / 'server.yaml').read_text(encoding='utf-8'),
'--project', args.project], check=True)
# Wait for the agent and cloud-init (status 2 is done with recoverable errors), then for the default route.
for _ in range(120):
if inside(args, 'cloud-init', 'status', '--wait').returncode in (0, 2):
break
time.sleep(5)
else:
sys.exit(f'{args.name} did not boot; remove it with --remove')
for _ in range(24):
route = re.search(r'\bdev (\S+)', inside(args, 'ip', '-4', '-o', 'route', 'show', 'default').stdout)
address = route and re.search(r' inet (\S+) ',
inside(args, 'ip', '-4', '-o', 'addr', 'show', 'dev', route.group(1)).stdout)
if address:
break
time.sleep(5)
else:
sys.exit(f'{args.name} has no default route or address; remove it with --remove')
nic, interface = route.group(1), ipaddress.IPv4Interface(address.group(1))
# Pin the leased address on the profile's NIC and proxy the host's --listen ports to the VM.
mac = inside(args, 'cat', f'/sys/class/net/{nic}/address').stdout.strip()
expanded = subprocess.run(['lxc', 'config', 'show', '--expanded', f'{args.remote}:{args.name}',
'--project', args.project], capture_output=True, text=True, check=True).stdout
device = next((m.group(1) for m in re.finditer(r'^\s*volatile\.(\S+)\.hwaddr: "?([0-9a-f:]+)"?$', expanded, re.M)
if m.group(2) == mac), None)
if not device:
sys.exit(f'no LXD device of {args.name} has the MAC address {mac} of {nic}')
subprocess.run(['lxc', 'config', 'device', 'override', f'{args.remote}:{args.name}', device,
f'ipv4.address={interface.ip}', '--project', args.project], check=True)
for host_port, vm_port in PROXIES:
subprocess.run(['lxc', 'config', 'device', 'add', f'{args.remote}:{args.name}', f'proxy-{vm_port}', 'proxy',
f'listen=tcp:{args.listen}:{host_port}', f'connect=tcp:{interface.ip}:{vm_port}', 'nat=true',
'--project', args.project], check=True)
# Install the package with the test administrator.
if not push(args, debs[-1], '/tmp/linux-manager.deb', '0644'):
sys.exit(f'could not copy {debs[-1].name} into {args.name}')
preseed = (f'linux-manager linux-manager/admin_username string admin\n'
f'linux-manager linux-manager/admin_password password {password}\n')
if inside(args, 'debconf-set-selections', input=preseed).returncode != 0:
sys.exit('debconf-set-selections failed')
result = inside(args, 'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '/tmp/linux-manager.deb')
log = ROOT / 'tmp' / f'vm-{args.name}-install.log'
log.write_text(result.stdout + result.stderr, encoding='utf-8')
if result.returncode != 0:
sys.exit(f'the package did not install; see {log}')
# Apply the configuration twice, as the server's certificates follow the first run.
text = (config.read_text(encoding='utf-8').replace('@ETH0@', nic).replace('@CIDR0@', str(interface.network))
.replace('@PSK@', psk))
rendered = ROOT / 'tmp' / f'vm-{args.name}.json'
rendered.touch(mode=0o600)
rendered.chmod(0o600)
rendered.write_text(text, encoding='utf-8')
pushed = push(args, rendered, '/etc/linux-manager/linux-manager.json', '0600')
rendered.unlink()
if not pushed:
sys.exit(f'could not copy {args.config} into {args.name}')
for _ in range(2):
if inside(args, 'systemctl', 'restart', 'linux-manager-setup').returncode != 0:
sys.exit(f'linux-manager-setup failed with {args.config}; see journalctl -u linux-manager-setup in the VM')
print(f'{args.name} is up at {interface.ip} with {args.config}')
print(f'administrator: admin password: {password} (also in {env})')
print(f'manager: http://{args.listen}:7080/ status: http://{args.listen}:7081/ ssh: {args.listen} port 2222')
print(f'shell: lxc exec {args.remote}:{args.name} --project {args.project} -- su -l')
print(f'remove: {sys.argv[0]} --remote {args.remote} --project {args.project} --name {args.name} --remove')
if __name__ == '__main__':
main()
+1 -1
View File
@@ -7,7 +7,7 @@ import sys
sys.path.insert(0, '/usr/lib/linux-manager')
import linuxmanager_users as users # noqa: E402
import linuxmanager_users as users
public_key, address = sys.argv[1:3]
with users.session() as connection: