feature: the router serves S3-compatible object storage with versitygw, with one S3 account per user and buckets owned by users.

debian-package/src/usr/lib/vpn-router/vpnrouter_s3.py

@@ -0,0 +1,446 @@
+"""The router's S3-compatible object storage, served by versitygw, for setup, vpn-router-s3 and the web service.
+
+versitygw runs as vpn-router-s3.service with its POSIX backend: each folder of DATA_DIR is a bucket and the files
+below it are its objects. Its internal IAM in IAM_DIR holds an account per access key; the root keys in ROOT_ENV are
+not stored there. Its S3 API listens on S3_PORT and its Admin API on ADMIN_PORT, both on the loopback address; Caddy
+serves the S3 API at https://<wan.local_fqdn>:<s3.port>.
+
+Each user or service account of the router's directory has at most one S3 account, an access key and its secret,
+recorded in ACCOUNTS with the time it was made. The secret is shown once, when the account is made or its secret
+replaced; ACCOUNTS keeps it, as the IAM does, so the account can be switched off and on with its user (sync). A
+bucket belongs to one S3 account, which alone may use it; the root keys reach every bucket.
+
+The Admin API and the S3 API are called with AWS Signature Version 4, signed with the root keys for the region
+s3.region. A request that cannot be done raises Failure, whose status says why: 2 invalid, 3 no such user, account or
+bucket, 4 it exists or the state does not allow it, 1 other.
+"""
+
+import contextlib
+import datetime
+import fcntl
+import hashlib
+import hmac
+import json
+import os
+import pathlib
+import re
+import secrets
+import string
+import subprocess
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ElementTree
+
+import ldap
+
+import vpnrouter
+import vpnrouter_users
+
+#: The pinned release of versitygw: its version and the SHA-256 of its .deb for each Debian architecture.
+VERSION = '1.10.0'
+SHA256 = {'amd64': 'd8a39e9cc7da6f17467e60795cb703cfcb29b3fe5db8227e341d90bc64727140',
+          'arm64': '5f81e0dec09d8f1f8ca3159de209543ceb03714dc0066cf7d353b8ef064a0136'}
+DEB_URL = 'https://github.com/versity/versitygw/releases/download/v{version}/versitygw_{version}_linux_{arch}.deb'
+
+#: The versitygw binary the .deb installs.
+BINARY = pathlib.Path('/usr/bin/versitygw')
+
+UNIT = 'vpn-router-s3.service'
+INSTALL_UNIT = 'vpn-router-install@s3.service'
+
+#: The buckets, one folder each, and the accounts of versitygw's internal IAM.
+S3_DIR = pathlib.Path('/var/lib/vpn-router/s3')
+DATA_DIR = S3_DIR / 'data'
+IAM_DIR = S3_DIR / 'iam'
+
+#: The environment of vpn-router-s3.service, which setup writes: the root keys, the ports, the IAM and the region.
+ROOT_ENV = pathlib.Path('/etc/vpn-router/s3.env')
+
+#: The S3 account of each user: name -> {"access_key", "secret_key", "created"}.
+ACCOUNTS = pathlib.Path('/etc/vpn-router/s3-accounts.json')
+ACCOUNTS_LOCK = pathlib.Path('/run/vpn-router/s3-accounts.lock')
+
+S3_PORT = 7448
+ADMIN_PORT = 7449
+
+#: The folder versitygw keeps the parts of multipart uploads in, inside each bucket.
+TEMPORARY = '.sgwtmp'
+
+#: A bucket name versitygw and S3 accept: 3 to 63 lowercase letters, digits, dots and hyphens, from and to a letter or
+#: a digit, and not an IPv4 address.
+BUCKET_NAME = re.compile(r'^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$')
+
+#: Seconds a call of versitygw may take.
+TIMEOUT = 30
+
+#: The characters of the access keys and the secrets made here.
+KEY_CHARACTERS = string.ascii_uppercase + string.digits
+SECRET_CHARACTERS = string.ascii_letters + string.digits
+
+
+class Failure(Exception):
+    """A request that cannot be done; status is 2 invalid, 3 no such user, account or bucket, 4 exists or not allowed,
+    1 other."""
+
+    def __init__(self, message, status=1):
+        super().__init__(message)
+        self.status = status
+
+
+def architecture():
+    """The Debian architecture of this host, such as amd64."""
+    return subprocess.run(['dpkg', '--print-architecture'], capture_output=True, text=True, check=True).stdout.strip()
+
+
+def installed():
+    """The installed version of versitygw, or None when it is not installed."""
+    result = subprocess.run(['dpkg-query', '-W', '-f=${Status}\t${Version}', 'versitygw'], capture_output=True, text=True)
+    status, _, version = result.stdout.partition('\t')
+    return version if result.returncode == 0 and status == 'install ok installed' else None
+
+
+def running():
+    """Whether vpn-router-s3.service is active."""
+    result = subprocess.run(['systemctl', 'is-active', UNIT], capture_output=True, text=True)
+    return result.stdout.strip() == 'active'
+
+
+def read_env():
+    """The settings of ROOT_ENV, {} while it does not exist."""
+    try:
+        text = ROOT_ENV.read_text(encoding='utf-8')
+    except FileNotFoundError:
+        return {}
+    return dict(line.split('=', 1) for line in text.splitlines() if '=' in line and not line.startswith('#'))
+
+
+def write_env(region):
+    """Write ROOT_ENV with the root keys it has, or new ones, and the region; whether it changed."""
+    current = read_env()
+    values = {'ROOT_ACCESS_KEY_ID': current.get('ROOT_ACCESS_KEY_ID') or 'VRROOT' + random_text(KEY_CHARACTERS, 14),
+              'ROOT_SECRET_ACCESS_KEY': current.get('ROOT_SECRET_ACCESS_KEY') or random_text(SECRET_CHARACTERS, 40),
+              'VGW_PORT': f'{vpnrouter.LOOPBACK}:{S3_PORT}', 'VGW_ADMIN_PORT': f'{vpnrouter.LOOPBACK}:{ADMIN_PORT}',
+              'VGW_IAM_DIR': str(IAM_DIR), 'VGW_REGION': region}
+    if values == current:
+        return False
+    ROOT_ENV.parent.mkdir(parents=True, exist_ok=True)
+    temporary = ROOT_ENV.with_name(ROOT_ENV.name + '.new')
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(descriptor, 'w', encoding='utf-8') as handle:
+        handle.write('# Written by vpn-router-setup for vpn-router-s3.service; edits are overwritten.\n')
+        handle.writelines(f'{key}={value}\n' for key, value in values.items())
+    os.replace(temporary, ROOT_ENV)
+    return True
+
+
+def random_text(characters, length):
+    return ''.join(secrets.choice(characters) for _ in range(length))
+
+
+def ensure_folders():
+    """Make DATA_DIR and IAM_DIR, readable by root only."""
+    for path in (S3_DIR, DATA_DIR, IAM_DIR):
+        path.mkdir(mode=0o700, parents=True, exist_ok=True)
+
+
+@contextlib.contextmanager
+def accounts_lock():
+    """Hold the lock that serializes the changes of ACCOUNTS."""
+    ACCOUNTS_LOCK.parent.mkdir(parents=True, exist_ok=True)
+    with open(ACCOUNTS_LOCK, 'w') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        yield
+
+
+def accounts():
+    """The S3 accounts of ACCOUNTS by user name, {} while it does not exist."""
+    try:
+        data = json.loads(ACCOUNTS.read_text(encoding='utf-8'))
+    except FileNotFoundError:
+        return {}
+    except ValueError as exc:
+        raise Failure(f'{ACCOUNTS} cannot be read: {exc}')
+    return data if isinstance(data, dict) else {}
+
+
+def save_accounts(data):
+    """Write ACCOUNTS by a rename, readable by root only."""
+    ACCOUNTS.parent.mkdir(parents=True, exist_ok=True)
+    temporary = ACCOUNTS.with_name(ACCOUNTS.name + '.new')
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(descriptor, 'w', encoding='utf-8') as handle:
+        json.dump(data, handle, indent=2)
+        handle.write('\n')
+    os.replace(temporary, ACCOUNTS)
+
+
+def uri_encode(text, slash=True):
+    """Text encoded as Signature Version 4 encodes it: every byte but A-Z, a-z, 0-9, -, ., _ and ~, and / when slash."""
+    return urllib.parse.quote(text, safe='-._~' + ('' if slash else '/'))
+
+
+def signed_headers(method, host, path, query, body, headers, access, secret, region, now):
+    """The headers of a request signed with AWS Signature Version 4 for the service s3 at the UTC time now: host,
+    x-amz-content-sha256, x-amz-date and the headers given, all signed, and Authorization."""
+    stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
+    digest = hashlib.sha256(body).hexdigest()
+    signed = {'host': host, 'x-amz-content-sha256': digest, 'x-amz-date': stamp,
+              **{key.lower(): value for key, value in headers.items()}}
+    names = sorted(signed)
+    canonical = '\n'.join([method, uri_encode(path, slash=False),
+                           '&'.join(f'{uri_encode(key)}={uri_encode(value)}' for key, value in sorted(query.items())),
+                           ''.join(f'{name}:{" ".join(str(signed[name]).split())}\n' for name in names),
+                           ';'.join(names), digest])
+    scope = f'{day}/{region}/s3/aws4_request'
+    to_sign = '\n'.join(['AWS4-HMAC-SHA256', stamp, scope, hashlib.sha256(canonical.encode()).hexdigest()])
+    key = ('AWS4' + secret).encode()
+    for part in (day, region, 's3', 'aws4_request'):
+        key = hmac.new(key, part.encode(), hashlib.sha256).digest()
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    return {**signed, 'Authorization': f'AWS4-HMAC-SHA256 Credential={access}/{scope}, '
+                                       f'SignedHeaders={";".join(names)}, Signature={signature}'}
+
+
+def call(method, port, path, query=None, body=b'', headers=None):
+    """Make a request of versitygw on the port, signed with the root keys; the status and the body of the answer.
+
+    Failure with versitygw's message when it answers an error, and when it cannot be reached.
+    """
+    env = read_env()
+    access, secret, region = env.get('ROOT_ACCESS_KEY_ID'), env.get('ROOT_SECRET_ACCESS_KEY'), env.get('VGW_REGION')
+    if not access or not secret or not region:
+        raise Failure('the object storage is not set up; setup writes its keys once s3.enabled is on', 4)
+    host = f'{vpnrouter.LOOPBACK}:{port}'
+    signed = signed_headers(method, host, path, query or {}, body, headers or {}, access, secret, region,
+                            datetime.datetime.now(datetime.timezone.utc))
+    canonical_query = '&'.join(f'{uri_encode(key)}={uri_encode(value)}' for key, value in sorted((query or {}).items()))
+    url = f'http://{host}{uri_encode(path, slash=False)}' + (f'?{canonical_query}' if canonical_query else '')
+    request = urllib.request.Request(url, data=body if body else None, method=method,
+                                     headers={key: value for key, value in signed.items() if key != 'host'})
+    try:
+        with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
+            return response.status, response.read()
+    except urllib.error.HTTPError as exc:
+        answer = exc.read()
+        code, message = error_of(answer)
+        status = 3 if exc.code == 404 or code in ('NoSuchBucket', 'XAdminUserNotFound') else \
+            4 if exc.code == 409 or code in ('XAdminUserExists', 'BucketAlreadyExists', 'BucketNotEmpty') else \
+            2 if exc.code == 400 else 1
+        raise Failure(f'versitygw refused {method} {path}: {message or code or exc.reason}', status)
+    except OSError as exc:
+        raise Failure(f'versitygw cannot be reached on {host}: {exc}; is {UNIT} running?')
+
+
+def local_name(element):
+    return element.tag.rsplit('}', 1)[-1]
+
+
+def error_of(body):
+    """The code and the message of an S3 or Admin API error document, empty strings when there is none."""
+    try:
+        root = ElementTree.fromstring(body)
+    except ElementTree.ParseError:
+        return '', body.decode(errors='replace').strip()
+    fields = {local_name(child): (child.text or '').strip() for child in root}
+    return fields.get('Code', ''), fields.get('Message', '')
+
+
+def admin(path, query=None, body=b''):
+    """Call the Admin API: PATCH of the path; the body of the answer."""
+    headers = {'Content-Type': 'application/xml'} if body else None
+    return call('PATCH', ADMIN_PORT, path, query, body, headers)[1]
+
+
+def account_xml(root, **fields):
+    """An XML document of the root element with the fields as its children."""
+    element = ElementTree.Element(root)
+    for name, value in fields.items():
+        ElementTree.SubElement(element, name).text = value
+    return ElementTree.tostring(element)
+
+
+def user_state(connection, name):
+    """Whether the user or service account exists and is enabled: None when it does not exist, else True or False."""
+    try:
+        return vpnrouter_users.summary(vpnrouter_users.entry(connection, name))['enabled']
+    except vpnrouter_users.Failure as failure:
+        if failure.status == 3:
+            return None
+        raise
+
+
+def shown(name, account):
+    """An account as the answers show it, without its secret."""
+    return {'owner': name, 'access_key': account['access_key'], 'created': account['created']}
+
+
+def list_accounts(owner=None):
+    """The accounts, each {owner, access_key, created}, sorted by owner; with owner, only that user's."""
+    return [shown(name, account) for name, account in sorted(accounts().items()) if owner in (None, name)]
+
+
+def account_add(name):
+    """Make the S3 account of the user, who must exist and have none: {owner, access_key, secret_key, created}."""
+    with vpnrouter_users.session() as connection:
+        if user_state(connection, name) is None:
+            raise Failure(f'user {name} does not exist', 3)
+    with accounts_lock():
+        data = accounts()
+        if name in data:
+            raise Failure(f'{name} has an S3 account, {data[name]["access_key"]}; replace its secret or remove it', 4)
+        account = {'access_key': 'VR' + random_text(KEY_CHARACTERS, 18),
+                   'secret_key': random_text(SECRET_CHARACTERS, 40),
+                   'created': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}
+        admin('/create-user', body=account_xml('Account', Access=account['access_key'], Secret=account['secret_key'],
+                                               Role='user'))
+        data[name] = account
+        save_accounts(data)
+    return {**shown(name, account), 'secret_key': account['secret_key']}
+
+
+def account_rotate(name):
+    """Replace the secret of the user's S3 account: {owner, access_key, secret_key, created}."""
+    with accounts_lock():
+        data = accounts()
+        if name not in data:
+            raise Failure(f'{name} has no S3 account', 3)
+        account = dict(data[name], secret_key=random_text(SECRET_CHARACTERS, 40))
+        admin('/update-user', {'access': account['access_key']},
+              account_xml('MutableProps', Secret=account['secret_key']))
+        data[name] = account
+        save_accounts(data)
+    sync(name)
+    return {**shown(name, account), 'secret_key': account['secret_key']}
+
+
+def account_remove(name):
+    """Remove the user's S3 account; its buckets keep it as their owner until another is chosen."""
+    with accounts_lock():
+        data = accounts()
+        if name not in data:
+            raise Failure(f'{name} has no S3 account', 3)
+        try:
+            admin('/delete-user', {'access': data[name]['access_key']})
+        except Failure as failure:
+            if failure.status != 3:
+                raise
+        account = data.pop(name)
+        save_accounts(data)
+    return {**shown(name, account), 'removed': True}
+
+
+def sync(name):
+    """Bring the user's S3 account in line with the user: removed with a user that no longer exists, its secret
+    replaced by an unknown one while the user is disabled, and its own secret while it is enabled. Nothing for a user
+    without an account, or while versitygw does not run."""
+    if name not in accounts() or not running():
+        return
+    with vpnrouter_users.session() as connection:
+        state = user_state(connection, name)
+    if state is None:
+        account_remove(name)
+        return
+    secret = accounts()[name]['secret_key'] if state else random_text(SECRET_CHARACTERS, 40)
+    admin('/update-user', {'access': accounts()[name]['access_key']}, account_xml('MutableProps', Secret=secret))
+
+
+#: The vpnrouter_users operations after which the user's S3 account follows the user (follow).
+FOLLOWING = ('disable', 'enable', 'remove')
+
+
+def follow(name):
+    """sync() the user's account after a vpnrouter_users operation of FOLLOWING; why it failed, or None. A failure
+    leaves the account as it is until setup's next run synchronizes it."""
+    try:
+        sync(name)
+    except (Failure, vpnrouter_users.Failure, ldap.LDAPError, OSError) as exc:
+        return f'the S3 account of {name} cannot follow the user: {exc}; setup synchronizes it at its next run'
+    return None
+
+
+def sync_all():
+    """sync() each account, as setup does once versitygw runs; the accounts versitygw lacks, as after a restore of the
+    configuration only, are made again. The failures are answered as messages."""
+    problems = []
+    try:
+        root = ElementTree.fromstring(admin('/list-users'))
+        known = {(child.findtext('Access') or '').strip() for child in root if local_name(child) == 'Account'}
+        for name, account in accounts().items():
+            if account['access_key'] not in known:
+                admin('/create-user', body=account_xml('Account', Access=account['access_key'],
+                                                       Secret=account['secret_key'], Role='user'))
+    except (Failure, ElementTree.ParseError) as exc:
+        return [f'the S3 accounts cannot be checked: {exc}']
+    for name in list(accounts()):
+        try:
+            sync(name)
+        except (Failure, vpnrouter_users.Failure, ldap.LDAPError, OSError) as exc:
+            problems.append(f'the S3 account of {name} cannot be synchronized: {exc}')
+    return problems
+
+
+def check_bucket(name):
+    if not isinstance(name, str) or not BUCKET_NAME.match(name) or '..' in name or \
+            re.fullmatch(r'\d+\.\d+\.\d+\.\d+', name):
+        raise Failure(f'{name!r} is not a bucket name: 3 to 63 lowercase letters, digits, dots and hyphens, from and to '
+                      'a letter or a digit', 2)
+
+
+def usage(bucket):
+    """The size in bytes and the number of objects of the bucket, from its folder, without the uploads in progress."""
+    size = count = 0
+    for folder, folders, files in os.walk(DATA_DIR / bucket):
+        if folder == str(DATA_DIR / bucket):
+            folders[:] = [item for item in folders if item != TEMPORARY]
+        for item in files:
+            try:
+                size += os.lstat(os.path.join(folder, item)).st_size
+                count += 1
+            except OSError:
+                pass
+    return size, count
+
+
+def buckets():
+    """The buckets, each {name, owner, access_key, size, objects}, sorted by name; owner is the user of the account
+    that owns it, None for an account no user has."""
+    root = ElementTree.fromstring(admin('/list-buckets'))
+    owners = {account['access_key']: name for name, account in accounts().items()}
+    found = []
+    for child in root:
+        if local_name(child) != 'Bucket':
+            continue
+        name, access = (child.findtext('Name') or '').strip(), (child.findtext('Owner') or '').strip()
+        size, count = usage(name)
+        found.append({'name': name, 'owner': owners.get(access), 'access_key': access, 'size': size, 'objects': count})
+    return sorted(found, key=lambda bucket: bucket['name'])
+
+
+def owner_key(owner):
+    """The access key of the S3 account of the user; Failure when the user has none."""
+    account = accounts().get(owner)
+    if account is None:
+        raise Failure(f'{owner} has no S3 account; make one first', 3)
+    return account['access_key']
+
+
+def bucket_add(name, owner):
+    """Make the bucket, owned by the user's S3 account."""
+    check_bucket(name)
+    call('PATCH', ADMIN_PORT, f'/{name}/create', headers={'x-vgw-owner': owner_key(owner)})
+    return {'name': name, 'owner': owner}
+
+
+def bucket_owner(name, owner):
+    """Give the bucket to the user's S3 account; versitygw replaces its ACL and policy with the default ones."""
+    check_bucket(name)
+    admin('/change-bucket-owner', {'bucket': name, 'owner': owner_key(owner)})
+    return {'name': name, 'owner': owner}
+
+
+def bucket_remove(name):
+    """Remove the bucket, which must be empty."""
+    check_bucket(name)
+    call('DELETE', S3_PORT, f'/{name}')
+    return {'name': name, 'removed': True}