feature: the tfvars store keeps Terraform modules, their instances and the instances' variable files in PostgreSQL.

server/index.ts

@@ -4,3 +4,6 @@ export { HttpError, errorHandler, notFoundHandler, strict } from './http.ts'
 export { QUIET, createLog, fail, timestampsOn, type Log } from './log.ts'
 export { exitOnSignals, gitVersion, readJsonObject } from './process.ts'
 export { decrypt, encrypt, encryptionKey } from './secrets.ts'
+export type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
+export { STATE_SCHEMA_PATTERN, TFVARS_FILE_PATTERN, TF_NAME_PATTERN } from '../shared/tfvars.ts'
+export { createTfvarsStore, tfvarsText, versionOf, type InstanceVariables, type TfvarsDocument, type TfvarsFile, type TfvarsStore } from './tfvarsStore.ts'

server/sql/tfvars_store.sql

@@ -0,0 +1,117 @@
+-- The store of Terraform modules, their instances and the instances' variable files: the tables tf_modules,
+-- tf_instances and tf_variable_files, and the functions and procedures the store calls. The store applies the file on
+-- its first use in the schema its connection's search path names; every statement can run again.
+
+CREATE TABLE IF NOT EXISTS tf_modules (
+  name             text PRIMARY KEY CHECK (name ~ '^[a-z][a-z0-9-]*$'),
+  root             text NOT NULL CHECK (root <> ''),
+  description      text NOT NULL DEFAULT '',
+  configurator_url text
+);
+
+CREATE TABLE IF NOT EXISTS tf_instances (
+  module       text NOT NULL REFERENCES tf_modules (name),
+  name         text NOT NULL CHECK (name ~ '^[a-z][a-z0-9-]*$'),
+  state_schema text NOT NULL UNIQUE CHECK (state_schema ~ '^[a-z_][a-z0-9_]*$'),
+  created      timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (module, name)
+);
+
+CREATE TABLE IF NOT EXISTS tf_variable_files (
+  module   text NOT NULL,
+  instance text NOT NULL,
+  name     text NOT NULL CHECK (name ~ '^[A-Za-z0-9_.-]+\.tfvars\.json$'),
+  content  jsonb NOT NULL CHECK (jsonb_typeof(content) = 'object'),
+  previous jsonb CHECK (jsonb_typeof(previous) = 'object'),
+  updated  timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (module, instance, name),
+  FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE
+);
+
+-- The modules, by name.
+CREATE OR REPLACE FUNCTION tf_modules_list()
+RETURNS TABLE (name text, root text, description text, configurator_url text)
+LANGUAGE sql STABLE
+SET search_path FROM CURRENT
+AS $$
+  SELECT m.name, m.root, m.description, m.configurator_url FROM tf_modules m ORDER BY m.name
+$$;
+
+-- Creates a module, or changes the one of that name.
+CREATE OR REPLACE PROCEDURE tf_module_put(module_name text, module_root text, module_description text, module_configurator_url text)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  INSERT INTO tf_modules (name, root, description, configurator_url)
+  VALUES (module_name, module_root, module_description, module_configurator_url)
+  ON CONFLICT (name) DO UPDATE SET root = excluded.root, description = excluded.description, configurator_url = excluded.configurator_url
+$$;
+
+-- Deletes a module; the foreign key of tf_instances refuses it while the module has instances.
+CREATE OR REPLACE PROCEDURE tf_module_delete(module_name text)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  DELETE FROM tf_modules WHERE name = module_name
+$$;
+
+-- The instances of a module, by name.
+CREATE OR REPLACE FUNCTION tf_instances_list(module_name text)
+RETURNS TABLE (module text, name text, state_schema text, created timestamptz)
+LANGUAGE sql STABLE
+SET search_path FROM CURRENT
+AS $$
+  SELECT i.module, i.name, i.state_schema, i.created FROM tf_instances i WHERE i.module = module_name ORDER BY i.name
+$$;
+
+-- Creates an instance of a module with the schema of its state.
+CREATE OR REPLACE PROCEDURE tf_instance_create(module_name text, instance_name text, instance_state_schema text)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  INSERT INTO tf_instances (module, name, state_schema) VALUES (module_name, instance_name, instance_state_schema)
+$$;
+
+-- Deletes an instance with its variable files.
+CREATE OR REPLACE PROCEDURE tf_instance_delete(module_name text, instance_name text)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  DELETE FROM tf_instances WHERE module = module_name AND name = instance_name
+$$;
+
+-- The variable files of an instance, by name: content, whether the content it replaced is kept, and the time of the
+-- last write.
+CREATE OR REPLACE FUNCTION tf_variable_files_read(module_name text, instance_name text)
+RETURNS TABLE (name text, content jsonb, previous boolean, updated timestamptz)
+LANGUAGE sql STABLE
+SET search_path FROM CURRENT
+AS $$
+  SELECT f.name, f.content, f.previous IS NOT NULL, f.updated FROM tf_variable_files f
+  WHERE f.module = module_name AND f.instance = instance_name ORDER BY f.name
+$$;
+
+-- Keeps other writers out until the end of the transaction, from the comparison of the version to the commit.
+CREATE OR REPLACE PROCEDURE tf_variable_files_lock()
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  LOCK TABLE tf_variable_files IN EXCLUSIVE MODE
+$$;
+
+-- Writes a variable file of an instance, keeping the content it replaces as previous.
+CREATE OR REPLACE PROCEDURE tf_variable_file_put(module_name text, instance_name text, file_name text, file_content jsonb)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  INSERT INTO tf_variable_files (module, instance, name, content) VALUES (module_name, instance_name, file_name, file_content)
+  ON CONFLICT (module, instance, name) DO UPDATE SET previous = tf_variable_files.content, content = excluded.content, updated = now()
+$$;
+
+-- Deletes a variable file of an instance.
+CREATE OR REPLACE PROCEDURE tf_variable_file_delete(module_name text, instance_name text, file_name text)
+LANGUAGE sql
+SET search_path FROM CURRENT
+AS $$
+  DELETE FROM tf_variable_files WHERE module = module_name AND instance = instance_name AND name = file_name
+$$;

server/tfvarsStore.test.ts

@@ -0,0 +1,95 @@
+import assert from 'node:assert/strict'
+import { after, before, test } from 'node:test'
+import pg from 'pg'
+import { HttpError } from './http.ts'
+import { createTfvarsStore, tfvarsText, type TfvarsStore } from './tfvarsStore.ts'
+
+// The tests create a scratch database on the server the PG* variables name, connected to through the database
+// postgres, and drop it at the end.
+const database = `tfvars_store_test_${process.pid}`
+const admin = new pg.Client({ database: 'postgres' })
+let pool: pg.Pool
+let store: TfvarsStore
+
+before(async () => {
+  await admin.connect()
+  await admin.query(`CREATE DATABASE ${database}`)
+  pool = new pg.Pool({ database })
+  store = createTfvarsStore(pool)
+})
+
+after(async () => {
+  await pool?.end()
+  await admin.query(`DROP DATABASE IF EXISTS ${database}`)
+  await admin.end()
+})
+
+const status = async (promise: Promise<unknown>) => {
+  try {
+    await promise
+  } catch (error) {
+    assert.ok(error instanceof HttpError, String(error))
+    return error.statusCode
+  }
+  return 0
+}
+
+test('modules are created, changed, listed and deleted', async () => {
+  await store.modules.put({ name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: null })
+  await store.modules.put({ name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: 'http://localhost:7083/' })
+  await store.modules.put({ name: 'app', root: 'app', description: '', configuratorUrl: null })
+  assert.deepEqual(await store.modules.list(), [
+    { name: 'app', root: 'app', description: '', configuratorUrl: null },
+    { name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: 'http://localhost:7083/' },
+  ])
+  await store.modules.delete('app')
+  assert.deepEqual((await store.modules.list()).map((module) => module.name), ['router'])
+  assert.equal(await status(store.modules.put({ name: 'Bad', root: 'x', description: '', configuratorUrl: null })), 400)
+})
+
+test('instances are created and refused for a missing module or a name in use', async () => {
+  await store.instances.create('router', 'platform', 'terraform_remote_state')
+  await store.instances.create('router', 'test', 'tf_router_test')
+  assert.deepEqual((await store.instances.list('router')).map((instance) => [instance.name, instance.stateSchema]), [
+    ['platform', 'terraform_remote_state'],
+    ['test', 'tf_router_test'],
+  ])
+  assert.equal(await status(store.instances.create('missing', 'x', 'tf_missing_x')), 404)
+  assert.equal(await status(store.instances.create('router', 'test', 'tf_other')), 409)
+  assert.equal(await status(store.instances.create('router', 'other', 'tf_router_test')), 409)
+  assert.equal(await status(store.modules.delete('router')), 409)
+})
+
+test('variable files are written with their version and refused against another version', async () => {
+  const variables = store.variables('router', 'test')
+  const empty = await variables.read()
+  assert.deepEqual(empty.files, [])
+  assert.equal(empty.version, '')
+  assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }), true)
+  const written = await variables.read()
+  assert.deepEqual(written.files.map((file) => [file.name, file.content.toString('utf8')]), [
+    ['platform.tfvars.json', tfvarsText({ a: 1 })],
+    ['secrets.tfvars.json', tfvarsText({ s: 'x' })],
+  ])
+  assert.deepEqual(written.status.map((file) => file.previous), [false, false])
+  assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), false)
+  assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), true)
+  const changed = await variables.read()
+  assert.deepEqual(changed.status.map((file) => file.previous), [true, false])
+  assert.notEqual(changed.version, written.version)
+  assert.equal(await variables.remove(written.version, 'secrets.tfvars.json'), false)
+  assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json'), true)
+  assert.deepEqual((await variables.read()).files.map((file) => file.name), ['platform.tfvars.json'])
+  assert.deepEqual((await store.variables('router', 'platform').read()).files, [])
+  assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) })), 400)
+  assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) })), 404)
+})
+
+test('an instance is deleted with its variable files, then its module', async () => {
+  await store.instances.delete('router', 'test')
+  await store.instances.delete('router', 'platform')
+  assert.deepEqual(await store.instances.list('router'), [])
+  assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_files')).rows[0].n, 0)
+  await store.modules.delete('router')
+  assert.deepEqual(await store.modules.list(), [])
+})

server/tfvarsStore.ts

@@ -0,0 +1,158 @@
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import type { Pool, PoolClient } from 'pg'
+import type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
+import { HttpError } from './http.ts'
+
+/** A variable file of an instance: its name and its content, the object jsonb keeps written as JSON text. */
+export type TfvarsFile = { name: string; content: Buffer }
+
+/** The variable files of an instance, by name, with their version, versionOf the files. */
+export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string }
+
+/** The variable files of one instance. */
+export type InstanceVariables = {
+  read(): Promise<TfvarsDocument>
+  /**
+   * Writes the given files, name and JSON text of an object, when the files are still those of version, keeping each
+   * replaced content as its previous one; answers false, writing nothing, when they changed since.
+   */
+  write(version: string, contents: Record<string, string>): Promise<boolean>
+  /** Deletes the file name when the files are still those of version; answers false, deleting nothing, otherwise. */
+  remove(version: string, name: string): Promise<boolean>
+}
+
+/** The modules, the instances and their variable files in PostgreSQL. */
+export type TfvarsStore = {
+  modules: {
+    list(): Promise<TfModule[]>
+    /** Creates the module, or changes the one of its name. */
+    put(module: TfModule): Promise<void>
+    /** Deletes the module; HttpError 409 while it has instances. */
+    delete(name: string): Promise<void>
+  }
+  instances: {
+    list(module: string): Promise<TfInstance[]>
+    /** HttpError 409 for a name or a state schema in use, 404 for a module that does not exist. */
+    create(module: string, name: string, stateSchema: string): Promise<void>
+    /** Deletes the instance with its variable files. */
+    delete(module: string, name: string): Promise<void>
+  }
+  variables(module: string, instance: string): InstanceVariables
+}
+
+/** The text of a variable file as both stores write it: the object as JSON with two spaces, and a final newline. */
+export function tfvarsText(content: unknown): string {
+  return `${JSON.stringify(content, null, 2)}\n`
+}
+
+/** Identifies the variable files: the name and the SHA-256 of the content of each, in name order. */
+export function versionOf(files: TfvarsFile[]): string {
+  return [...files].sort((a, b) => a.name.localeCompare(b.name))
+    .map((file) => `${file.name}=${createHash('sha256').update(file.content).digest('hex')}`).join(':')
+}
+
+const SQL = new URL('./sql/tfvars_store.sql', import.meta.url)
+
+type FileRow = { name: string; content: object; previous: boolean; updated: Date }
+
+/**
+ * The store of sql/tfvars_store.sql in the database of pool, applied at the first use in the schema of the
+ * connection's search path. A refused change is an HttpError: 409 for a name in use or a module that has instances,
+ * 404 for a module or an instance that does not exist, 400 for a value the tables do not accept.
+ */
+export function createTfvarsStore(pool: Pool): TfvarsStore {
+  let created: Promise<unknown> | null = null
+  const ready = () => (created ??= pool.query(readFileSync(SQL, 'utf8')))
+  const query = async <T extends object>(text: string, values: unknown[]): Promise<T[]> => {
+    await ready()
+    try {
+      return (await pool.query<T>(text, values)).rows
+    } catch (error) {
+      throw refused(error)
+    }
+  }
+  const document = (rows: FileRow[]): TfvarsDocument => {
+    const files = rows.map((row) => ({ name: row.name, content: Buffer.from(tfvarsText(row.content), 'utf8') }))
+    return {
+      files,
+      status: rows.map((row, index) => ({ name: row.name, bytes: files[index].content.length, modified: row.updated.toISOString(), previous: row.previous })),
+      version: versionOf(files),
+    }
+  }
+  /** Runs change in a transaction that holds the lock of the variable files, when they are still those of version. */
+  const locked = async (module: string, instance: string, version: string, change: (client: PoolClient) => Promise<void>) => {
+    await ready()
+    const client = await pool.connect()
+    try {
+      await client.query('BEGIN')
+      await client.query('CALL tf_variable_files_lock()')
+      const { rows } = await client.query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])
+      if (document(rows).version !== version) {
+        await client.query('ROLLBACK')
+        return false
+      }
+      await change(client)
+      await client.query('COMMIT')
+      return true
+    } catch (error) {
+      await client.query('ROLLBACK')
+      throw refused(error)
+    } finally {
+      client.release()
+    }
+  }
+  return {
+    modules: {
+      list: async () => (await query<{ name: string; root: string; description: string; configurator_url: string | null }>('SELECT * FROM tf_modules_list()', []))
+        .map((row) => ({ name: row.name, root: row.root, description: row.description, configuratorUrl: row.configurator_url })),
+      async put(module) {
+        await query('CALL tf_module_put($1, $2, $3, $4)', [module.name, module.root, module.description, module.configuratorUrl])
+      },
+      async delete(name) {
+        await query('CALL tf_module_delete($1)', [name])
+      },
+    },
+    instances: {
+      list: async (module) => (await query<{ module: string; name: string; state_schema: string; created: Date }>('SELECT * FROM tf_instances_list($1)', [module]))
+        .map((row) => ({ module: row.module, name: row.name, stateSchema: row.state_schema, created: row.created.toISOString() })),
+      async create(module, name, stateSchema) {
+        await query('CALL tf_instance_create($1, $2, $3)', [module, name, stateSchema])
+      },
+      async delete(module, name) {
+        await query('CALL tf_instance_delete($1, $2)', [module, name])
+      },
+    },
+    variables: (module, instance) => ({
+      read: async () => document(await query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])),
+      write: (version, contents) => locked(module, instance, version, async (client) => {
+        for (const [name, text] of Object.entries(contents)) {
+          await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text])
+        }
+      }),
+      remove: (version, name) => locked(module, instance, version, async (client) => {
+        await client.query('CALL tf_variable_file_delete($1, $2, $3)', [module, instance, name])
+      }),
+    }),
+  }
+}
+
+/** The HttpError of an error of PostgreSQL that refuses a change, by its SQLSTATE; any other error as it is. */
+function refused(error: unknown): unknown {
+  const { code, constraint, detail } = error as { code?: string; constraint?: string; detail?: string }
+  const reason = detail ?? (error as Error).message
+  switch (code) {
+    case '23505':
+      return new HttpError(409, `The name is in use (${reason}).`)
+    case '23503':
+      if (reason.includes('is still referenced')) {
+        return new HttpError(409, `The module has instances; delete them first (${reason}).`)
+      }
+      return new HttpError(404, `The ${constraint === 'tf_instances_module_fkey' ? 'module' : 'instance'} does not exist (${reason}).`)
+    case '23514':
+    case '22P02':
+      return new HttpError(400, `The value is not allowed (${reason}).`)
+    default:
+      return error
+  }
+}