feature: every write of an instance's variable files is kept as a revision with the schema version of their layout and the application that wrote them.
Release / release (push) Successful in 1m8s

This commit is contained in:
slawek committed 2026-11-06 22:22:35 +01:00
1 parent 7c335c7ed2
commit 5e62e143b6
11 files changed
+286 -37

No files matched your search

+12 -2
View File
@@ -4,6 +4,16 @@ export { HttpError, errorHandler, notFoundHandler, strict } from './http.ts'
export { QUIET, createLog, fail, timestampsOn, type Log } from './log.ts'
export { exitOnSignals, gitVersion, readJsonObject } from './process.ts'
export { decrypt, encrypt, encryptionKey } from './secrets.ts'
export type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
export type { TfInstance, TfModule, TfvarsFileStatus, TfvarsRevision } from '../shared/tfvars.ts'
export { STATE_SCHEMA_PATTERN, TFVARS_FILE_PATTERN, TF_NAME_PATTERN } from '../shared/tfvars.ts'
export { createTfvarsStore, tfvarsText, versionOf, type InstanceVariables, type TfvarsDocument, type TfvarsFile, type TfvarsStore } from './tfvarsStore.ts'
export {
createTfvarsStore,
tfvarsText,
versionOf,
type InstanceVariables,
type RevisionMeta,
type TfvarsDocument,
type TfvarsFile,
type TfvarsRevisionFiles,
type TfvarsStore,
} from './tfvarsStore.ts'
+58 -2
View File
@@ -1,6 +1,7 @@
-- The store of Terraform modules, their instances and the instances' variable files: the tables tf_modules,
-- tf_instances and tf_variable_files, and the functions and procedures the store calls. The store applies the file on
-- its first use in the schema its connection's search path names; every statement can run again.
-- tf_instances, tf_variable_files and tf_variable_revisions, and the functions and procedures the store calls. The
-- store applies the file on its first use in the schema its connection's search path names; every statement can run
-- again.
CREATE TABLE IF NOT EXISTS tf_modules (
name text PRIMARY KEY CHECK (name ~ '^[a-z][a-z0-9-]*$'),
@@ -28,6 +29,28 @@ CREATE TABLE IF NOT EXISTS tf_variable_files (
FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE
);
-- Every write of an instance's variable files: all its files after the write, by name, the schema version of their
-- layout, null when not known, and the application that wrote them.
CREATE TABLE IF NOT EXISTS tf_variable_revisions (
module text NOT NULL,
instance text NOT NULL,
revision integer NOT NULL CHECK (revision > 0),
files jsonb NOT NULL CHECK (jsonb_typeof(files) = 'object'),
schema integer CHECK (schema > 0),
producer text NOT NULL CHECK (producer <> ''),
saved timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (module, instance, revision),
FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE
);
-- The files of an instance that has no revision are its revision 1, of an unknown schema and producer.
INSERT INTO tf_variable_revisions (module, instance, revision, files, schema, producer, saved)
SELECT f.module, f.instance, 1, jsonb_object_agg(f.name, f.content), NULL, 'unknown', max(f.updated)
FROM tf_variable_files f
WHERE NOT EXISTS (SELECT 1 FROM tf_variable_revisions r WHERE r.module = f.module AND r.instance = f.instance)
GROUP BY f.module, f.instance
ON CONFLICT DO NOTHING;
-- The modules, by name.
CREATE OR REPLACE FUNCTION tf_modules_list()
RETURNS TABLE (name text, root text, description text, configurator_url text)
@@ -115,3 +138,36 @@ SET search_path FROM CURRENT
AS $$
DELETE FROM tf_variable_files WHERE module = module_name AND instance = instance_name AND name = file_name
$$;
-- Records the variable files of an instance as its next revision, with the schema of their layout and their producer;
-- called with the lock of tf_variable_files_lock held.
CREATE OR REPLACE PROCEDURE tf_variable_revision_add(module_name text, instance_name text, revision_schema integer, revision_producer text)
LANGUAGE sql
SET search_path FROM CURRENT
AS $$
INSERT INTO tf_variable_revisions (module, instance, revision, files, schema, producer)
SELECT module_name, instance_name,
(SELECT coalesce(max(r.revision), 0) + 1 FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name),
(SELECT coalesce(jsonb_object_agg(f.name, f.content), '{}'::jsonb) FROM tf_variable_files f WHERE f.module = module_name AND f.instance = instance_name),
revision_schema, revision_producer
$$;
-- The revisions of an instance, newest first, with the names of their files.
CREATE OR REPLACE FUNCTION tf_variable_revisions_list(module_name text, instance_name text)
RETURNS TABLE (revision integer, schema integer, producer text, saved timestamptz, names text[])
LANGUAGE sql STABLE
SET search_path FROM CURRENT
AS $$
SELECT r.revision, r.schema, r.producer, r.saved, ARRAY(SELECT jsonb_object_keys(r.files) ORDER BY 1)
FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name ORDER BY r.revision DESC
$$;
-- A revision of an instance with its files.
CREATE OR REPLACE FUNCTION tf_variable_revision_get(module_name text, instance_name text, revision_number integer)
RETURNS TABLE (revision integer, schema integer, producer text, saved timestamptz, files jsonb)
LANGUAGE sql STABLE
SET search_path FROM CURRENT
AS $$
SELECT r.revision, r.schema, r.producer, r.saved, r.files
FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name AND r.revision = revision_number
$$;
+41 -7
View File
@@ -60,29 +60,62 @@ test('instances are created and refused for a missing module or a name in use',
assert.equal(await status(store.modules.delete('router')), 409)
})
const META = { schema: 1, producer: 'Test Configurator 1.0.0' }
test('variable files are written with their version and refused against another version', async () => {
const variables = store.variables('router', 'test')
const empty = await variables.read()
assert.deepEqual(empty.files, [])
assert.equal(empty.version, '')
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }), true)
assert.equal(empty.revision, null)
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }, META), true)
const written = await variables.read()
assert.deepEqual(written.files.map((file) => [file.name, file.content.toString('utf8')]), [
['platform.tfvars.json', tfvarsText({ a: 1 })],
['secrets.tfvars.json', tfvarsText({ s: 'x' })],
])
assert.deepEqual(written.status.map((file) => file.previous), [false, false])
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), false)
assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), true)
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }, META), false)
assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }, META), true)
const changed = await variables.read()
assert.deepEqual(changed.status.map((file) => file.previous), [true, false])
assert.notEqual(changed.version, written.version)
assert.equal(await variables.remove(written.version, 'secrets.tfvars.json'), false)
assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json'), true)
assert.equal(await variables.remove(written.version, 'secrets.tfvars.json', META), false)
assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json', META), true)
assert.deepEqual((await variables.read()).files.map((file) => file.name), ['platform.tfvars.json'])
assert.deepEqual((await store.variables('router', 'platform').read()).files, [])
assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) })), 400)
assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) })), 404)
assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) }, META)), 400)
assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) }, META)), 404)
})
test('every write is a revision with its files, schema and producer', async () => {
const variables = store.variables('router', 'test')
assert.deepEqual((await variables.revisions()).map((revision) => [revision.revision, revision.schema, revision.producer, revision.names]), [
[3, 1, META.producer, ['platform.tfvars.json']],
[2, 1, META.producer, ['platform.tfvars.json', 'secrets.tfvars.json']],
[1, 1, META.producer, ['platform.tfvars.json', 'secrets.tfvars.json']],
])
const current = await variables.read()
assert.equal(await variables.write(current.version, {}, META), true)
assert.equal(await variables.write(current.version, { 'platform.tfvars.json': tfvarsText({ a: 3 }) }, { schema: null, producer: 'manage.mjs import' }), true)
const newest = (await variables.read()).revision
assert.deepEqual(newest && [newest.revision, newest.schema, newest.producer], [4, null, 'manage.mjs import'])
const first = await variables.revision(1)
assert.deepEqual(first?.files.map((file) => [file.name, file.content.toString('utf8')]), [
['platform.tfvars.json', tfvarsText({ a: 1 })],
['secrets.tfvars.json', tfvarsText({ s: 'x' })],
])
assert.equal(await variables.revision(9), null)
assert.deepEqual(await store.variables('router', 'platform').revisions(), [])
})
test('the files of an instance without revisions become its revision 1 of an unknown producer', async () => {
await pool.query(`INSERT INTO tf_variable_files (module, instance, name, content) VALUES ('router', 'platform', 'a.tfvars.json', '{"b": 1}')`)
await pool.query('DELETE FROM tf_variable_revisions')
const again = createTfvarsStore(pool)
const revisions = await again.variables('router', 'platform').revisions()
assert.deepEqual(revisions.map((revision) => [revision.revision, revision.schema, revision.producer, revision.names]), [[1, null, 'unknown', ['a.tfvars.json']]])
assert.deepEqual((await again.variables('router', 'test').revisions()).map((revision) => revision.revision), [1])
})
test('an instance is deleted with its variable files, then its module', async () => {
@@ -90,6 +123,7 @@ test('an instance is deleted with its variable files, then its module', async ()
await store.instances.delete('router', 'platform')
assert.deepEqual(await store.instances.list('router'), [])
assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_files')).rows[0].n, 0)
assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_revisions')).rows[0].n, 0)
await store.modules.delete('router')
assert.deepEqual(await store.modules.list(), [])
})
+81 -17
View File
@@ -1,25 +1,42 @@
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import type { Pool, PoolClient } from 'pg'
import type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
import type { TfInstance, TfModule, TfvarsFileStatus, TfvarsRevision } from '../shared/tfvars.ts'
import { HttpError } from './http.ts'
/** A variable file of an instance: its name and its content, the object jsonb keeps written as JSON text. */
export type TfvarsFile = { name: string; content: Buffer }
/** The variable files of an instance, by name, with their version, versionOf the files. */
export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string }
/**
* The variable files of an instance, by name, with their version, versionOf the files, and their newest revision, null
* for an instance never written.
*/
export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string; revision: TfvarsRevision | null }
/** A revision with its files, by name. */
export type TfvarsRevisionFiles = TfvarsRevision & { files: TfvarsFile[] }
/** What a write records with the files: the schema version of their layout, null when not known, and the producer. */
export type RevisionMeta = { schema: number | null; producer: string }
/** The variable files of one instance. */
export type InstanceVariables = {
read(): Promise<TfvarsDocument>
/**
* Writes the given files, name and JSON text of an object, when the files are still those of version, keeping each
* replaced content as its previous one; answers false, writing nothing, when they changed since.
* replaced content as its previous one, and records all files as the next revision with meta; answers false, writing
* nothing, when they changed since. A write of no file records no revision.
*/
write(version: string, contents: Record<string, string>): Promise<boolean>
/** Deletes the file name when the files are still those of version; answers false, deleting nothing, otherwise. */
remove(version: string, name: string): Promise<boolean>
write(version: string, contents: Record<string, string>, meta: RevisionMeta): Promise<boolean>
/**
* Deletes the file name when the files are still those of version, and records the remaining files as the next
* revision with meta; answers false, deleting nothing, otherwise.
*/
remove(version: string, name: string, meta: RevisionMeta): Promise<boolean>
/** The revisions, newest first. */
revisions(): Promise<TfvarsRevision[]>
/** The revision of the number with its files; null when there is none. */
revision(revision: number): Promise<TfvarsRevisionFiles | null>
}
/** The modules, the instances and their variable files in PostgreSQL. */
@@ -56,6 +73,15 @@ const SQL = new URL('./sql/tfvars_store.sql', import.meta.url)
type FileRow = { name: string; content: object; previous: boolean; updated: Date }
type RevisionRow = { revision: number; schema: number | null; producer: string; saved: Date; names: string[] }
const revisionOf = (row: Omit<RevisionRow, 'names'>, names: string[]): TfvarsRevision =>
({ revision: row.revision, schema: row.schema, producer: row.producer, saved: row.saved.toISOString(), names })
/** The files of an object of revision files, by name, written as JSON text. */
const filesOf = (files: Record<string, object>): TfvarsFile[] =>
Object.keys(files).sort().map((name) => ({ name, content: Buffer.from(tfvarsText(files[name]), 'utf8') }))
/**
* The store of sql/tfvars_store.sql in the database of pool, applied at the first use in the schema of the
* connection's search path. A refused change is an HttpError: 409 for a name in use or a module that has instances,
@@ -72,27 +98,49 @@ export function createTfvarsStore(pool: Pool): TfvarsStore {
throw refused(error)
}
}
const document = (rows: FileRow[]): TfvarsDocument => {
const document = (rows: FileRow[], newest: RevisionRow | undefined): TfvarsDocument => {
const files = rows.map((row) => ({ name: row.name, content: Buffer.from(tfvarsText(row.content), 'utf8') }))
return {
files,
status: rows.map((row, index) => ({ name: row.name, bytes: files[index].content.length, modified: row.updated.toISOString(), previous: row.previous })),
version: versionOf(files),
revision: newest === undefined ? null : revisionOf(newest, newest.names),
}
}
/** Runs change in a transaction that holds the lock of the variable files, when they are still those of version. */
const locked = async (module: string, instance: string, version: string, change: (client: PoolClient) => Promise<void>) => {
/** The files and the newest revision of an instance, read in one snapshot. */
const read = async (module: string, instance: string): Promise<TfvarsDocument> => {
await ready()
const client = await pool.connect()
try {
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
const files = await client.query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])
const revisions = await client.query<RevisionRow>('SELECT * FROM tf_variable_revisions_list($1, $2) LIMIT 1', [module, instance])
await client.query('COMMIT')
return document(files.rows, revisions.rows[0])
} catch (error) {
await client.query('ROLLBACK')
throw refused(error)
} finally {
client.release()
}
}
/**
* Runs change in a transaction that holds the lock of the variable files, when they are still those of version, and
* records the files after it as the next revision with meta.
*/
const locked = async (module: string, instance: string, version: string, meta: RevisionMeta, change: (client: PoolClient) => Promise<void>) => {
await ready()
const client = await pool.connect()
try {
await client.query('BEGIN')
await client.query('CALL tf_variable_files_lock()')
const { rows } = await client.query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])
if (document(rows).version !== version) {
if (document(rows, undefined).version !== version) {
await client.query('ROLLBACK')
return false
}
await change(client)
await client.query('CALL tf_variable_revision_add($1, $2, $3, $4)', [module, instance, meta.schema, meta.producer])
await client.query('COMMIT')
return true
} catch (error) {
@@ -124,15 +172,31 @@ export function createTfvarsStore(pool: Pool): TfvarsStore {
},
},
variables: (module, instance) => ({
read: async () => document(await query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])),
write: (version, contents) => locked(module, instance, version, async (client) => {
for (const [name, text] of Object.entries(contents)) {
await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text])
read: () => read(module, instance),
write: async (version, contents, meta) => {
if (Object.keys(contents).length === 0) {
return (await read(module, instance)).version === version
}
}),
remove: (version, name) => locked(module, instance, version, async (client) => {
return locked(module, instance, version, meta, async (client) => {
for (const [name, text] of Object.entries(contents)) {
await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text])
}
})
},
remove: (version, name, meta) => locked(module, instance, version, meta, async (client) => {
await client.query('CALL tf_variable_file_delete($1, $2, $3)', [module, instance, name])
}),
revisions: async () => (await query<RevisionRow>('SELECT * FROM tf_variable_revisions_list($1, $2)', [module, instance]))
.map((row) => revisionOf(row, row.names)),
revision: async (revision) => {
const [row] = await query<Omit<RevisionRow, 'names'> & { files: Record<string, object> }>(
'SELECT * FROM tf_variable_revision_get($1, $2, $3)', [module, instance, revision])
if (row === undefined) {
return null
}
const files = filesOf(row.files)
return { ...revisionOf(row, files.map((file) => file.name)), files }
},
}),
}
}