feature: every write of an instance's variable files is kept as a revision with the schema version of their layout and the application that wrote them.
Release / release (push) Successful in 1m8s
Release / release (push) Successful in 1m8s
This commit is contained in:
1 parent
7c335c7ed2
commit
5e62e143b6
11 files changed
+286
-37
No files matched your search
+12
-2
@@ -4,6 +4,16 @@ export { HttpError, errorHandler, notFoundHandler, strict } from './http.ts'
|
||||
export { QUIET, createLog, fail, timestampsOn, type Log } from './log.ts'
|
||||
export { exitOnSignals, gitVersion, readJsonObject } from './process.ts'
|
||||
export { decrypt, encrypt, encryptionKey } from './secrets.ts'
|
||||
export type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
|
||||
export type { TfInstance, TfModule, TfvarsFileStatus, TfvarsRevision } from '../shared/tfvars.ts'
|
||||
export { STATE_SCHEMA_PATTERN, TFVARS_FILE_PATTERN, TF_NAME_PATTERN } from '../shared/tfvars.ts'
|
||||
export { createTfvarsStore, tfvarsText, versionOf, type InstanceVariables, type TfvarsDocument, type TfvarsFile, type TfvarsStore } from './tfvarsStore.ts'
|
||||
export {
|
||||
createTfvarsStore,
|
||||
tfvarsText,
|
||||
versionOf,
|
||||
type InstanceVariables,
|
||||
type RevisionMeta,
|
||||
type TfvarsDocument,
|
||||
type TfvarsFile,
|
||||
type TfvarsRevisionFiles,
|
||||
type TfvarsStore,
|
||||
} from './tfvarsStore.ts'
|
||||
@@ -1,6 +1,7 @@
|
||||
-- The store of Terraform modules, their instances and the instances' variable files: the tables tf_modules,
|
||||
-- tf_instances and tf_variable_files, and the functions and procedures the store calls. The store applies the file on
|
||||
-- its first use in the schema its connection's search path names; every statement can run again.
|
||||
-- tf_instances, tf_variable_files and tf_variable_revisions, and the functions and procedures the store calls. The
|
||||
-- store applies the file on its first use in the schema its connection's search path names; every statement can run
|
||||
-- again.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tf_modules (
|
||||
name text PRIMARY KEY CHECK (name ~ '^[a-z][a-z0-9-]*$'),
|
||||
@@ -28,6 +29,28 @@ CREATE TABLE IF NOT EXISTS tf_variable_files (
|
||||
FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Every write of an instance's variable files: all its files after the write, by name, the schema version of their
|
||||
-- layout, null when not known, and the application that wrote them.
|
||||
CREATE TABLE IF NOT EXISTS tf_variable_revisions (
|
||||
module text NOT NULL,
|
||||
instance text NOT NULL,
|
||||
revision integer NOT NULL CHECK (revision > 0),
|
||||
files jsonb NOT NULL CHECK (jsonb_typeof(files) = 'object'),
|
||||
schema integer CHECK (schema > 0),
|
||||
producer text NOT NULL CHECK (producer <> ''),
|
||||
saved timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (module, instance, revision),
|
||||
FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- The files of an instance that has no revision are its revision 1, of an unknown schema and producer.
|
||||
INSERT INTO tf_variable_revisions (module, instance, revision, files, schema, producer, saved)
|
||||
SELECT f.module, f.instance, 1, jsonb_object_agg(f.name, f.content), NULL, 'unknown', max(f.updated)
|
||||
FROM tf_variable_files f
|
||||
WHERE NOT EXISTS (SELECT 1 FROM tf_variable_revisions r WHERE r.module = f.module AND r.instance = f.instance)
|
||||
GROUP BY f.module, f.instance
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- The modules, by name.
|
||||
CREATE OR REPLACE FUNCTION tf_modules_list()
|
||||
RETURNS TABLE (name text, root text, description text, configurator_url text)
|
||||
@@ -115,3 +138,36 @@ SET search_path FROM CURRENT
|
||||
AS $$
|
||||
DELETE FROM tf_variable_files WHERE module = module_name AND instance = instance_name AND name = file_name
|
||||
$$;
|
||||
|
||||
-- Records the variable files of an instance as its next revision, with the schema of their layout and their producer;
|
||||
-- called with the lock of tf_variable_files_lock held.
|
||||
CREATE OR REPLACE PROCEDURE tf_variable_revision_add(module_name text, instance_name text, revision_schema integer, revision_producer text)
|
||||
LANGUAGE sql
|
||||
SET search_path FROM CURRENT
|
||||
AS $$
|
||||
INSERT INTO tf_variable_revisions (module, instance, revision, files, schema, producer)
|
||||
SELECT module_name, instance_name,
|
||||
(SELECT coalesce(max(r.revision), 0) + 1 FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name),
|
||||
(SELECT coalesce(jsonb_object_agg(f.name, f.content), '{}'::jsonb) FROM tf_variable_files f WHERE f.module = module_name AND f.instance = instance_name),
|
||||
revision_schema, revision_producer
|
||||
$$;
|
||||
|
||||
-- The revisions of an instance, newest first, with the names of their files.
|
||||
CREATE OR REPLACE FUNCTION tf_variable_revisions_list(module_name text, instance_name text)
|
||||
RETURNS TABLE (revision integer, schema integer, producer text, saved timestamptz, names text[])
|
||||
LANGUAGE sql STABLE
|
||||
SET search_path FROM CURRENT
|
||||
AS $$
|
||||
SELECT r.revision, r.schema, r.producer, r.saved, ARRAY(SELECT jsonb_object_keys(r.files) ORDER BY 1)
|
||||
FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name ORDER BY r.revision DESC
|
||||
$$;
|
||||
|
||||
-- A revision of an instance with its files.
|
||||
CREATE OR REPLACE FUNCTION tf_variable_revision_get(module_name text, instance_name text, revision_number integer)
|
||||
RETURNS TABLE (revision integer, schema integer, producer text, saved timestamptz, files jsonb)
|
||||
LANGUAGE sql STABLE
|
||||
SET search_path FROM CURRENT
|
||||
AS $$
|
||||
SELECT r.revision, r.schema, r.producer, r.saved, r.files
|
||||
FROM tf_variable_revisions r WHERE r.module = module_name AND r.instance = instance_name AND r.revision = revision_number
|
||||
$$;
|
||||
@@ -60,29 +60,62 @@ test('instances are created and refused for a missing module or a name in use',
|
||||
assert.equal(await status(store.modules.delete('router')), 409)
|
||||
})
|
||||
|
||||
const META = { schema: 1, producer: 'Test Configurator 1.0.0' }
|
||||
|
||||
test('variable files are written with their version and refused against another version', async () => {
|
||||
const variables = store.variables('router', 'test')
|
||||
const empty = await variables.read()
|
||||
assert.deepEqual(empty.files, [])
|
||||
assert.equal(empty.version, '')
|
||||
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }), true)
|
||||
assert.equal(empty.revision, null)
|
||||
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }, META), true)
|
||||
const written = await variables.read()
|
||||
assert.deepEqual(written.files.map((file) => [file.name, file.content.toString('utf8')]), [
|
||||
['platform.tfvars.json', tfvarsText({ a: 1 })],
|
||||
['secrets.tfvars.json', tfvarsText({ s: 'x' })],
|
||||
])
|
||||
assert.deepEqual(written.status.map((file) => file.previous), [false, false])
|
||||
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), false)
|
||||
assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), true)
|
||||
assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }, META), false)
|
||||
assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }, META), true)
|
||||
const changed = await variables.read()
|
||||
assert.deepEqual(changed.status.map((file) => file.previous), [true, false])
|
||||
assert.notEqual(changed.version, written.version)
|
||||
assert.equal(await variables.remove(written.version, 'secrets.tfvars.json'), false)
|
||||
assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json'), true)
|
||||
assert.equal(await variables.remove(written.version, 'secrets.tfvars.json', META), false)
|
||||
assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json', META), true)
|
||||
assert.deepEqual((await variables.read()).files.map((file) => file.name), ['platform.tfvars.json'])
|
||||
assert.deepEqual((await store.variables('router', 'platform').read()).files, [])
|
||||
assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) })), 400)
|
||||
assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) })), 404)
|
||||
assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) }, META)), 400)
|
||||
assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) }, META)), 404)
|
||||
})
|
||||
|
||||
test('every write is a revision with its files, schema and producer', async () => {
|
||||
const variables = store.variables('router', 'test')
|
||||
assert.deepEqual((await variables.revisions()).map((revision) => [revision.revision, revision.schema, revision.producer, revision.names]), [
|
||||
[3, 1, META.producer, ['platform.tfvars.json']],
|
||||
[2, 1, META.producer, ['platform.tfvars.json', 'secrets.tfvars.json']],
|
||||
[1, 1, META.producer, ['platform.tfvars.json', 'secrets.tfvars.json']],
|
||||
])
|
||||
const current = await variables.read()
|
||||
assert.equal(await variables.write(current.version, {}, META), true)
|
||||
assert.equal(await variables.write(current.version, { 'platform.tfvars.json': tfvarsText({ a: 3 }) }, { schema: null, producer: 'manage.mjs import' }), true)
|
||||
const newest = (await variables.read()).revision
|
||||
assert.deepEqual(newest && [newest.revision, newest.schema, newest.producer], [4, null, 'manage.mjs import'])
|
||||
const first = await variables.revision(1)
|
||||
assert.deepEqual(first?.files.map((file) => [file.name, file.content.toString('utf8')]), [
|
||||
['platform.tfvars.json', tfvarsText({ a: 1 })],
|
||||
['secrets.tfvars.json', tfvarsText({ s: 'x' })],
|
||||
])
|
||||
assert.equal(await variables.revision(9), null)
|
||||
assert.deepEqual(await store.variables('router', 'platform').revisions(), [])
|
||||
})
|
||||
|
||||
test('the files of an instance without revisions become its revision 1 of an unknown producer', async () => {
|
||||
await pool.query(`INSERT INTO tf_variable_files (module, instance, name, content) VALUES ('router', 'platform', 'a.tfvars.json', '{"b": 1}')`)
|
||||
await pool.query('DELETE FROM tf_variable_revisions')
|
||||
const again = createTfvarsStore(pool)
|
||||
const revisions = await again.variables('router', 'platform').revisions()
|
||||
assert.deepEqual(revisions.map((revision) => [revision.revision, revision.schema, revision.producer, revision.names]), [[1, null, 'unknown', ['a.tfvars.json']]])
|
||||
assert.deepEqual((await again.variables('router', 'test').revisions()).map((revision) => revision.revision), [1])
|
||||
})
|
||||
|
||||
test('an instance is deleted with its variable files, then its module', async () => {
|
||||
@@ -90,6 +123,7 @@ test('an instance is deleted with its variable files, then its module', async ()
|
||||
await store.instances.delete('router', 'platform')
|
||||
assert.deepEqual(await store.instances.list('router'), [])
|
||||
assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_files')).rows[0].n, 0)
|
||||
assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_revisions')).rows[0].n, 0)
|
||||
await store.modules.delete('router')
|
||||
assert.deepEqual(await store.modules.list(), [])
|
||||
})
|
||||
+81
-17
@@ -1,25 +1,42 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import type { Pool, PoolClient } from 'pg'
|
||||
import type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts'
|
||||
import type { TfInstance, TfModule, TfvarsFileStatus, TfvarsRevision } from '../shared/tfvars.ts'
|
||||
import { HttpError } from './http.ts'
|
||||
|
||||
/** A variable file of an instance: its name and its content, the object jsonb keeps written as JSON text. */
|
||||
export type TfvarsFile = { name: string; content: Buffer }
|
||||
|
||||
/** The variable files of an instance, by name, with their version, versionOf the files. */
|
||||
export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string }
|
||||
/**
|
||||
* The variable files of an instance, by name, with their version, versionOf the files, and their newest revision, null
|
||||
* for an instance never written.
|
||||
*/
|
||||
export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string; revision: TfvarsRevision | null }
|
||||
|
||||
/** A revision with its files, by name. */
|
||||
export type TfvarsRevisionFiles = TfvarsRevision & { files: TfvarsFile[] }
|
||||
|
||||
/** What a write records with the files: the schema version of their layout, null when not known, and the producer. */
|
||||
export type RevisionMeta = { schema: number | null; producer: string }
|
||||
|
||||
/** The variable files of one instance. */
|
||||
export type InstanceVariables = {
|
||||
read(): Promise<TfvarsDocument>
|
||||
/**
|
||||
* Writes the given files, name and JSON text of an object, when the files are still those of version, keeping each
|
||||
* replaced content as its previous one; answers false, writing nothing, when they changed since.
|
||||
* replaced content as its previous one, and records all files as the next revision with meta; answers false, writing
|
||||
* nothing, when they changed since. A write of no file records no revision.
|
||||
*/
|
||||
write(version: string, contents: Record<string, string>): Promise<boolean>
|
||||
/** Deletes the file name when the files are still those of version; answers false, deleting nothing, otherwise. */
|
||||
remove(version: string, name: string): Promise<boolean>
|
||||
write(version: string, contents: Record<string, string>, meta: RevisionMeta): Promise<boolean>
|
||||
/**
|
||||
* Deletes the file name when the files are still those of version, and records the remaining files as the next
|
||||
* revision with meta; answers false, deleting nothing, otherwise.
|
||||
*/
|
||||
remove(version: string, name: string, meta: RevisionMeta): Promise<boolean>
|
||||
/** The revisions, newest first. */
|
||||
revisions(): Promise<TfvarsRevision[]>
|
||||
/** The revision of the number with its files; null when there is none. */
|
||||
revision(revision: number): Promise<TfvarsRevisionFiles | null>
|
||||
}
|
||||
|
||||
/** The modules, the instances and their variable files in PostgreSQL. */
|
||||
@@ -56,6 +73,15 @@ const SQL = new URL('./sql/tfvars_store.sql', import.meta.url)
|
||||
|
||||
type FileRow = { name: string; content: object; previous: boolean; updated: Date }
|
||||
|
||||
type RevisionRow = { revision: number; schema: number | null; producer: string; saved: Date; names: string[] }
|
||||
|
||||
const revisionOf = (row: Omit<RevisionRow, 'names'>, names: string[]): TfvarsRevision =>
|
||||
({ revision: row.revision, schema: row.schema, producer: row.producer, saved: row.saved.toISOString(), names })
|
||||
|
||||
/** The files of an object of revision files, by name, written as JSON text. */
|
||||
const filesOf = (files: Record<string, object>): TfvarsFile[] =>
|
||||
Object.keys(files).sort().map((name) => ({ name, content: Buffer.from(tfvarsText(files[name]), 'utf8') }))
|
||||
|
||||
/**
|
||||
* The store of sql/tfvars_store.sql in the database of pool, applied at the first use in the schema of the
|
||||
* connection's search path. A refused change is an HttpError: 409 for a name in use or a module that has instances,
|
||||
@@ -72,27 +98,49 @@ export function createTfvarsStore(pool: Pool): TfvarsStore {
|
||||
throw refused(error)
|
||||
}
|
||||
}
|
||||
const document = (rows: FileRow[]): TfvarsDocument => {
|
||||
const document = (rows: FileRow[], newest: RevisionRow | undefined): TfvarsDocument => {
|
||||
const files = rows.map((row) => ({ name: row.name, content: Buffer.from(tfvarsText(row.content), 'utf8') }))
|
||||
return {
|
||||
files,
|
||||
status: rows.map((row, index) => ({ name: row.name, bytes: files[index].content.length, modified: row.updated.toISOString(), previous: row.previous })),
|
||||
version: versionOf(files),
|
||||
revision: newest === undefined ? null : revisionOf(newest, newest.names),
|
||||
}
|
||||
}
|
||||
/** Runs change in a transaction that holds the lock of the variable files, when they are still those of version. */
|
||||
const locked = async (module: string, instance: string, version: string, change: (client: PoolClient) => Promise<void>) => {
|
||||
/** The files and the newest revision of an instance, read in one snapshot. */
|
||||
const read = async (module: string, instance: string): Promise<TfvarsDocument> => {
|
||||
await ready()
|
||||
const client = await pool.connect()
|
||||
try {
|
||||
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
|
||||
const files = await client.query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])
|
||||
const revisions = await client.query<RevisionRow>('SELECT * FROM tf_variable_revisions_list($1, $2) LIMIT 1', [module, instance])
|
||||
await client.query('COMMIT')
|
||||
return document(files.rows, revisions.rows[0])
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK')
|
||||
throw refused(error)
|
||||
} finally {
|
||||
client.release()
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Runs change in a transaction that holds the lock of the variable files, when they are still those of version, and
|
||||
* records the files after it as the next revision with meta.
|
||||
*/
|
||||
const locked = async (module: string, instance: string, version: string, meta: RevisionMeta, change: (client: PoolClient) => Promise<void>) => {
|
||||
await ready()
|
||||
const client = await pool.connect()
|
||||
try {
|
||||
await client.query('BEGIN')
|
||||
await client.query('CALL tf_variable_files_lock()')
|
||||
const { rows } = await client.query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])
|
||||
if (document(rows).version !== version) {
|
||||
if (document(rows, undefined).version !== version) {
|
||||
await client.query('ROLLBACK')
|
||||
return false
|
||||
}
|
||||
await change(client)
|
||||
await client.query('CALL tf_variable_revision_add($1, $2, $3, $4)', [module, instance, meta.schema, meta.producer])
|
||||
await client.query('COMMIT')
|
||||
return true
|
||||
} catch (error) {
|
||||
@@ -124,15 +172,31 @@ export function createTfvarsStore(pool: Pool): TfvarsStore {
|
||||
},
|
||||
},
|
||||
variables: (module, instance) => ({
|
||||
read: async () => document(await query<FileRow>('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])),
|
||||
write: (version, contents) => locked(module, instance, version, async (client) => {
|
||||
for (const [name, text] of Object.entries(contents)) {
|
||||
await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text])
|
||||
read: () => read(module, instance),
|
||||
write: async (version, contents, meta) => {
|
||||
if (Object.keys(contents).length === 0) {
|
||||
return (await read(module, instance)).version === version
|
||||
}
|
||||
}),
|
||||
remove: (version, name) => locked(module, instance, version, async (client) => {
|
||||
return locked(module, instance, version, meta, async (client) => {
|
||||
for (const [name, text] of Object.entries(contents)) {
|
||||
await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text])
|
||||
}
|
||||
})
|
||||
},
|
||||
remove: (version, name, meta) => locked(module, instance, version, meta, async (client) => {
|
||||
await client.query('CALL tf_variable_file_delete($1, $2, $3)', [module, instance, name])
|
||||
}),
|
||||
revisions: async () => (await query<RevisionRow>('SELECT * FROM tf_variable_revisions_list($1, $2)', [module, instance]))
|
||||
.map((row) => revisionOf(row, row.names)),
|
||||
revision: async (revision) => {
|
||||
const [row] = await query<Omit<RevisionRow, 'names'> & { files: Record<string, object> }>(
|
||||
'SELECT * FROM tf_variable_revision_get($1, $2, $3)', [module, instance, revision])
|
||||
if (row === undefined) {
|
||||
return null
|
||||
}
|
||||
const files = filesOf(row.files)
|
||||
return { ...revisionOf(row, files.map((file) => file.name)), files }
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user