From 3d3a700f11bd516596cd428c344cd0312f868f14 Mon Sep 17 00:00:00 2001 From: Slawomir Koszewski Date: Wed, 28 Oct 2026 23:58:54 +0100 Subject: [PATCH] feature: the tfvars store keeps Terraform modules, their instances and the instances' variable files in PostgreSQL. --- ADR.md | 23 +++++ CHANGELOG.md | 13 +++ README.md | 38 +++++++- build.mjs | 8 +- package-lock.json | 172 +++++++++++++++++++++++++++++++++++- package.json | 7 +- server/index.ts | 3 + server/sql/tfvars_store.sql | 117 ++++++++++++++++++++++++ server/tfvarsStore.test.ts | 95 ++++++++++++++++++++ server/tfvarsStore.ts | 158 +++++++++++++++++++++++++++++++++ shared/tfvars.ts | 36 ++++++++ tsconfig.build.json | 3 +- web/index.ts | 2 + 13 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 server/sql/tfvars_store.sql create mode 100644 server/tfvarsStore.test.ts create mode 100644 server/tfvarsStore.ts create mode 100644 shared/tfvars.ts diff --git a/ADR.md b/ADR.md index cd33292..775e837 100644 --- a/ADR.md +++ b/ADR.md @@ -95,3 +95,26 @@ object can be written one after the other. The kind stays the store's, not a fie Consequences: A key of `config.json` holding a dot cannot be the key of a store. A single key reads and writes as before, so applications of one-key stores need no change. The application checks the object of a path, such as the keys it allows in it, itself. + +## ADR-5: Terraform modules, instances and their variable files are tables shared through the toolkit + +Status: Accepted + +Context: Cloud Platform deploys many instances of many Terraform modules, and each module's +configurator, a separate application, edits the variables of its instances. Both have to read +and write the same modules, instances and variable files, with the same versions, so that a +write from an outdated read is refused whichever application made it. + +Decision: The toolkit holds the tables and their functions and procedures, +`server/sql/tfvars_store.sql`, and `createTfvarsStore`, which both applications use with their own +`pg` pool. An instance names the schema of its state, unique among instances. A variable file is +a row of its instance, its object as `jsonb` with the one it replaced. The version of an +instance's files is the name and the SHA-256 of each file's text, `tfvarsText` of its object, in +name order; a write locks the table, compares the version and writes in one transaction. A change +the tables refuse is an `HttpError` with the status the routes answer. `pg` is an optional peer +dependency, as the applications that do not use the store do not need it. + +Consequences: Every application of the store works on the same rows and refuses the same +conflicts, and changes of the tables reach them with a release of the toolkit. The objects' keys +are in `jsonb`'s order. The store is tested against a real PostgreSQL server, which the release +workflow does not have, so `npm test` runs on the developer's machine. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b41c..18663f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.4.0 + +feature: the tfvars store keeps Terraform modules, their instances and the instances' variable files in PostgreSQL. + +- `createTfvarsStore(pool)`: the tables `tf_modules`, `tf_instances` and `tf_variable_files`, + created at the first use, with `modules` (`list`, `put`, `delete`), `instances` (`list`, + `create`, `delete`) and `variables(module, instance)` (`read`, `write`, `remove`), whose writes + take the version read and answer `false` when the files changed since. +- `versionOf`, `tfvarsText`, and the types `TfModule`, `TfInstance` and `TfvarsFileStatus` with + the patterns of their names, also exported from `/web`. +- `pg` is an optional peer dependency, for the store. +- `npm test` tests the store against a scratch database of the server the `PG*` variables name. + ## 0.3.0 feature: the key of a credential store may be a path of keys, so stores of several kinds share one object of config.json. diff --git a/README.md b/README.md index 2f783eb..f3d1e92 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ code; `vite` for the Vite helpers. | `readJsonObject(path, options)` | a JSON object file and its version, the SHA-256 of its bytes; optionally `{}` for a missing file | | `encryptionKey()`, `encrypt(text, key)`, `decrypt(encrypted, key)` | the key of the secrets and AES-256-GCM encryption; see [Credential store](#credential-store) | | `CredentialStore`, `credentialProblems`, `credentialValues`, `registerCredentialRoutes`, `POSTGRESQL_KIND` | the credential store; see [Credential store](#credential-store) | +| `createTfvarsStore`, `versionOf`, `tfvarsText`, `TfModule`, `TfInstance`, `TfvarsFileStatus`, `TF_NAME_PATTERN`, `TFVARS_FILE_PATTERN`, `STATE_SCHEMA_PATTERN` | the Terraform modules, their instances and their variable files in PostgreSQL; see [Tfvars store](#tfvars-store) | ## sk-react-toolkit/web @@ -79,6 +80,7 @@ code; `vite` for the Vite helpers. | `RowDialog`, `useValidation` | a dialog that edits a value validated by the server as it changes, and the validation hook | | `createBrowserSettings(keys, event)` | switches kept in localStorage and followed across tabs | | `CredentialsButton` | the dialog of a credential store; see [Credential store](#credential-store) | +| `TfModule`, `TfInstance`, `TfvarsFileStatus`, `TF_NAME_PATTERN`, `TFVARS_FILE_PATTERN`, `STATE_SCHEMA_PATTERN` | the types and name patterns of the [Tfvars store](#tfvars-store) | ## sk-react-toolkit/vite @@ -159,11 +161,43 @@ them; a tab with a problem has a red label. A secret shows only whether it is ke replaces it and Remove drops it. Save writes each store that changed, each write with the version the previous one answers. +## Tfvars store + +The tfvars store keeps Terraform modules, their instances and the variable files of each +instance in PostgreSQL, for Cloud Platform, which deploys the instances, and the configurators, +which edit their variables. `createTfvarsStore(pool)` takes a `pg` pool and applies +`server/sql/tfvars_store.sql` at its first use, in the schema of the connection's search path: + +| Table | Rows | +| --- | --- | +| `tf_modules` | `name`, `root`, the folder of the Terraform code, `description` and `configurator_url` | +| `tf_instances` | `module`, `name` and `state_schema`, the schema of the instance's state in the pg backend, unique, and `created` | +| `tf_variable_files` | `module`, `instance`, `name` (`*.tfvars.json`), `content` and `previous`, the object it replaced, as `jsonb`, and `updated` | + +Names of modules and instances are `TF_NAME_PATTERN`, of variable files `TFVARS_FILE_PATTERN`, +of state schemas `STATE_SCHEMA_PATTERN`. + +- `modules.list()`, `modules.put(module)`, which creates or changes, and `modules.delete(name)`. +- `instances.list(module)`, `instances.create(module, name, stateSchema)` and + `instances.delete(module, name)`, which deletes its variable files too. +- `variables(module, instance)`: `read()` answers the files, their sizes and times and the + version, `versionOf` the files, the name and the SHA-256 of each in name order. `write(version, + contents)` writes files given by name as JSON text and `remove(version, name)` deletes one; both + lock the table, compare the version and change in one transaction, and answer `false`, changing + nothing, when the files changed since the version. + +A change the tables refuse throws `HttpError`: 409 for a name or a state schema in use and for a +module that has instances, 404 for a module or an instance that does not exist, 400 for a name or +a content the tables do not accept. The content of a file is the object as `jsonb` keeps it, +written by `tfvarsText`, so the order of object keys is `jsonb`'s; arrays keep theirs. + ## Development `npm install` installs the peer dependencies as development dependencies; `npm run typecheck` -and `npm run lint` check the code. `npm run build` (`build.mjs`) builds the published package in -`dist/`: the code compiled to JavaScript with its declarations, README.md, CHANGELOG.md and a +and `npm run lint` check the code. `npm test` tests the tfvars store in a scratch database it +creates and drops on the PostgreSQL server the `PG*` variables name, connected to through the +database `postgres`. `npm run build` (`build.mjs`) builds the published package in +`dist/`: the code compiled to JavaScript with its declarations, the SQL of the server, README.md, CHANGELOG.md and a `package.json` whose exports name the compiled files. ## Releases diff --git a/build.mjs b/build.mjs index c81bcfd..3bd11cd 100755 --- a/build.mjs +++ b/build.mjs @@ -1,10 +1,11 @@ #!/usr/bin/env node // Builds the published package in dist/: the toolkit compiled to JavaScript with its declarations, the declaration of -// the font plugin's virtual module, README.md, CHANGELOG.md and a package.json whose exports name the compiled files. +// the font plugin's virtual module, the SQL of the server, README.md, CHANGELOG.md and a package.json whose exports +// name the compiled files. import { spawnSync } from 'node:child_process'; -import { copyFileSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { copyFileSync, cpSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; const ROOT = import.meta.dirname; @@ -34,6 +35,9 @@ copyFileSync(join(ROOT, 'web', 'virtual-font.d.ts'), join(DIST, 'web', 'virtual- const renderApp = join(DIST, 'web', 'renderApp.d.ts'); writeFileSync(renderApp, `/// \n${readFileSync(renderApp, 'utf8')}`); +// The tfvars store reads its SQL next to the compiled module. +cpSync(join(ROOT, 'server', 'sql'), join(DIST, 'server', 'sql'), { recursive: true }); + for (const file of ['README.md', 'CHANGELOG.md']) copyFileSync(join(ROOT, file), join(DIST, file)); const source = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')); diff --git a/package-lock.json b/package-lock.json index f6cfa93..e549d11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@koszewscy/sk-react-toolkit", - "version": "0.3.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@koszewscy/sk-react-toolkit", - "version": "0.3.0", + "version": "0.4.0", "devDependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", @@ -14,10 +14,12 @@ "@mui/icons-material": "^9.5.0", "@mui/material": "^9.5.0", "@types/node": "^26.6.5", + "@types/pg": "^8.23.1", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", "fastify": "^5.12.7", "oxlint": "^1.89.0", + "pg": "^8.23.1", "react": "^19.3.1", "react-dom": "^19.3.1", "typescript": "^7.0.2", @@ -33,6 +35,7 @@ "@mui/icons-material": "^9.5.0", "@mui/material": "^9.5.0", "fastify": "^5.12.7", + "pg": "^8.23.1", "react": "^19.3.1", "react-dom": "^19.3.1", "vite": "^8.3.5" @@ -56,6 +59,9 @@ "fastify": { "optional": true }, + "pg": { + "optional": true + }, "react": { "optional": true }, @@ -1539,6 +1545,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/prop-types": { "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", @@ -3131,6 +3149,103 @@ "node": ">=8" } }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3220,6 +3335,49 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/process-warning": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.1.tgz", @@ -3744,6 +3902,16 @@ "optional": true } } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/package.json b/package.json index 2162481..8a735fb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@koszewscy/sk-react-toolkit", - "version": "0.3.0", + "version": "0.4.0", "description": "Shared code of the React and Fastify web applications: web components, server helpers and the Vite font plugin.", "private": true, "type": "module", @@ -19,6 +19,7 @@ "scripts": { "typecheck": "tsc -p tsconfig.json", "lint": "oxlint", + "test": "node --test server/tfvarsStore.test.ts", "build": "node build.mjs" }, "peerDependencies": { @@ -28,6 +29,7 @@ "@mui/icons-material": "^9.5.0", "@mui/material": "^9.5.0", "fastify": "^5.12.7", + "pg": "^8.23.1", "react": "^19.3.1", "react-dom": "^19.3.1", "vite": "^8.3.5" @@ -35,6 +37,7 @@ "peerDependenciesMeta": { "@fastify/static": { "optional": true }, "fastify": { "optional": true }, + "pg": { "optional": true }, "vite": { "optional": true }, "@emotion/react": { "optional": true }, "@emotion/styled": { "optional": true }, @@ -50,10 +53,12 @@ "@mui/icons-material": "^9.5.0", "@mui/material": "^9.5.0", "@types/node": "^26.6.5", + "@types/pg": "^8.23.1", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", "fastify": "^5.12.7", "oxlint": "^1.89.0", + "pg": "^8.23.1", "react": "^19.3.1", "react-dom": "^19.3.1", "typescript": "^7.0.2", diff --git a/server/index.ts b/server/index.ts index 40c9a89..c5f7f8c 100644 --- a/server/index.ts +++ b/server/index.ts @@ -4,3 +4,6 @@ export { HttpError, errorHandler, notFoundHandler, strict } from './http.ts' export { QUIET, createLog, fail, timestampsOn, type Log } from './log.ts' export { exitOnSignals, gitVersion, readJsonObject } from './process.ts' export { decrypt, encrypt, encryptionKey } from './secrets.ts' +export type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts' +export { STATE_SCHEMA_PATTERN, TFVARS_FILE_PATTERN, TF_NAME_PATTERN } from '../shared/tfvars.ts' +export { createTfvarsStore, tfvarsText, versionOf, type InstanceVariables, type TfvarsDocument, type TfvarsFile, type TfvarsStore } from './tfvarsStore.ts' diff --git a/server/sql/tfvars_store.sql b/server/sql/tfvars_store.sql new file mode 100644 index 0000000..71eddee --- /dev/null +++ b/server/sql/tfvars_store.sql @@ -0,0 +1,117 @@ +-- The store of Terraform modules, their instances and the instances' variable files: the tables tf_modules, +-- tf_instances and tf_variable_files, and the functions and procedures the store calls. The store applies the file on +-- its first use in the schema its connection's search path names; every statement can run again. + +CREATE TABLE IF NOT EXISTS tf_modules ( + name text PRIMARY KEY CHECK (name ~ '^[a-z][a-z0-9-]*$'), + root text NOT NULL CHECK (root <> ''), + description text NOT NULL DEFAULT '', + configurator_url text +); + +CREATE TABLE IF NOT EXISTS tf_instances ( + module text NOT NULL REFERENCES tf_modules (name), + name text NOT NULL CHECK (name ~ '^[a-z][a-z0-9-]*$'), + state_schema text NOT NULL UNIQUE CHECK (state_schema ~ '^[a-z_][a-z0-9_]*$'), + created timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (module, name) +); + +CREATE TABLE IF NOT EXISTS tf_variable_files ( + module text NOT NULL, + instance text NOT NULL, + name text NOT NULL CHECK (name ~ '^[A-Za-z0-9_.-]+\.tfvars\.json$'), + content jsonb NOT NULL CHECK (jsonb_typeof(content) = 'object'), + previous jsonb CHECK (jsonb_typeof(previous) = 'object'), + updated timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (module, instance, name), + FOREIGN KEY (module, instance) REFERENCES tf_instances (module, name) ON DELETE CASCADE +); + +-- The modules, by name. +CREATE OR REPLACE FUNCTION tf_modules_list() +RETURNS TABLE (name text, root text, description text, configurator_url text) +LANGUAGE sql STABLE +SET search_path FROM CURRENT +AS $$ + SELECT m.name, m.root, m.description, m.configurator_url FROM tf_modules m ORDER BY m.name +$$; + +-- Creates a module, or changes the one of that name. +CREATE OR REPLACE PROCEDURE tf_module_put(module_name text, module_root text, module_description text, module_configurator_url text) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + INSERT INTO tf_modules (name, root, description, configurator_url) + VALUES (module_name, module_root, module_description, module_configurator_url) + ON CONFLICT (name) DO UPDATE SET root = excluded.root, description = excluded.description, configurator_url = excluded.configurator_url +$$; + +-- Deletes a module; the foreign key of tf_instances refuses it while the module has instances. +CREATE OR REPLACE PROCEDURE tf_module_delete(module_name text) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + DELETE FROM tf_modules WHERE name = module_name +$$; + +-- The instances of a module, by name. +CREATE OR REPLACE FUNCTION tf_instances_list(module_name text) +RETURNS TABLE (module text, name text, state_schema text, created timestamptz) +LANGUAGE sql STABLE +SET search_path FROM CURRENT +AS $$ + SELECT i.module, i.name, i.state_schema, i.created FROM tf_instances i WHERE i.module = module_name ORDER BY i.name +$$; + +-- Creates an instance of a module with the schema of its state. +CREATE OR REPLACE PROCEDURE tf_instance_create(module_name text, instance_name text, instance_state_schema text) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + INSERT INTO tf_instances (module, name, state_schema) VALUES (module_name, instance_name, instance_state_schema) +$$; + +-- Deletes an instance with its variable files. +CREATE OR REPLACE PROCEDURE tf_instance_delete(module_name text, instance_name text) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + DELETE FROM tf_instances WHERE module = module_name AND name = instance_name +$$; + +-- The variable files of an instance, by name: content, whether the content it replaced is kept, and the time of the +-- last write. +CREATE OR REPLACE FUNCTION tf_variable_files_read(module_name text, instance_name text) +RETURNS TABLE (name text, content jsonb, previous boolean, updated timestamptz) +LANGUAGE sql STABLE +SET search_path FROM CURRENT +AS $$ + SELECT f.name, f.content, f.previous IS NOT NULL, f.updated FROM tf_variable_files f + WHERE f.module = module_name AND f.instance = instance_name ORDER BY f.name +$$; + +-- Keeps other writers out until the end of the transaction, from the comparison of the version to the commit. +CREATE OR REPLACE PROCEDURE tf_variable_files_lock() +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + LOCK TABLE tf_variable_files IN EXCLUSIVE MODE +$$; + +-- Writes a variable file of an instance, keeping the content it replaces as previous. +CREATE OR REPLACE PROCEDURE tf_variable_file_put(module_name text, instance_name text, file_name text, file_content jsonb) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + INSERT INTO tf_variable_files (module, instance, name, content) VALUES (module_name, instance_name, file_name, file_content) + ON CONFLICT (module, instance, name) DO UPDATE SET previous = tf_variable_files.content, content = excluded.content, updated = now() +$$; + +-- Deletes a variable file of an instance. +CREATE OR REPLACE PROCEDURE tf_variable_file_delete(module_name text, instance_name text, file_name text) +LANGUAGE sql +SET search_path FROM CURRENT +AS $$ + DELETE FROM tf_variable_files WHERE module = module_name AND instance = instance_name AND name = file_name +$$; diff --git a/server/tfvarsStore.test.ts b/server/tfvarsStore.test.ts new file mode 100644 index 0000000..221321b --- /dev/null +++ b/server/tfvarsStore.test.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict' +import { after, before, test } from 'node:test' +import pg from 'pg' +import { HttpError } from './http.ts' +import { createTfvarsStore, tfvarsText, type TfvarsStore } from './tfvarsStore.ts' + +// The tests create a scratch database on the server the PG* variables name, connected to through the database +// postgres, and drop it at the end. +const database = `tfvars_store_test_${process.pid}` +const admin = new pg.Client({ database: 'postgres' }) +let pool: pg.Pool +let store: TfvarsStore + +before(async () => { + await admin.connect() + await admin.query(`CREATE DATABASE ${database}`) + pool = new pg.Pool({ database }) + store = createTfvarsStore(pool) +}) + +after(async () => { + await pool?.end() + await admin.query(`DROP DATABASE IF EXISTS ${database}`) + await admin.end() +}) + +const status = async (promise: Promise) => { + try { + await promise + } catch (error) { + assert.ok(error instanceof HttpError, String(error)) + return error.statusCode + } + return 0 +} + +test('modules are created, changed, listed and deleted', async () => { + await store.modules.put({ name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: null }) + await store.modules.put({ name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: 'http://localhost:7083/' }) + await store.modules.put({ name: 'app', root: 'app', description: '', configuratorUrl: null }) + assert.deepEqual(await store.modules.list(), [ + { name: 'app', root: 'app', description: '', configuratorUrl: null }, + { name: 'router', root: 'router/terraform', description: 'VPN router', configuratorUrl: 'http://localhost:7083/' }, + ]) + await store.modules.delete('app') + assert.deepEqual((await store.modules.list()).map((module) => module.name), ['router']) + assert.equal(await status(store.modules.put({ name: 'Bad', root: 'x', description: '', configuratorUrl: null })), 400) +}) + +test('instances are created and refused for a missing module or a name in use', async () => { + await store.instances.create('router', 'platform', 'terraform_remote_state') + await store.instances.create('router', 'test', 'tf_router_test') + assert.deepEqual((await store.instances.list('router')).map((instance) => [instance.name, instance.stateSchema]), [ + ['platform', 'terraform_remote_state'], + ['test', 'tf_router_test'], + ]) + assert.equal(await status(store.instances.create('missing', 'x', 'tf_missing_x')), 404) + assert.equal(await status(store.instances.create('router', 'test', 'tf_other')), 409) + assert.equal(await status(store.instances.create('router', 'other', 'tf_router_test')), 409) + assert.equal(await status(store.modules.delete('router')), 409) +}) + +test('variable files are written with their version and refused against another version', async () => { + const variables = store.variables('router', 'test') + const empty = await variables.read() + assert.deepEqual(empty.files, []) + assert.equal(empty.version, '') + assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 1 }), 'secrets.tfvars.json': tfvarsText({ s: 'x' }) }), true) + const written = await variables.read() + assert.deepEqual(written.files.map((file) => [file.name, file.content.toString('utf8')]), [ + ['platform.tfvars.json', tfvarsText({ a: 1 })], + ['secrets.tfvars.json', tfvarsText({ s: 'x' })], + ]) + assert.deepEqual(written.status.map((file) => file.previous), [false, false]) + assert.equal(await variables.write(empty.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), false) + assert.equal(await variables.write(written.version, { 'platform.tfvars.json': tfvarsText({ a: 2 }) }), true) + const changed = await variables.read() + assert.deepEqual(changed.status.map((file) => file.previous), [true, false]) + assert.notEqual(changed.version, written.version) + assert.equal(await variables.remove(written.version, 'secrets.tfvars.json'), false) + assert.equal(await variables.remove(changed.version, 'secrets.tfvars.json'), true) + assert.deepEqual((await variables.read()).files.map((file) => file.name), ['platform.tfvars.json']) + assert.deepEqual((await store.variables('router', 'platform').read()).files, []) + assert.equal(await status(variables.write((await variables.read()).version, { 'notes.json': tfvarsText({}) })), 400) + assert.equal(await status(store.variables('router', 'missing').write('', { 'a.tfvars.json': tfvarsText({}) })), 404) +}) + +test('an instance is deleted with its variable files, then its module', async () => { + await store.instances.delete('router', 'test') + await store.instances.delete('router', 'platform') + assert.deepEqual(await store.instances.list('router'), []) + assert.equal((await pool.query('SELECT count(*)::int AS n FROM tf_variable_files')).rows[0].n, 0) + await store.modules.delete('router') + assert.deepEqual(await store.modules.list(), []) +}) diff --git a/server/tfvarsStore.ts b/server/tfvarsStore.ts new file mode 100644 index 0000000..7733430 --- /dev/null +++ b/server/tfvarsStore.ts @@ -0,0 +1,158 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import type { Pool, PoolClient } from 'pg' +import type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts' +import { HttpError } from './http.ts' + +/** A variable file of an instance: its name and its content, the object jsonb keeps written as JSON text. */ +export type TfvarsFile = { name: string; content: Buffer } + +/** The variable files of an instance, by name, with their version, versionOf the files. */ +export type TfvarsDocument = { files: TfvarsFile[]; status: TfvarsFileStatus[]; version: string } + +/** The variable files of one instance. */ +export type InstanceVariables = { + read(): Promise + /** + * Writes the given files, name and JSON text of an object, when the files are still those of version, keeping each + * replaced content as its previous one; answers false, writing nothing, when they changed since. + */ + write(version: string, contents: Record): Promise + /** Deletes the file name when the files are still those of version; answers false, deleting nothing, otherwise. */ + remove(version: string, name: string): Promise +} + +/** The modules, the instances and their variable files in PostgreSQL. */ +export type TfvarsStore = { + modules: { + list(): Promise + /** Creates the module, or changes the one of its name. */ + put(module: TfModule): Promise + /** Deletes the module; HttpError 409 while it has instances. */ + delete(name: string): Promise + } + instances: { + list(module: string): Promise + /** HttpError 409 for a name or a state schema in use, 404 for a module that does not exist. */ + create(module: string, name: string, stateSchema: string): Promise + /** Deletes the instance with its variable files. */ + delete(module: string, name: string): Promise + } + variables(module: string, instance: string): InstanceVariables +} + +/** The text of a variable file as both stores write it: the object as JSON with two spaces, and a final newline. */ +export function tfvarsText(content: unknown): string { + return `${JSON.stringify(content, null, 2)}\n` +} + +/** Identifies the variable files: the name and the SHA-256 of the content of each, in name order. */ +export function versionOf(files: TfvarsFile[]): string { + return [...files].sort((a, b) => a.name.localeCompare(b.name)) + .map((file) => `${file.name}=${createHash('sha256').update(file.content).digest('hex')}`).join(':') +} + +const SQL = new URL('./sql/tfvars_store.sql', import.meta.url) + +type FileRow = { name: string; content: object; previous: boolean; updated: Date } + +/** + * The store of sql/tfvars_store.sql in the database of pool, applied at the first use in the schema of the + * connection's search path. A refused change is an HttpError: 409 for a name in use or a module that has instances, + * 404 for a module or an instance that does not exist, 400 for a value the tables do not accept. + */ +export function createTfvarsStore(pool: Pool): TfvarsStore { + let created: Promise | null = null + const ready = () => (created ??= pool.query(readFileSync(SQL, 'utf8'))) + const query = async (text: string, values: unknown[]): Promise => { + await ready() + try { + return (await pool.query(text, values)).rows + } catch (error) { + throw refused(error) + } + } + const document = (rows: FileRow[]): TfvarsDocument => { + const files = rows.map((row) => ({ name: row.name, content: Buffer.from(tfvarsText(row.content), 'utf8') })) + return { + files, + status: rows.map((row, index) => ({ name: row.name, bytes: files[index].content.length, modified: row.updated.toISOString(), previous: row.previous })), + version: versionOf(files), + } + } + /** Runs change in a transaction that holds the lock of the variable files, when they are still those of version. */ + const locked = async (module: string, instance: string, version: string, change: (client: PoolClient) => Promise) => { + await ready() + const client = await pool.connect() + try { + await client.query('BEGIN') + await client.query('CALL tf_variable_files_lock()') + const { rows } = await client.query('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance]) + if (document(rows).version !== version) { + await client.query('ROLLBACK') + return false + } + await change(client) + await client.query('COMMIT') + return true + } catch (error) { + await client.query('ROLLBACK') + throw refused(error) + } finally { + client.release() + } + } + return { + modules: { + list: async () => (await query<{ name: string; root: string; description: string; configurator_url: string | null }>('SELECT * FROM tf_modules_list()', [])) + .map((row) => ({ name: row.name, root: row.root, description: row.description, configuratorUrl: row.configurator_url })), + async put(module) { + await query('CALL tf_module_put($1, $2, $3, $4)', [module.name, module.root, module.description, module.configuratorUrl]) + }, + async delete(name) { + await query('CALL tf_module_delete($1)', [name]) + }, + }, + instances: { + list: async (module) => (await query<{ module: string; name: string; state_schema: string; created: Date }>('SELECT * FROM tf_instances_list($1)', [module])) + .map((row) => ({ module: row.module, name: row.name, stateSchema: row.state_schema, created: row.created.toISOString() })), + async create(module, name, stateSchema) { + await query('CALL tf_instance_create($1, $2, $3)', [module, name, stateSchema]) + }, + async delete(module, name) { + await query('CALL tf_instance_delete($1, $2)', [module, name]) + }, + }, + variables: (module, instance) => ({ + read: async () => document(await query('SELECT * FROM tf_variable_files_read($1, $2)', [module, instance])), + write: (version, contents) => locked(module, instance, version, async (client) => { + for (const [name, text] of Object.entries(contents)) { + await client.query('CALL tf_variable_file_put($1, $2, $3, $4::jsonb)', [module, instance, name, text]) + } + }), + remove: (version, name) => locked(module, instance, version, async (client) => { + await client.query('CALL tf_variable_file_delete($1, $2, $3)', [module, instance, name]) + }), + }), + } +} + +/** The HttpError of an error of PostgreSQL that refuses a change, by its SQLSTATE; any other error as it is. */ +function refused(error: unknown): unknown { + const { code, constraint, detail } = error as { code?: string; constraint?: string; detail?: string } + const reason = detail ?? (error as Error).message + switch (code) { + case '23505': + return new HttpError(409, `The name is in use (${reason}).`) + case '23503': + if (reason.includes('is still referenced')) { + return new HttpError(409, `The module has instances; delete them first (${reason}).`) + } + return new HttpError(404, `The ${constraint === 'tf_instances_module_fkey' ? 'module' : 'instance'} does not exist (${reason}).`) + case '23514': + case '22P02': + return new HttpError(400, `The value is not allowed (${reason}).`) + default: + return error + } +} diff --git a/shared/tfvars.ts b/shared/tfvars.ts new file mode 100644 index 0000000..1841e0a --- /dev/null +++ b/shared/tfvars.ts @@ -0,0 +1,36 @@ +/** A registered Terraform module: a folder of Terraform code, relative to the modules folder of Cloud Platform. */ +export type TfModule = { + name: string + root: string + description: string + /** The address of the module's configurator, the application that edits the variables of its instances. */ + configuratorUrl: string | null +} + +/** A deployment of a module, whose state is kept in the schema stateSchema of the pg backend. */ +export type TfInstance = { + module: string + name: string + stateSchema: string + /** ISO 8601. */ + created: string +} + +/** A variable file of an instance, without its content. */ +export type TfvarsFileStatus = { + name: string + bytes: number + /** ISO 8601, the time of the last write. */ + modified: string + /** Whether the content the last write replaced is kept. */ + previous: boolean +} + +/** The names of modules and instances: a lower-case letter, then lower-case letters, digits and hyphens. */ +export const TF_NAME_PATTERN = /^[a-z][a-z0-9-]*$/ + +/** The names of variable files: letters, digits, '_', '.' and '-', ending in .tfvars.json. */ +export const TFVARS_FILE_PATTERN = /^[A-Za-z0-9_.-]+\.tfvars\.json$/ + +/** The names of pg schemas of a state: a lower-case letter or '_', then lower-case letters, digits and '_'. */ +export const STATE_SCHEMA_PATTERN = /^[a-z_][a-z0-9_]*$/ diff --git a/tsconfig.build.json b/tsconfig.build.json index 219d89f..5d97adb 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -6,5 +6,6 @@ "outDir": "dist", "declaration": true, "rewriteRelativeImportExtensions": true - } + }, + "exclude": ["dist", "**/*.test.ts"] } diff --git a/web/index.ts b/web/index.ts index 6ce86b0..11ce470 100644 --- a/web/index.ts +++ b/web/index.ts @@ -1,4 +1,6 @@ export type { CredentialField, CredentialKind, CredentialProfile, CredentialsDocument } from '../shared/credentials.ts' +export type { TfInstance, TfModule, TfvarsFileStatus } from '../shared/tfvars.ts' +export { STATE_SCHEMA_PATTERN, TFVARS_FILE_PATTERN, TF_NAME_PATTERN } from '../shared/tfvars.ts' export { ApiError, request, type Validation } from './api.ts' export { createBrowserSettings } from './browserSettings.ts' export { default as Card } from './Card.tsx'