aboviq/emigrate
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(postgres): implement the first version of the PostgreSQL plugin

Browse source
This commit is contained in:
Joakim Carlstein 2023-12-19 13:18:56 +01:00 • committed by Joakim Carlstein
parent 3d34b8ba13
commit 17c4723bb8
16 changed files with 761 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/cli/src/cli.ts
View file

@ -56,7 +56,7 @@ Options:
Examples:
emigrate up --directory src/migrations -s fs
emigrate up -d ./migrations --storage @emigrate/storage-mysql
emigrate up -d ./migrations --storage @emigrate/mysql
emigrate up -d src/migrations -s postgres -r json --dry
`;

4
packages/mysql/README.md
View file

@ -1,6 +1,6 @@
# @emigrate/storage-mysql
# @emigrate/mysql
A MySQL plugin for Emigrate. Uses a MySQL database for storing migration history. Can load and generate .sql migration files.
A MySQL plugin for Emigrate. Uses a MySQL database for storing the migration history. Can load and generate .sql migration files.
The table used for storing the migration history is compatible with the [immigration-mysql](https://github.com/joakimbeng/immigration-mysql) package, so you can use this together with the [@emigrate/cli](../cli) as a drop-in replacement for that package.

177
packages/postgres/README.md Normal file
View file

@ -0,0 +1,177 @@
# @emigrate/postgres
A PostgreSQL plugin for Emigrate. Uses a PostgreSQL database for storing the migration history. Can load and generate .sql migration files.
The table used for storing the migration history is compatible with the [immigration-postgres](https://github.com/aboviq/immigration-postgres) package, so you can use this together with the [@emigrate/cli](../cli) as a drop-in replacement for that package.
## Description
This plugin is actually three different Emigrate plugins in one:
1. A [storage plugin](#using-the-storage-plugin) for storing the migration history in a PostgreSQL database.
2. A [loader plugin](#using-the-loader-plugin) for loading .sql migration files and be able to execute them as part of the migration process.
3. A [generator plugin](#using-the-generator-plugin) for generating .sql migration files.
## Installation
Install the plugin in your project, alongside the Emigrate CLI:
```bash
npm install --save-dev @emigrate/cli @emigrate/postgres
```
## Usage
### Using the storage plugin
See [Options](#options) below for the default values and how to configure the plugin using environment variables.
Configure the storage in your `emigrate.config.js` file:
```js
export default {
directory: 'migrations',
storage: 'postgres', // the @emigrate/ prefix is optional
};
```
Or use the CLI options `--storage` (or `-s`)
```bash
emigrate up --storage postgres # the @emigrate/ prefix is optional
```
#### Storage plugin with custom options
Configure the storage in your `emigrate.config.js` file by importing the `createPostgresStorage` function (see [Options](#options) for available options).
In this mode the plugin will _not_ use any of the environment variables for configuration.
```js
import { createPostgresStorage } from '@emigrate/postgres';
export default {
directory: 'migrations',
storage: createPostgresStorage({ table: 'migrations', connection: { ... } }), // All connection options are passed to postgres()
};
```
Or use the CLI option `--storage` (or `-s`) and use environment variables (see [Options](#options) for available variables).
```bash
POSTGRES_URL=postgres://user:pass@host/db emigrate up --storage postgres # the @emigrate/ prefix is optional
```
### Using the loader plugin
The loader plugin is used to transform .sql migration files into JavaScript functions that can be executed by the "up" command.
See [Options](#options) below for the default values and how to configure the plugin using environment variables.
Configure the loader in your `emigrate.config.js` file:
```js
export default {
directory: 'migrations',
plugins: ['postgres'], // the @emigrate/ prefix is optional
};
```
Or by importing the default export from the plugin:
```js
import postgresPlugin from '@emigrate/postgres';
export default {
directory: 'migrations',
plugins: [postgresPlugin],
};
```
**NOTE:** Using the root level `plugins` option will load the plugin for all commands, which means the [generator plugin](#using-the-generator-plugin) will be used by default for the "new" command as well. If you only want to use the loader plugin, use the `up.plugins` option instead:
```js
export default {
directory: 'migrations',
up: {
plugins: ['postgres'], // the @emigrate/ prefix is optional
// or:
plugins: [import('@emigrate/postgres')],
},
};
```
The loader plugin can also be loaded using the CLI option `--plugin` (or `-p`) together with the "up" command:
```bash
emigrate up --plugin postgres # the @emigrate/ prefix is optional
```
### Using the generator plugin
The generator plugin is used to generate skeleton .sql migration files inside your migration directory.
Configure the generator in your `emigrate.config.js` file:
```js
export default {
directory: 'migrations',
plugins: ['postgres'], // the @emigrate/ prefix is optional
};
```
Or by importing the default export from the plugin:
```js
import postgresPlugin from '@emigrate/postgres';
export default {
directory: 'migrations',
plugins: [postgresPlugin],
};
```
**NOTE:** Using the root level `plugins` option will load the plugin for all commands, which means the [loader plugin](#using-the-loader-plugin) will be used by default for the "up" command as well. If you only want to use the generator plugin, use the `new.plugins` option instead:
```js
export default {
directory: 'migrations',
new: {
plugins: ['postgres'], // the @emigrate/ prefix is optional
// or:
plugins: [import('@emigrate/postgres')],
},
};
```
The generator plugin can also be loaded using the CLI option `--plugin` (or `-p`) together with the "new" command:
```bash
emigrate new --plugin postgres My new migration file # the @emigrate/ prefix is optional
```
#### Loader plugin with custom options
Configure the loader in your `emigrate.config.js` file by importing the `createPostgresLoader` function (see [Options](#options) for available options).
In this mode the plugin will _not_ use any of the environment variables for configuration.
```js
import { createPostgresLoader } from '@emigrate/postgres';
export default {
directory: 'migrations',
plugins: [
createPostgresLoader({ connection: { ... } }), // All connection options are passed to postgres()
],
};
```
## Options
The storage plugin accepts the following options:
| Option | Applies to | Description | Default | Environment variable |
| ------------ | -------------------------- | -------------------------------------------------------------------------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------- |
| `table` | storage plugin | The name of the table to use for storing the migrations. | `migrations` | `POSTGRES_TABLE` |
| `connection` | storage and loader plugins | The connection options to pass to [`postgres()`](https://github.com/porsager/postgres#connection). | `{}` | `POSTGRES_URL` or `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` |

51
packages/postgres/package.json Normal file
View file

@ -0,0 +1,51 @@
{
"name": "@emigrate/postgres",
"version": "0.0.0",
"publishConfig": {
"access": "public"
},
"description": "A PostgreSQL plugin for Emigrate. Uses a PostgreSQL database for storing migration history. Can load and generate .sql migration files.",
"main": "dist/index.js",
"types": "dist/index.d.js",
"type": "module",
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
}
},
"files": [
"dist"
],
"scripts": {
"build": "tsc --pretty",
"build:watch": "tsc --pretty --watch",
"lint": "xo --cwd=../.. $(pwd)"
},
"keywords": [
"emigrate",
"emigrate-storage",
"emigrate-loader",
"emigrate-plugin",
"emigrate-generator",
"migrations",
"postgres",
"postgresql"
],
"author": "Aboviq AB <dev@aboviq.com> (https://www.aboviq.com)",
"homepage": "https://github.com/aboviq/emigrate/tree/main/packages/postgres#readme",
"repository": "https://github.com/aboviq/emigrate/tree/main/packages/postgres",
"bugs": "https://github.com/aboviq/emigrate/issues",
"license": "MIT",
"dependencies": {
"@emigrate/plugin-tools": "workspace:*",
"@emigrate/types": "workspace:*",
"postgres": "3.4.3"
},
"devDependencies": {
"@emigrate/tsconfig": "workspace:*"
},
"volta": {
"extends": "../../package.json"
}
}

252
packages/postgres/src/index.ts Normal file
View file

@ -0,0 +1,252 @@
import process from 'node:process';
import postgres, { type Options, type PostgresType, type Sql } from 'postgres';
import { getTimestampPrefix, sanitizeMigrationName } from '@emigrate/plugin-tools';
import {
type MigrationMetadata,
type EmigrateStorage,
type LoaderPlugin,
type Storage,
type MigrationMetadataFinished,
type GenerateMigrationFunction,
type GeneratorPlugin,
type SerializedError,
type MigrationHistoryEntry,
} from '@emigrate/types';
const defaultTable = 'migrations';
type ConnectionOptions = Options<Record<string, PostgresType>>;
export type PostgresStorageOptions = {
table?: string;
/**
* @see https://github.com/porsager/postgres#connection
*/
connection: ConnectionOptions | string;
};
export type PostgresLoaderOptions = {
/**
* @see https://github.com/porsager/postgres#connection
*/
connection: ConnectionOptions | string;
};
const getPool = (connection: ConnectionOptions | string) => {
if (typeof connection === 'string') {
return postgres(connection);
}
return postgres(connection);
};
const lockMigration = async (sql: Sql, table: string, migration: MigrationMetadata) => {
const result = await sql`
INSERT INTO ${sql(table)} (name, status, date)
VALUES (${migration.name}, ${'locked'}, NOW())
ON CONFLICT (name) DO NOTHING
`;
return result.count === 1;
};
const unlockMigration = async (sql: Sql, table: string, migration: MigrationMetadata) => {
const result = await sql`
DELETE FROM ${sql(table)}
WHERE
name = ${migration.name}
AND status = ${'locked'}
`;
return result.count === 1;
};
const finishMigration = async (
sql: Sql,
table: string,
migration: MigrationMetadataFinished,
_error?: SerializedError,
) => {
const result = await sql`
UPDATE
${sql(table)}
SET
status = ${migration.status},
date = NOW()
WHERE
name = ${migration.name}
AND status = ${'locked'}
`;
return result.count === 1;
};
const deleteMigration = async (sql: Sql, table: string, migration: MigrationMetadata) => {
const result = await sql`
DELETE FROM ${sql(table)}
WHERE
name = ${migration.name}
AND status <> ${'locked'}
`;
return result.count === 1;
};
const initializeTable = async (sql: Sql, table: string) => {
const [row] = await sql<Array<{ exists: 1 }>>`
SELECT 1 as exists
FROM
information_schema.tables
WHERE
table_schema = 'public'
AND table_name = ${table}
`;
if (row?.exists) {
return;
}
// This table definition is compatible with the one used by the immigration-postgres package
await sql`
CREATE TABLE ${sql(table)} (
name varchar(255) not null primary key,
status varchar(32),
date timestamptz not null
);
`;
};
export const createPostgresStorage = ({
table = defaultTable,
connection,
}: PostgresStorageOptions): EmigrateStorage => {
return {
async initializeStorage() {
const sql = getPool(connection);
try {
await initializeTable(sql, table);
} catch (error) {
await sql.end();
throw error;
}
const storage: Storage = {
async lock(migrations) {
const lockedMigrations: MigrationMetadata[] = [];
for await (const migration of migrations) {
if (await lockMigration(sql, table, migration)) {
lockedMigrations.push(migration);
}
}
return lockedMigrations;
},
async unlock(migrations) {
for await (const migration of migrations) {
await unlockMigration(sql, table, migration);
}
},
async remove(migration) {
await deleteMigration(sql, table, migration);
},
async *getHistory() {
const query = sql<Array<Exclude<MigrationHistoryEntry, 'error'>>>`
SELECT
*
FROM
${sql(table)}
WHERE
status <> ${'locked'}
ORDER BY
date ASC
`.cursor();
for await (const [row] of query) {
if (!row) {
continue;
}
if (row.status === 'failed') {
yield {
...row,
error: { name: 'Error', message: 'Unknown error' },
};
continue;
}
yield row;
}
},
async onSuccess(migration) {
await finishMigration(sql, table, migration);
},
async onError(migration, error) {
await finishMigration(sql, table, migration, error);
},
async end() {
await sql.end();
},
};
return storage;
},
};
};
export const { initializeStorage } = createPostgresStorage({
table: process.env['POSTGRES_TABLE'],
connection: process.env['POSTGRES_URL'] ?? {
host: process.env['POSTGRES_HOST'],
port: process.env['POSTGRES_PORT'] ? Number.parseInt(process.env['POSTGRES_PORT'], 10) : undefined,
user: process.env['POSTGRES_USER'],
password: process.env['POSTGRES_PASSWORD'],
database: process.env['POSTGRES_DB'],
},
});
export const createPostgresLoader = ({ connection }: PostgresLoaderOptions): LoaderPlugin => {
return {
loadableExtensions: ['.sql'],
async loadMigration(migration) {
return async () => {
const sql = getPool(connection);
try {
// @ts-expect-error The "simple" option is not documented, but it exists
await sql.file(migration.filePath, { simple: true });
} finally {
await sql.end();
}
};
},
};
};
export const { loadableExtensions, loadMigration } = createPostgresLoader({
connection: process.env['POSTGRES_URL'] ?? {
host: process.env['POSTGRES_HOST'],
port: process.env['POSTGRES_PORT'] ? Number.parseInt(process.env['POSTGRES_PORT'], 10) : undefined,
user: process.env['POSTGRES_USER'],
password: process.env['POSTGRES_PASSWORD'],
database: process.env['POSTGRES_DB'],
},
});
export const generateMigration: GenerateMigrationFunction = async (name) => {
return {
filename: `${getTimestampPrefix()}_${sanitizeMigrationName(name)}.sql`,
content: `-- Migration: ${name}
`,
};
};
const defaultExport: EmigrateStorage & LoaderPlugin & GeneratorPlugin = {
initializeStorage,
loadableExtensions,
loadMigration,
generateMigration,
};
export default defaultExport;

8
packages/postgres/tsconfig.json Normal file
View file

@ -0,0 +1,8 @@
{
"extends": "@emigrate/tsconfig/build.json",
"compilerOptions": {
"outDir": "dist"
},
"include": ["src"],
"exclude": ["node_modules", "dist"]
}