feat(mysql): implement the first version of the MySQL plugin package
This commit is contained in:
parent
a8db22680e
commit
334e2099bb
6 changed files with 604 additions and 0 deletions
177
packages/mysql/README.md
Normal file
177
packages/mysql/README.md
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
# @emigrate/storage-mysql
|
||||
|
||||
A MySQL plugin for Emigrate. Uses a MySQL database for storing migration history. Can load and generate .sql migration files.
|
||||
|
||||
The table used for storing the migration history is compatible with the [immigration-mysql](https://github.com/joakimbeng/immigration-mysql) package, so you can use this together with the [@emigrate/cli](../cli) as a drop-in replacement for that package.
|
||||
|
||||
## Description
|
||||
|
||||
This plugin is actually three different Emigrate plugins in one:
|
||||
|
||||
1. A [storage plugin](#using-the-storage-plugin) for storing the migration history in a MySQL database.
|
||||
2. A [loader plugin](#using-the-loader-plugin) for loading .sql migration files and be able to execute them as part of the migration process.
|
||||
3. A [generator plugin](#using-the-generator-plugin) for generating .sql migration files.
|
||||
|
||||
## Installation
|
||||
|
||||
Install the plugin in your project, alongside the Emigrate CLI:
|
||||
|
||||
```bash
|
||||
npm install --save-dev @emigrate/cli @emigrate/mysql
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Using the storage plugin
|
||||
|
||||
See [Options](#options) below for the default values and how to configure the plugin using environment variables.
|
||||
|
||||
Configure the storage in your `emigrate.config.js` file:
|
||||
|
||||
```js
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
storage: 'mysql', // the @emigrate/ prefix is optional
|
||||
};
|
||||
```
|
||||
|
||||
Or use the CLI options `--storage` (or `-s`)
|
||||
|
||||
```bash
|
||||
emigrate up --storage mysql # the @emigrate/ prefix is optional
|
||||
```
|
||||
|
||||
#### Storage plugin with custom options
|
||||
|
||||
Configure the storage in your `emigrate.config.js` file by importing the `createMysqlStorage` function (see [Options](#options) for available options).
|
||||
|
||||
In this mode the plugin will _not_ use any of the environment variables for configuration.
|
||||
|
||||
```js
|
||||
import { createMysqlStorage } from '@emigrate/mysql';
|
||||
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
storage: createMysqlStorage({ table: 'migrations', connection: { ... } }), // All connection options are passed to mysql.createConnection()
|
||||
};
|
||||
```
|
||||
|
||||
Or use the CLI option `--storage` (or `-s`) and use environment variables (see [Options](#options) for available variables).
|
||||
|
||||
```bash
|
||||
MYSQL_URL=mysql://user:pass@host/db emigrate up --storage mysql # the @emigrate/ prefix is optional
|
||||
```
|
||||
|
||||
### Using the loader plugin
|
||||
|
||||
The loader plugin is used to transform .sql migration files into JavaScript functions that can be executed by the "up" command.
|
||||
|
||||
See [Options](#options) below for the default values and how to configure the plugin using environment variables.
|
||||
|
||||
Configure the loader in your `emigrate.config.js` file:
|
||||
|
||||
```js
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
plugins: ['mysql'], // the @emigrate/ prefix is optional
|
||||
};
|
||||
```
|
||||
|
||||
Or by importing the default export from the plugin:
|
||||
|
||||
```js
|
||||
import mysqlPlugin from '@emigrate/mysql';
|
||||
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
plugins: [mysqlPlugin],
|
||||
};
|
||||
```
|
||||
|
||||
**NOTE:** Using the root level `plugins` option will load the plugin for all commands, which means the [generator plugin](#using-the-generator-plugin) will be used by default for the "new" command as well. If you only want to use the loader plugin, use the `up.plugins` option instead:
|
||||
|
||||
```js
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
up: {
|
||||
plugins: ['mysql'], // the @emigrate/ prefix is optional
|
||||
// or:
|
||||
plugins: [import('@emigrate/mysql')],
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
The loader plugin can also be loaded using the CLI option `--plugin` (or `-p`) together with the "up" command:
|
||||
|
||||
```bash
|
||||
emigrate up --plugin mysql # the @emigrate/ prefix is optional
|
||||
```
|
||||
|
||||
### Using the generator plugin
|
||||
|
||||
The generator plugin is used to generate skeleton .sql migration files inside your migration directory.
|
||||
|
||||
Configure the generator in your `emigrate.config.js` file:
|
||||
|
||||
```js
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
plugins: ['mysql'], // the @emigrate/ prefix is optional
|
||||
};
|
||||
```
|
||||
|
||||
Or by importing the default export from the plugin:
|
||||
|
||||
```js
|
||||
import mysqlPlugin from '@emigrate/mysql';
|
||||
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
plugins: [mysqlPlugin],
|
||||
};
|
||||
```
|
||||
|
||||
**NOTE:** Using the root level `plugins` option will load the plugin for all commands, which means the [loader plugin](#using-the-loader-plugin) will be used by default for the "up" command as well. If you only want to use the generator plugin, use the `new.plugins` option instead:
|
||||
|
||||
```js
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
new: {
|
||||
plugins: ['mysql'], // the @emigrate/ prefix is optional
|
||||
// or:
|
||||
plugins: [import('@emigrate/mysql')],
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
The generator plugin can also be loaded using the CLI option `--plugin` (or `-p`) together with the "new" command:
|
||||
|
||||
```bash
|
||||
emigrate new --plugin mysql My new migration file # the @emigrate/ prefix is optional
|
||||
```
|
||||
|
||||
#### Loader plugin with custom options
|
||||
|
||||
Configure the loader in your `emigrate.config.js` file by importing the `createMysqlLoader` function (see [Options](#options) for available options).
|
||||
|
||||
In this mode the plugin will _not_ use any of the environment variables for configuration.
|
||||
|
||||
```js
|
||||
import { createMysqlLoader } from '@emigrate/mysql';
|
||||
|
||||
export default {
|
||||
directory: 'migrations',
|
||||
plugins: [
|
||||
createMysqlLoader({ connection: { ... } }), // All connection options are passed to mysql.createConnection()
|
||||
],
|
||||
};
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
The storage plugin accepts the following options:
|
||||
|
||||
| Option | Applies to | Description | Default | Environment variable |
|
||||
| ------------ | -------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------ | ---------------------------------------------------------------------------------------------- |
|
||||
| `table` | storage plugin | The name of the table to use for storing the migrations. | `migrations` | `MYSQL_TABLE` |
|
||||
| `connection` | storage and loader plugins | The connection options to pass to [`mysql.createConnection()`](https://github.com/mysqljs/mysql#connection-options). | `{}` | `MYSQL_URL` or `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_USER`, `MYSQL_PASSWORD` and `MYSQL_DATABASE` |
|
||||
49
packages/mysql/package.json
Normal file
49
packages/mysql/package.json
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{
|
||||
"name": "@emigrate/mysql",
|
||||
"version": "0.0.0",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"description": "A MySQL plugin for Emigrate. Uses a MySQL database for storing migration history. Can load and generate .sql migration files.",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.js",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc --pretty",
|
||||
"build:watch": "tsc --pretty --watch",
|
||||
"lint": "xo --cwd=../.. $(pwd)"
|
||||
},
|
||||
"keywords": [
|
||||
"emigrate",
|
||||
"emigrate-storage",
|
||||
"emigrate-loader",
|
||||
"emigrate-plugin",
|
||||
"emigrate-generator",
|
||||
"migrations",
|
||||
"mysql"
|
||||
],
|
||||
"author": "Aboviq AB <dev@aboviq.com> (https://www.aboviq.com)",
|
||||
"homepage": "https://github.com/aboviq/emigrate/tree/main/packages/mysql#readme",
|
||||
"repository": "https://github.com/aboviq/emigrate/tree/main/packages/mysql",
|
||||
"bugs": "https://github.com/aboviq/emigrate/issues",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@emigrate/plugin-tools": "workspace:*",
|
||||
"mysql2": "3.6.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@emigrate/tsconfig": "workspace:*"
|
||||
},
|
||||
"volta": {
|
||||
"extends": "../../package.json"
|
||||
}
|
||||
}
|
||||
286
packages/mysql/src/index.ts
Normal file
286
packages/mysql/src/index.ts
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
import process from 'node:process';
|
||||
import fs from 'node:fs/promises';
|
||||
import {
|
||||
createConnection,
|
||||
createPool,
|
||||
escapeId,
|
||||
type ConnectionOptions,
|
||||
type PoolOptions,
|
||||
type Pool,
|
||||
type ResultSetHeader,
|
||||
type RowDataPacket,
|
||||
} from 'mysql2/promise';
|
||||
import {
|
||||
type MigrationMetadata,
|
||||
type EmigrateStorage,
|
||||
type LoaderPlugin,
|
||||
type Storage,
|
||||
type MigrationStatus,
|
||||
type MigrationMetadataFinished,
|
||||
type GenerateMigrationFunction,
|
||||
type GeneratorPlugin,
|
||||
} from '@emigrate/plugin-tools/types';
|
||||
import { getTimestampPrefix, sanitizeMigrationName } from '@emigrate/plugin-tools';
|
||||
|
||||
const defaultTable = 'migrations';
|
||||
|
||||
export type MysqlStorageOptions = {
|
||||
table?: string;
|
||||
/**
|
||||
* @see https://github.com/mysqljs/mysql#connection-options
|
||||
*/
|
||||
connection: PoolOptions | string;
|
||||
};
|
||||
|
||||
export type MysqlLoaderOptions = {
|
||||
/**
|
||||
* @see https://github.com/mysqljs/mysql#connection-options
|
||||
*/
|
||||
connection: ConnectionOptions | string;
|
||||
};
|
||||
|
||||
const getConnection = async (connection: ConnectionOptions | string) => {
|
||||
if (typeof connection === 'string') {
|
||||
const uri = new URL(connection);
|
||||
|
||||
// client side connectTimeout is unstable in mysql2 library
|
||||
// it throws an error you can't catch and crashes node
|
||||
// best to leave this at 0 (disabled)
|
||||
uri.searchParams.set('connectTimeout', '0');
|
||||
uri.searchParams.set('multipleStatements', 'true');
|
||||
|
||||
return createConnection(uri.toString());
|
||||
}
|
||||
|
||||
return createConnection({
|
||||
...connection,
|
||||
// client side connectTimeout is unstable in mysql2 library
|
||||
// it throws an error you can't catch and crashes node
|
||||
// best to leave this at 0 (disabled)
|
||||
connectTimeout: 0,
|
||||
multipleStatements: true,
|
||||
});
|
||||
};
|
||||
|
||||
const getPool = (connection: PoolOptions | string) => {
|
||||
if (typeof connection === 'string') {
|
||||
const uri = new URL(connection);
|
||||
|
||||
// client side connectTimeout is unstable in mysql2 library
|
||||
// it throws an error you can't catch and crashes node
|
||||
// best to leave this at 0 (disabled)
|
||||
uri.searchParams.set('connectTimeout', '0');
|
||||
|
||||
return createPool(uri.toString());
|
||||
}
|
||||
|
||||
return createPool({
|
||||
...connection,
|
||||
// client side connectTimeout is unstable in mysql2 library
|
||||
// it throws an error you can't catch and crashes node
|
||||
// best to leave this at 0 (disabled)
|
||||
connectTimeout: 0,
|
||||
});
|
||||
};
|
||||
|
||||
type HistoryEntry = {
|
||||
name: string;
|
||||
status: MigrationStatus;
|
||||
date: Date;
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
const lockMigration = async (pool: Pool, table: string, migration: MigrationMetadata) => {
|
||||
const [result] = await pool.execute<ResultSetHeader>({
|
||||
sql: `
|
||||
INSERT INTO ${escapeId(table)} (name, status, date)
|
||||
VALUES (?, ?, NOW())
|
||||
ON DUPLICATE KEY UPDATE name = name
|
||||
`,
|
||||
values: [migration.name, 'locked'],
|
||||
});
|
||||
|
||||
return result.affectedRows === 1;
|
||||
};
|
||||
|
||||
const unlockMigration = async (pool: Pool, table: string, migration: MigrationMetadata) => {
|
||||
const [result] = await pool.execute<ResultSetHeader>({
|
||||
sql: `
|
||||
DELETE FROM ${escapeId(table)}
|
||||
WHERE
|
||||
name = ?
|
||||
AND status = ?
|
||||
`,
|
||||
values: [migration.name, 'locked'],
|
||||
});
|
||||
|
||||
return result.affectedRows === 1;
|
||||
};
|
||||
|
||||
const finishMigration = async (pool: Pool, table: string, migration: MigrationMetadataFinished) => {
|
||||
const [result] = await pool.execute<ResultSetHeader>({
|
||||
sql: `
|
||||
UPDATE
|
||||
${escapeId(table)}
|
||||
SET
|
||||
status = ?,
|
||||
date = NOW()
|
||||
WHERE
|
||||
name = ?
|
||||
AND status = ?
|
||||
`,
|
||||
values: [migration.status, migration.name, 'locked'],
|
||||
});
|
||||
|
||||
return result.affectedRows === 1;
|
||||
};
|
||||
|
||||
const deleteMigration = async (pool: Pool, table: string, migration: MigrationMetadata) => {
|
||||
const [result] = await pool.execute<ResultSetHeader>({
|
||||
sql: `
|
||||
DELETE FROM ${escapeId(table)}
|
||||
WHERE
|
||||
name = ?
|
||||
AND status <> ?
|
||||
`,
|
||||
values: [migration.name, 'locked'],
|
||||
});
|
||||
|
||||
return result.affectedRows === 1;
|
||||
};
|
||||
|
||||
const initializeTable = async (pool: Pool, table: string) => {
|
||||
// This table definition is compatible with the one used by the immigration-mysql package
|
||||
await pool.execute(`
|
||||
CREATE TABLE IF NOT EXISTS ${escapeId(table)} (
|
||||
name varchar(255) not null primary key,
|
||||
status varchar(32),
|
||||
date datetime not null
|
||||
) Engine=InnoDB;
|
||||
`);
|
||||
};
|
||||
|
||||
export const createMysqlStorage = ({ table = defaultTable, connection }: MysqlStorageOptions): EmigrateStorage => {
|
||||
return {
|
||||
async initializeStorage() {
|
||||
const pool = getPool(connection);
|
||||
|
||||
try {
|
||||
await initializeTable(pool, table);
|
||||
|
||||
const storage: Storage = {
|
||||
async lock(migrations) {
|
||||
const lockedMigrations: MigrationMetadata[] = [];
|
||||
|
||||
for await (const migration of migrations) {
|
||||
if (await lockMigration(pool, table, migration)) {
|
||||
lockedMigrations.push(migration);
|
||||
}
|
||||
}
|
||||
|
||||
return lockedMigrations;
|
||||
},
|
||||
async unlock(migrations) {
|
||||
for await (const migration of migrations) {
|
||||
await unlockMigration(pool, table, migration);
|
||||
}
|
||||
},
|
||||
async remove(migration) {
|
||||
await deleteMigration(pool, table, migration);
|
||||
},
|
||||
async *getHistory() {
|
||||
const [rows] = await pool.execute<Array<RowDataPacket & HistoryEntry>>({
|
||||
sql: `
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
${escapeId(table)}
|
||||
WHERE
|
||||
status <> ?
|
||||
ORDER BY
|
||||
date ASC
|
||||
`,
|
||||
values: ['locked'],
|
||||
});
|
||||
|
||||
for (const row of rows) {
|
||||
yield {
|
||||
name: row.name,
|
||||
status: row.status,
|
||||
date: new Date(row.date),
|
||||
// FIXME: Migrate the migrations table to support the error column
|
||||
error: row.status === 'failed' ? new Error('Unknown error reason') : undefined,
|
||||
};
|
||||
}
|
||||
},
|
||||
async onSuccess(migration) {
|
||||
await finishMigration(pool, table, migration);
|
||||
},
|
||||
async onError(migration, error) {
|
||||
await finishMigration(pool, table, { ...migration, status: 'failed', error });
|
||||
},
|
||||
};
|
||||
|
||||
return storage;
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const { initializeStorage } = createMysqlStorage({
|
||||
table: process.env['MYSQL_TABLE'],
|
||||
connection: process.env['MYSQL_URL'] ?? {
|
||||
host: process.env['MYSQL_HOST'],
|
||||
port: process.env['MYSQL_PORT'] ? Number.parseInt(process.env['MYSQL_PORT'], 10) : undefined,
|
||||
user: process.env['MYSQL_USER'],
|
||||
password: process.env['MYSQL_PASSWORD'],
|
||||
database: process.env['MYSQL_DATABASE'],
|
||||
},
|
||||
});
|
||||
|
||||
export const createMysqlLoader = ({ connection }: MysqlLoaderOptions): LoaderPlugin => {
|
||||
return {
|
||||
loadableExtensions: ['.sql'],
|
||||
async loadMigration(migration) {
|
||||
return async () => {
|
||||
const contents = await fs.readFile(migration.filePath, 'utf8');
|
||||
const conn = await getConnection(connection);
|
||||
|
||||
try {
|
||||
await conn.query(contents);
|
||||
} finally {
|
||||
await conn.end();
|
||||
}
|
||||
};
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const { loadableExtensions, loadMigration } = createMysqlLoader({
|
||||
connection: process.env['MYSQL_URL'] ?? {
|
||||
host: process.env['MYSQL_HOST'],
|
||||
port: process.env['MYSQL_PORT'] ? Number.parseInt(process.env['MYSQL_PORT'], 10) : undefined,
|
||||
user: process.env['MYSQL_USER'],
|
||||
password: process.env['MYSQL_PASSWORD'],
|
||||
database: process.env['MYSQL_DATABASE'],
|
||||
},
|
||||
});
|
||||
|
||||
export const generateMigration: GenerateMigrationFunction = async (name) => {
|
||||
return {
|
||||
filename: `${getTimestampPrefix()}_${sanitizeMigrationName(name)}.sql`,
|
||||
content: `-- Migration: ${name}
|
||||
`,
|
||||
};
|
||||
};
|
||||
|
||||
const defaultExport: EmigrateStorage & LoaderPlugin & GeneratorPlugin = {
|
||||
initializeStorage,
|
||||
loadableExtensions,
|
||||
loadMigration,
|
||||
generateMigration,
|
||||
};
|
||||
|
||||
export default defaultExport;
|
||||
8
packages/mysql/tsconfig.json
Normal file
8
packages/mysql/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"extends": "@emigrate/tsconfig/build.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue