From d8a6a2428ac86bdcf580177521a9706f37bff081 Mon Sep 17 00:00:00 2001 From: Joakim Carlstein Date: Fri, 24 Nov 2023 15:45:17 +0100 Subject: [PATCH] feat(remove): implement the "remove" command for removing migrations from the history --- .changeset/silly-walls-raise.md | 7 ++ packages/cli/src/cli.ts | 86 +++++++++++++++++++- packages/cli/src/commands/remove.ts | 110 ++++++++++++++++++++++++++ packages/cli/src/commands/up.ts | 8 +- packages/cli/src/errors.ts | 19 +++++ packages/cli/src/get-config.ts | 2 +- packages/cli/src/get-duration.ts | 6 ++ packages/cli/src/get-migration.ts | 42 ++++++++++ packages/cli/src/reporters/default.ts | 27 ++++++- packages/cli/src/types.ts | 1 + packages/plugin-tools/src/types.ts | 34 +++++++- packages/storage-fs/src/index.ts | 52 +++++++++--- 12 files changed, 370 insertions(+), 24 deletions(-) create mode 100644 .changeset/silly-walls-raise.md create mode 100644 packages/cli/src/commands/remove.ts create mode 100644 packages/cli/src/get-duration.ts create mode 100644 packages/cli/src/get-migration.ts diff --git a/.changeset/silly-walls-raise.md b/.changeset/silly-walls-raise.md new file mode 100644 index 0000000..d89dad5 --- /dev/null +++ b/.changeset/silly-walls-raise.md @@ -0,0 +1,7 @@ +--- +'@emigrate/plugin-tools': minor +'@emigrate/storage-fs': minor +'@emigrate/cli': minor +--- + +Implement the "remove" command for removing migration entries from the history diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index cc7f18a..f423486 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -123,6 +123,10 @@ const newMigration: Action = async (args) => { Create a new migration file with the given name in the specified directory +Arguments: + + name The name of the migration file to create (required) + Options: -h, --help Show this help message and exit @@ -239,9 +243,86 @@ Examples: } }; +const remove: Action = async (args) => { + const config = await getConfig('remove'); + const { values, positionals } = parseArgs({ + args, + options: { + help: { + type: 'boolean', + short: 'h', + }, + directory: { + type: 'string', + short: 'd', + }, + force: { + type: 'boolean', + short: 'f', + }, + reporter: { + type: 'string', + short: 'r', + }, + storage: { + type: 'string', + short: 's', + }, + }, + allowPositionals: true, + }); + + const usage = `Usage: emigrate remove [options] + +Remove entries from the migration history. +This is useful if you want to retry a migration that has failed. + +Arguments: + + name The name of the migration file to remove from the history (required) + +Options: + + -h, --help Show this help message and exit + -d, --directory The directory where the migration files are located (required) + -r, --reporter The reporter to use for reporting the removal process + -s, --storage The storage to use to get the migration history (required) + -f, --force Force removal of the migration history entry even if the migration file does not exist + or it's in a non-failed state + +Examples: + + emigrate remove -d migrations -s fs 20231122120529381_some_migration_file.js + emigrate remove --directory ./migrations --storage postgres 20231122120529381_some_migration_file.sql +`; + + if (values.help) { + console.log(usage); + process.exitCode = 1; + return; + } + + const { directory = config.directory, storage = config.storage, reporter = config.reporter, force } = values; + + try { + const { default: removeCommand } = await import('./commands/remove.js'); + await removeCommand({ directory, storage, reporter, force }, positionals[0] ?? ''); + } catch (error) { + if (error instanceof ShowUsageError) { + console.error(error.message, '\n'); + console.log(usage); + process.exitCode = 1; + return; + } + + throw error; + } +}; + const commands: Record = { up, list, + remove, new: newMigration, }; @@ -259,9 +340,10 @@ if (!action) { Commands: - up Run all pending migrations + up Run all pending migrations (or do a dry run) new Create a new migration file - list List all migrations + list List all migrations and their status + remove Remove entries from the migration history `); process.exit(1); } diff --git a/packages/cli/src/commands/remove.ts b/packages/cli/src/commands/remove.ts new file mode 100644 index 0000000..5aaccad --- /dev/null +++ b/packages/cli/src/commands/remove.ts @@ -0,0 +1,110 @@ +import process from 'node:process'; +import { getOrLoadReporter, getOrLoadStorage } from '@emigrate/plugin-tools'; +import { type MigrationHistoryEntry, type MigrationMetadataFinished } from '@emigrate/plugin-tools/types'; +import { + BadOptionError, + MigrationNotRunError, + MissingArgumentsError, + MissingOptionError, + OptionNeededError, +} from '../errors.js'; +import { type Config } from '../types.js'; +import { getMigration } from '../get-migration.js'; +import { getDuration } from '../get-duration.js'; + +type ExtraFlags = { + force?: boolean; +}; + +const lazyDefaultReporter = async () => import('../reporters/default.js'); + +export default async function removeCommand( + { directory, reporter: reporterConfig, storage: storageConfig, force }: Config & ExtraFlags, + name: string, +) { + if (!directory) { + throw new MissingOptionError('directory'); + } + + if (!name) { + throw new MissingArgumentsError('name'); + } + + const cwd = process.cwd(); + const storagePlugin = await getOrLoadStorage([storageConfig]); + + if (!storagePlugin) { + throw new BadOptionError('storage', 'No storage found, please specify a storage using the storage option'); + } + + const storage = await storagePlugin.initializeStorage(); + const reporter = await getOrLoadReporter([reporterConfig ?? lazyDefaultReporter]); + + if (!reporter) { + throw new BadOptionError( + 'reporter', + 'No reporter found, please specify an existing reporter using the reporter option', + ); + } + + const migrationFile = await getMigration(cwd, directory, name, !force); + + const finishedMigrations: MigrationMetadataFinished[] = []; + let historyEntry: MigrationHistoryEntry | undefined; + let removalError: Error | undefined; + + for await (const migrationHistoryEntry of storage.getHistory()) { + if (migrationHistoryEntry.name !== migrationFile.name) { + continue; + } + + if (migrationHistoryEntry.status === 'done' && !force) { + throw new OptionNeededError( + 'force', + `The migration "${migrationFile.name}" is not in a failed state. Use the "force" option to force its removal`, + ); + } + + historyEntry = migrationHistoryEntry; + } + + await reporter.onInit?.({ command: 'remove', cwd, dry: false, directory }); + + await reporter.onMigrationRemoveStart?.(migrationFile); + + const start = process.hrtime(); + + if (historyEntry) { + try { + await storage.remove(migrationFile); + + const duration = getDuration(start); + const finishedMigration: MigrationMetadataFinished = { ...migrationFile, status: 'done', duration }; + + await reporter.onMigrationRemoveSuccess?.(finishedMigration); + + finishedMigrations.push(finishedMigration); + } catch (error) { + removalError = error instanceof Error ? error : new Error(String(error)); + } + } else if (!removalError) { + removalError = new MigrationNotRunError( + `Migration "${migrationFile.name}" is not in the migration history`, + migrationFile, + ); + } + + if (removalError) { + const duration = getDuration(start); + const finishedMigration: MigrationMetadataFinished = { + ...migrationFile, + status: 'failed', + error: removalError, + duration, + }; + await reporter.onMigrationRemoveError?.(finishedMigration, removalError); + finishedMigrations.push(finishedMigration); + } + + await reporter.onFinished?.(finishedMigrations, removalError); +} diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts index 757c1da..1a37c71 100644 --- a/packages/cli/src/commands/up.ts +++ b/packages/cli/src/commands/up.ts @@ -18,16 +18,12 @@ import { import { type Config } from '../types.js'; import { withLeadingPeriod } from '../with-leading-period.js'; import { getMigrations } from '../get-migrations.js'; +import { getDuration } from '../get-duration.js'; type ExtraFlags = { dry?: boolean; }; -const getDuration = (start: [number, number]) => { - const [seconds, nanoseconds] = process.hrtime(start); - return seconds * 1000 + nanoseconds / 1_000_000; -}; - const lazyDefaultReporter = async () => import('../reporters/default.js'); const lazyPluginLoaderJs = async () => import('../plugin-loader-js.js'); @@ -81,7 +77,7 @@ export default async function upCommand({ relativeFilePath: path.relative(cwd, filePath), extension: withLeadingPeriod(path.extname(migrationHistoryEntry.name)), error: new MigrationHistoryError( - `Migration ${migrationHistoryEntry.name} is in a failed state, please fix it first`, + `Migration ${migrationHistoryEntry.name} is in a failed state, please fix and remove it first`, migrationHistoryEntry, ), directory, diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index a6ecdba..d9ab3ba 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -27,6 +27,15 @@ export class MissingArgumentsError extends ShowUsageError { } } +export class OptionNeededError extends ShowUsageError { + constructor( + public option: string, + message: string, + ) { + super('ERR_OPT_NEEDED', message); + } +} + export class BadOptionError extends ShowUsageError { constructor( public option: string, @@ -70,3 +79,13 @@ export class MigrationRunError extends EmigrateError { super('ERR_MIGRATION_RUN', message, options); } } + +export class MigrationNotRunError extends EmigrateError { + constructor( + message: string, + public metadata: MigrationMetadata, + options?: ErrorOptions, + ) { + super('ERR_MIGRATION_NOT_RUN', message, options); + } +} diff --git a/packages/cli/src/get-config.ts b/packages/cli/src/get-config.ts index b0bfafe..0d9dbd4 100644 --- a/packages/cli/src/get-config.ts +++ b/packages/cli/src/get-config.ts @@ -1,7 +1,7 @@ import { cosmiconfig } from 'cosmiconfig'; import { type Config, type EmigrateConfig } from './types.js'; -const commands = ['up', 'list', 'new'] as const; +const commands = ['up', 'list', 'new', 'remove'] as const; type Command = (typeof commands)[number]; export const getConfig = async (command: Command): Promise => { diff --git a/packages/cli/src/get-duration.ts b/packages/cli/src/get-duration.ts new file mode 100644 index 0000000..9d29d2f --- /dev/null +++ b/packages/cli/src/get-duration.ts @@ -0,0 +1,6 @@ +import process from 'node:process'; + +export const getDuration = (start: [number, number]) => { + const [seconds, nanoseconds] = process.hrtime(start); + return seconds * 1000 + nanoseconds / 1_000_000; +}; diff --git a/packages/cli/src/get-migration.ts b/packages/cli/src/get-migration.ts new file mode 100644 index 0000000..bdb801d --- /dev/null +++ b/packages/cli/src/get-migration.ts @@ -0,0 +1,42 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { type MigrationMetadata } from '@emigrate/plugin-tools/types'; +import { withLeadingPeriod } from './with-leading-period.js'; +import { OptionNeededError } from './errors.js'; + +const checkMigrationFile = async (name: string, filePath: string) => { + try { + const stats = await fs.stat(filePath); + + if (!stats.isFile()) { + throw new Error('Not a file'); + } + } catch { + throw new OptionNeededError( + 'force', + `The given migration name "${name}" does not exist or is not a file. Use the "force" option to ignore this error`, + ); + } +}; + +export const getMigration = async ( + cwd: string, + directory: string, + name: string, + requireExists = true, +): Promise => { + const filePath = path.resolve(cwd, directory, name); + + if (requireExists) { + await checkMigrationFile(name, filePath); + } + + return { + name, + filePath, + relativeFilePath: path.relative(cwd, filePath), + extension: withLeadingPeriod(path.extname(name)), + directory, + cwd, + }; +}; diff --git a/packages/cli/src/reporters/default.ts b/packages/cli/src/reporters/default.ts index c709de1..7cf3f5c 100644 --- a/packages/cli/src/reporters/default.ts +++ b/packages/cli/src/reporters/default.ts @@ -193,7 +193,7 @@ const getSummary = ( const statusLine = [ failed ? red.bold(`${failed} failed`) : '', - done ? green.bold(`${done} ${command === 'new' ? 'created' : 'done'}`) : '', + done ? green.bold(`${done} ${command === 'new' ? 'created' : command === 'remove' ? 'removed' : 'done'}`) : '', skipped ? yellow.bold(`${skipped} skipped`) : '', pending ? cyan.bold(`${pending} pending`) : '', ] @@ -267,6 +267,19 @@ class DefaultFancyReporter implements Required { this.#migrations = [migration]; } + onMigrationRemoveStart(migration: MigrationMetadata): Awaitable { + this.#migrations = [migration]; + this.#activeMigration = migration; + } + + onMigrationRemoveSuccess(migration: MigrationMetadataFinished): Awaitable { + this.#finishMigration(migration); + } + + onMigrationRemoveError(migration: MigrationMetadataFinished, _error: Error): Awaitable { + this.#finishMigration(migration); + } + onMigrationStart(migration: MigrationMetadata): void | PromiseLike { this.#activeMigration = migration; } @@ -372,6 +385,18 @@ class DefaultReporter implements Required { console.log(getMigrationText(migration)); } + onMigrationRemoveStart(migration: MigrationMetadata): Awaitable { + console.log(getMigrationText(migration)); + } + + onMigrationRemoveSuccess(migration: MigrationMetadataFinished): Awaitable { + console.log(getMigrationText(migration)); + } + + onMigrationRemoveError(migration: MigrationMetadataFinished, _error: Error): Awaitable { + console.error(getMigrationText(migration)); + } + onMigrationStart(migration: MigrationMetadata): void | PromiseLike { console.log(getMigrationText(migration, migration)); } diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 27daea4..1cb58db 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -17,4 +17,5 @@ export type EmigrateConfig = Config & { up?: Config; new?: Config; list?: Config; + remove?: Config; }; diff --git a/packages/plugin-tools/src/types.ts b/packages/plugin-tools/src/types.ts index 23d254c..5386b4b 100644 --- a/packages/plugin-tools/src/types.ts +++ b/packages/plugin-tools/src/types.ts @@ -32,6 +32,15 @@ export type Storage = { * @param migrations The previously successfully locked migrations that should now be unlocked. */ unlock(migrations: MigrationMetadata[]): Promise; + + /** + * Remove a migration from the history. + * + * This is used to remove a migration from the history which is needed for failed migrations to be re-executed. + * + * @param migration The migration that should be removed from the history. + */ + remove(migration: MigrationMetadata): Promise; /** * Get the history of previously executed migrations. * @@ -154,7 +163,7 @@ export type ReporterInitParameters = { /** * The command that is being executed */ - command: 'up' | 'new' | 'list'; + command: 'up' | 'new' | 'list' | 'remove'; /** * The directory where the migration files are located */ @@ -197,6 +206,24 @@ export type EmigrateReporter = Partial<{ * This is only called when the command is 'new'. */ onNewMigration(migration: MigrationMetadata, content: string): Awaitable; + /** + * Called when a migration is about to be removed from the migration history. + * + * This is only called when the command is 'remove'. + */ + onMigrationRemoveStart(migration: MigrationMetadata): Awaitable; + /** + * Called when a migration is successfully removed from the migration history. + * + * This is only called when the command is 'remove'. + */ + onMigrationRemoveSuccess(migration: MigrationMetadataFinished): Awaitable; + /** + * Called when a migration couldn't be removed from the migration history. + * + * This is only called when the command is 'remove'. + */ + onMigrationRemoveError(migration: MigrationMetadataFinished, error: Error): Awaitable; /** * Called when a migration is about to be executed. * @@ -230,15 +257,18 @@ export type EmigrateReporter = Partial<{ * Will be called when a migration is skipped because a previous migration failed, * it couldn't be successfully locked, or in case of a dry run when the command is "up". * When the command is "list" this will be called for each pending migration (i.e. those that have not run yet). + * When the command is "remove" this will be called when the removal of some migrations are skipped + * because the removal of a previous migration failed. * * @param migration Information about the migration that was skipped. */ onMigrationSkip(migration: MigrationMetadataFinished): Awaitable; /** - * Called as a final step after all migrations have been executed or listed. + * Called as a final step after all migrations have been executed, listed or removed. * * This is called either after all migrations have been listed successfully for the "list" command * or for the "up" command when they are executed successfully, at the end of a dry run, or when a migration has failed. + * It is also called after migrations have been removed from the history with the "remove" command. * It is also called after a migration file has been generated with the "new" command. * * @param migrations Information about all migrations that were executed or listed, their status and any error that occurred. diff --git a/packages/storage-fs/src/index.ts b/packages/storage-fs/src/index.ts index 36d93d4..9edc12d 100644 --- a/packages/storage-fs/src/index.ts +++ b/packages/storage-fs/src/index.ts @@ -55,31 +55,59 @@ export default function storageFs({ filename }: StorageFsOptions): EmigrateStora }, }; - await fs.writeFile(filePath, JSON.stringify(newHistory, undefined, 2)); + try { + await fs.writeFile(filePath, JSON.stringify(newHistory, undefined, 2)); + } catch (error) { + throw new Error(`Failed to write migration history to file: ${filePath}`, { cause: error }); + } }); return lastUpdate; }; + const acquireLock = async () => { + try { + const fd = await fs.open(lockFilePath, 'wx'); + + await fd.close(); + } catch (error) { + throw new Error('Could not acquire file lock for migrations', { cause: error }); + } + }; + + const releaseLock = async () => { + try { + await fs.unlink(lockFilePath); + } catch { + // Ignore + } + }; + return { async initializeStorage() { return { async lock(migrations) { - try { - const fd = await fs.open(lockFilePath, 'wx'); + await acquireLock(); - await fd.close(); - - return migrations; - } catch { - throw new Error('Could not acquire file lock for migrations'); - } + return migrations; }, async unlock() { + await releaseLock(); + }, + async remove(migration) { + await acquireLock(); + try { - await fs.unlink(lockFilePath); - } catch { - // Ignore + const history = await read(); + + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete history[migration.name]; + + await fs.writeFile(filePath, JSON.stringify(history, undefined, 2)); + } catch (error) { + throw new Error(`Failed to remove migration from history: ${migration.name}`, { cause: error }); + } finally { + await releaseLock(); } }, async *getHistory() {