feat(cli): improve the "up" commands --from and --to options

The given values can either be migration names or relative paths to migration files.
The given migration must exist to avoid accidentally running migrations that wasn't intended to run.

packages/cli/README.md

@@ -28,22 +28,35 @@ Run all pending migrations
 Options:
 
   -h, --help              Show this help message and exit
+
   -d, --directory <path>  The directory where the migration files are located (required)
+
   -i, --import <module>   Additional modules/packages to import before running the migrations (can be specified multiple times)
                           For example if you want to use Dotenv to load environment variables or when using TypeScript
+
   -s, --storage <name>    The storage to use for where to store the migration history (required)
+
   -p, --plugin <name>     The plugin(s) to use (can be specified multiple times)
+
   -r, --reporter <name>   The reporter to use for reporting the migration progress
+
   -l, --limit <count>     Limit the number of migrations to run
-  -f, --from <name>       Start running migrations from the given migration name, the given name doesn't need to exist
-                          and is compared in lexicographical order
-  -t, --to <name>         Skip migrations after the given migration name, the given name doesn't need to exist
-                          and is compared in lexicographical order
+
+  -f, --from <name/path>  Start running migrations from the given migration name or relative file path to a migration file,
+                          the given name or path needs to exist. The same migration and those after it lexicographically will be run
+
+  -t, --to <name/path>    Skip migrations after the given migration name or relative file path to a migration file,
+                          the given name or path needs to exist. The same migration and those before it lexicographically will be run
+
   --dry                   List the pending migrations that would be run without actually running them
+
   --color                 Force color output (this option is passed to the reporter)
+
   --no-color              Disable color output (this option is passed to the reporter)
+
   --no-execution          Mark the migrations as executed and successful without actually running them,
                           which is useful if you want to mark migrations as successful after running them manually
+
   --abort-respite <sec>   The number of seconds to wait before abandoning running migrations after the command has been aborted (default: 10)
 
 Examples:

packages/cli/src/cli.ts

@@ -93,22 +93,35 @@ Run all pending migrations
 Options:
 
   -h, --help              Show this help message and exit
+
   -d, --directory <path>  The directory where the migration files are located (required)
+
   -i, --import <module>   Additional modules/packages to import before running the migrations (can be specified multiple times)
                           For example if you want to use Dotenv to load environment variables or when using TypeScript
+
   -s, --storage <name>    The storage to use for where to store the migration history (required)
+
   -p, --plugin <name>     The plugin(s) to use (can be specified multiple times)
+
   -r, --reporter <name>   The reporter to use for reporting the migration progress
+
   -l, --limit <count>     Limit the number of migrations to run
-  -f, --from <name>       Start running migrations from the given migration name, the given name doesn't need to exist
-                          and is compared in lexicographical order
-  -t, --to <name>         Skip migrations after the given migration name, the given name doesn't need to exist
-                          and is compared in lexicographical order
+
+  -f, --from <name/path>  Start running migrations from the given migration name or relative file path to a migration file,
+                          the given name or path needs to exist. The same migration and those after it lexicographically will be run
+
+  -t, --to <name/path>    Skip migrations after the given migration name or relative file path to a migration file,
+                          the given name or path needs to exist. The same migration and those before it lexicographically will be run
+
   --dry                   List the pending migrations that would be run without actually running them
+
   --color                 Force color output (this option is passed to the reporter)
+
   --no-color              Disable color output (this option is passed to the reporter)
+
   --no-execution          Mark the migrations as executed and successful without actually running them,
                           which is useful if you want to mark migrations as successful after running them manually
+
   --abort-respite <sec>   The number of seconds to wait before abandoning running migrations after the command has been aborted (default: ${DEFAULT_RESPITE_SECONDS})
 
 Examples:

packages/cli/src/commands/list.ts

@@ -3,7 +3,6 @@ import { BadOptionError, MissingOptionError, StorageInitError, toError } from '.
 import { type Config } from '../types.js';
 import { exec } from '../exec.js';
 import { migrationRunner } from '../migration-runner.js';
-import { arrayFromAsync } from '../array-from-async.js';
 import { collectMigrations } from '../collect-migrations.js';
 import { version } from '../get-package-info.js';
 
@@ -56,7 +55,7 @@ export default async function listCommand({
       dry: true,
       reporter,
       storage,
-      migrations: await arrayFromAsync(collectedMigrations),
+      migrations: collectedMigrations,
       async validate() {
         // No-op
       },

packages/cli/src/commands/up.test.ts

@@ -278,7 +278,100 @@ describe('up', () => {
       ]);
     });
 
-    it('returns 0 and finishes without an error when pending migrations after given "from" parameter are run successfully, even when the "from" is not an existing migration', async () => {
+    it('returns 0 and finishes without an error when pending migrations after given "from" parameter are run successfully', async () => {
+      const migration = mock.fn(async () => {
+        // Success
+      });
+      const { reporter, run } = getUpCommand(
+        [
+          '1_some_already_run_migration.js',
+          '2_some_migration.js',
+          '3_existing_migration.js',
+          '4_some_other_migration.js',
+        ],
+        getStorage(['1_some_already_run_migration.js']),
+        [
+          {
+            loadableExtensions: ['.js'],
+            async loadMigration() {
+              return migration;
+            },
+          },
+        ],
+      );
+
+      const exitCode = await run({ from: '3_existing_migration.js' });
+
+      assert.strictEqual(exitCode, 0, 'Exit code');
+      assertPreconditionsFulfilled({ dry: false }, reporter, [
+        { name: '2_some_migration.js', status: 'skipped' },
+        { name: '3_existing_migration.js', status: 'done', started: true },
+        { name: '4_some_other_migration.js', status: 'done', started: true },
+      ]);
+      assert.strictEqual(migration.mock.calls.length, 2);
+    });
+
+    it('returns 0 and finishes without an error when pending migrations after given "from" parameter are run successfully, when the "from" parameter is a relative path', async () => {
+      const migration = mock.fn(async () => {
+        // Success
+      });
+      const { reporter, run } = getUpCommand(
+        [
+          '1_some_already_run_migration.js',
+          '2_some_migration.js',
+          '3_existing_migration.js',
+          '4_some_other_migration.js',
+        ],
+        getStorage(['1_some_already_run_migration.js']),
+        [
+          {
+            loadableExtensions: ['.js'],
+            async loadMigration() {
+              return migration;
+            },
+          },
+        ],
+      );
+
+      const exitCode = await run({ from: 'migrations/3_existing_migration.js' });
+
+      assert.strictEqual(exitCode, 0, 'Exit code');
+      assertPreconditionsFulfilled({ dry: false }, reporter, [
+        { name: '2_some_migration.js', status: 'skipped' },
+        { name: '3_existing_migration.js', status: 'done', started: true },
+        { name: '4_some_other_migration.js', status: 'done', started: true },
+      ]);
+      assert.strictEqual(migration.mock.calls.length, 2);
+    });
+
+    it('returns 0 and runs all pending migrations, if "from" is an already executed migration', async () => {
+      const migration = mock.fn(async () => {
+        // Success
+      });
+      const { reporter, run } = getUpCommand(
+        ['1_some_already_run_migration.js', '2_some_migration.js', '4_some_other_migration.js'],
+        getStorage(['1_some_already_run_migration.js']),
+        [
+          {
+            loadableExtensions: ['.js'],
+            async loadMigration() {
+              return migration;
+            },
+          },
+        ],
+      );
+
+      const exitCode = await run({ from: '1_some_already_run_migration.js' });
+
+      assert.strictEqual(exitCode, 0, 'Exit code');
+      assertPreconditionsFulfilled({ dry: false }, reporter, [
+        { name: '2_some_migration.js', status: 'done', started: true },
+        { name: '4_some_other_migration.js', status: 'done', started: true },
+      ]);
+      assert.strictEqual(migration.mock.calls.length, 2);
+    });
+
+    it('returns 1 and finishes with an error when the given "from" migration name does not exist', async () => {
       const migration = mock.fn(async () => {
         // Success
       });
@@ -297,30 +390,71 @@ describe('up', () => {
 
       const exitCode = await run({ from: '3_non_existing_migration.js' });
 
-      assert.strictEqual(exitCode, 0, 'Exit code');
-      assertPreconditionsFulfilled({ dry: false }, reporter, [
-        { name: '2_some_migration.js', status: 'skipped' },
-        { name: '4_some_other_migration.js', status: 'done', started: true },
-      ]);
-      assert.strictEqual(migration.mock.calls.length, 1);
+      assert.strictEqual(exitCode, 1, 'Exit code');
+      assertPreconditionsFulfilled(
+        { dry: false },
+        reporter,
+        [
+          { name: '2_some_migration.js', status: 'skipped' },
+          { name: '4_some_other_migration.js', status: 'skipped' },
+        ],
+        BadOptionError.fromOption(
+          'from',
+          'The "from" migration: "migrations/3_non_existing_migration.js" was not found',
+        ),
+      );
+      assert.strictEqual(migration.mock.calls.length, 0);
     });
 
-    it('returns 0 and finishes without an error when pending migrations after given "from" parameter are validated and listed successfully in dry-mode, even when the "from" is not an existing migration', async () => {
+    it('returns 0 and finishes without an error when pending migrations after given "from" parameter are validated and listed successfully in dry-mode', async () => {
       const { reporter, run } = getUpCommand(
-        ['1_some_already_run_migration.js', '2_some_migration.js', '4_some_other_migration.js'],
+        ['1_some_already_run_migration.js', '2_some_migration.js', '3_some_other_migration.js'],
         getStorage(['1_some_already_run_migration.js']),
       );
 
-      const exitCode = await run({ dry: true, from: '3_non_existing_migration.js' });
+      const exitCode = await run({ dry: true, from: '3_some_other_migration.js' });
 
       assert.strictEqual(exitCode, 0, 'Exit code');
       assertPreconditionsFulfilled({ dry: true }, reporter, [
         { name: '2_some_migration.js', status: 'skipped' },
-        { name: '4_some_other_migration.js', status: 'pending' },
+        { name: '3_some_other_migration.js', status: 'pending' },
       ]);
     });
 
-    it('returns 0 and finishes without an error when pending migrations before given "to" parameter are run successfully, even when the "to" is not an existing migration', async () => {
+    it('returns 0 and finishes without an error when pending migrations before given "to" parameter are run successfully', async () => {
+      const migration = mock.fn(async () => {
+        // Success
+      });
+      const { reporter, run } = getUpCommand(
+        [
+          '1_some_already_run_migration.js',
+          '2_some_migration.js',
+          '3_existing_migration.js',
+          '4_some_other_migration.js',
+        ],
+        getStorage(['1_some_already_run_migration.js']),
+        [
+          {
+            loadableExtensions: ['.js'],
+            async loadMigration() {
+              return migration;
+            },
+          },
+        ],
+      );
+
+      const exitCode = await run({ to: '3_existing_migration.js' });
+
+      assert.strictEqual(exitCode, 0, 'Exit code');
+      assertPreconditionsFulfilled({ dry: false }, reporter, [
+        { name: '2_some_migration.js', status: 'done', started: true },
+        { name: '3_existing_migration.js', status: 'done', started: true },
+        { name: '4_some_other_migration.js', status: 'skipped' },
+      ]);
+      assert.strictEqual(migration.mock.calls.length, 2);
+    });
+
+    it('returns 1 and finishes with an error when the given "to" migration name does not exist', async () => {
       const migration = mock.fn(async () => {
         // Success
       });
@@ -339,25 +473,63 @@ describe('up', () => {
 
       const exitCode = await run({ to: '3_non_existing_migration.js' });
 
-      assert.strictEqual(exitCode, 0, 'Exit code');
-      assertPreconditionsFulfilled({ dry: false }, reporter, [
-        { name: '2_some_migration.js', status: 'done', started: true },
-        { name: '4_some_other_migration.js', status: 'skipped' },
-      ]);
-      assert.strictEqual(migration.mock.calls.length, 1);
+      assert.strictEqual(exitCode, 1, 'Exit code');
+      assertPreconditionsFulfilled(
+        { dry: false },
+        reporter,
+        [
+          { name: '2_some_migration.js', status: 'skipped' },
+          { name: '4_some_other_migration.js', status: 'skipped' },
+        ],
+        BadOptionError.fromOption('to', 'The "to" migration: "migrations/3_non_existing_migration.js" was not found'),
+      );
+      assert.strictEqual(migration.mock.calls.length, 0);
     });
 
-    it('returns 0 and finishes without an error when pending migrations after given "to" parameter are validated and listed successfully in dry-mode, even when the "to" is not an existing migration', async () => {
+    it('returns 0 and runs no migrations, if "to" is an already executed migration', async () => {
+      const migration = mock.fn(async () => {
+        // Success
+      });
       const { reporter, run } = getUpCommand(
         ['1_some_already_run_migration.js', '2_some_migration.js', '4_some_other_migration.js'],
         getStorage(['1_some_already_run_migration.js']),
+        [
+          {
+            loadableExtensions: ['.js'],
+            async loadMigration() {
+              return migration;
+            },
+          },
+        ],
       );
 
-      const exitCode = await run({ dry: true, to: '3_non_existing_migration.js' });
+      const exitCode = await run({ to: '1_some_already_run_migration.js' });
+
+      assert.strictEqual(exitCode, 0, 'Exit code');
+      assertPreconditionsFulfilled({ dry: false }, reporter, [
+        { name: '2_some_migration.js', status: 'skipped' },
+        { name: '4_some_other_migration.js', status: 'skipped' },
+      ]);
+      assert.strictEqual(migration.mock.calls.length, 0);
+    });
+
+    it('returns 0 and finishes without an error when pending migrations after given "to" parameter are validated and listed successfully in dry-mode', async () => {
+      const { reporter, run } = getUpCommand(
+        [
+          '1_some_already_run_migration.js',
+          '2_some_migration.js',
+          '3_existing_migration.js',
+          '4_some_other_migration.js',
+        ],
+        getStorage(['1_some_already_run_migration.js']),
+      );
+
+      const exitCode = await run({ dry: true, to: '3_existing_migration.js' });
 
       assert.strictEqual(exitCode, 0, 'Exit code');
       assertPreconditionsFulfilled({ dry: true }, reporter, [
         { name: '2_some_migration.js', status: 'pending' },
+        { name: '3_existing_migration.js', status: 'pending' },
         { name: '4_some_other_migration.js', status: 'skipped' },
       ]);
     });

packages/cli/src/commands/up.ts

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import { getOrLoadPlugins, getOrLoadReporter, getOrLoadStorage } from '@emigrate/plugin-tools';
 import { isFinishedMigration, type LoaderPlugin } from '@emigrate/types';
 import { BadOptionError, MigrationLoadError, MissingOptionError, StorageInitError, toError } from '../errors.js';
@@ -6,9 +7,7 @@ import { withLeadingPeriod } from '../with-leading-period.js';
 import { type GetMigrationsFunction } from '../get-migrations.js';
 import { exec } from '../exec.js';
 import { migrationRunner } from '../migration-runner.js';
-import { filterAsync } from '../filter-async.js';
 import { collectMigrations } from '../collect-migrations.js';
-import { arrayFromAsync } from '../array-from-async.js';
 import { version } from '../get-package-info.js';
 
 type ExtraFlags = {
@@ -72,10 +71,7 @@ export default async function upCommand({
   }
 
   try {
-    const collectedMigrations = filterAsync(
-      collectMigrations(cwd, directory, storage.getHistory(), getMigrations),
-      (migration) => !isFinishedMigration(migration) || migration.status === 'failed',
-    );
+    const collectedMigrations = collectMigrations(cwd, directory, storage.getHistory(), getMigrations);
 
     const loaderPlugins = await getOrLoadPlugins('loader', [lazyPluginLoaderJs, ...plugins]);
 
@@ -93,6 +89,14 @@ export default async function upCommand({
       return loaderByExtension.get(extension);
     };
 
+    if (from && !from.includes(path.sep)) {
+      from = path.join(directory, from);
+    }
+
+    if (to && !to.includes(path.sep)) {
+      to = path.join(directory, to);
+    }
+
     const error = await migrationRunner({
       dry,
       limit,
@@ -102,7 +106,10 @@ export default async function upCommand({
       abortRespite,
       reporter,
       storage,
-      migrations: await arrayFromAsync(collectedMigrations),
+      migrations: collectedMigrations,
+      migrationFilter(migration) {
+        return !isFinishedMigration(migration) || migration.status === 'failed';
+      },
       async validate(migration) {
         if (noExecution) {
           return;

packages/cli/src/filter-async.ts

@@ -1,13 +0,0 @@
-export function filterAsync<T, S extends T>(
-  iterable: AsyncIterable<T>,
-  filter: (item: T) => item is S,
-): AsyncIterable<S>;
-export function filterAsync<T>(iterable: AsyncIterable<T>, filter: (item: T) => unknown): AsyncIterable<T>;
-
-export async function* filterAsync<T>(iterable: AsyncIterable<T>, filter: (item: T) => unknown): AsyncIterable<T> {
-  for await (const item of iterable) {
-    if (filter(item)) {
-      yield item;
-    }
-  }
-}

packages/cli/src/migration-runner.ts

@@ -9,7 +9,7 @@ import {
   type FailedMigrationMetadata,
   type SuccessfulMigrationMetadata,
 } from '@emigrate/types';
-import { toError, EmigrateError, MigrationRunError, toSerializedError } from './errors.js';
+import { toError, EmigrateError, MigrationRunError, toSerializedError, BadOptionError } from './errors.js';
 import { exec } from './exec.js';
 import { getDuration } from './get-duration.js';
 
@@ -22,7 +22,8 @@ type MigrationRunnerParameters = {
   abortRespite?: number;
   reporter: EmigrateReporter;
   storage: Storage;
-  migrations: Array<MigrationMetadata | MigrationMetadataFinished>;
+  migrations: AsyncIterable<MigrationMetadata | MigrationMetadataFinished>;
+  migrationFilter?: (migration: MigrationMetadata | MigrationMetadataFinished) => boolean;
   validate: (migration: MigrationMetadata) => Promise<void>;
   execute: (migration: MigrationMetadata) => Promise<void>;
 };
@@ -39,9 +40,9 @@ export const migrationRunner = async ({
   migrations,
   validate,
   execute,
+  migrationFilter = () => true,
 }: MigrationRunnerParameters): Promise<Error | undefined> => {
-  await reporter.onCollectedMigrations?.(migrations);
-
+  const collectedMigrations: Array<MigrationMetadata | MigrationMetadataFinished> = [];
   const validatedMigrations: Array<MigrationMetadata | MigrationMetadataFinished> = [];
   const migrationsToLock: MigrationMetadata[] = [];
 
@@ -63,15 +64,32 @@ export const migrationRunner = async ({
     { once: true },
   );
 
+  let fromFound = false;
+  let toFound = false;
+
   for await (const migration of migrations) {
+    if (from && migration.relativeFilePath === from) {
+      fromFound = true;
+    }
+
+    if (to && migration.relativeFilePath === to) {
+      toFound = true;
+    }
+
+    if (!migrationFilter(migration)) {
+      continue;
+    }
+
+    collectedMigrations.push(migration);
+
     if (isFinishedMigration(migration)) {
       skip ||= migration.status === 'failed' || migration.status === 'skipped';
 
       validatedMigrations.push(migration);
     } else if (
       skip ||
-      Boolean(from && migration.name < from) ||
-      Boolean(to && migration.name > to) ||
+      Boolean(from && migration.relativeFilePath < from) ||
+      Boolean(to && migration.relativeFilePath > to) ||
       (limit && migrationsToLock.length >= limit)
     ) {
       validatedMigrations.push({
@@ -107,6 +125,32 @@ export const migrationRunner = async ({
     }
   }
 
+  await reporter.onCollectedMigrations?.(collectedMigrations);
+
+  let optionError: Error | undefined;
+
+  if (from && !fromFound) {
+    optionError = BadOptionError.fromOption('from', `The "from" migration: "${from}" was not found`);
+  } else if (to && !toFound) {
+    optionError = BadOptionError.fromOption('to', `The "to" migration: "${to}" was not found`);
+  }
+
+  if (optionError) {
+    dry = true;
+    skip = true;
+
+    for (const migration of migrationsToLock) {
+      const validatedIndex = validatedMigrations.indexOf(migration);
+
+      validatedMigrations[validatedIndex] = {
+        ...migration,
+        status: 'skipped',
+      };
+    }
+
+    migrationsToLock.length = 0;
+  }
+
   const [lockedMigrations, lockError] = dry
     ? [migrationsToLock]
     : await exec(async () => storage.lock(migrationsToLock), { abortSignal, abortRespite });
@@ -227,7 +271,11 @@ export const migrationRunner = async ({
         ? MigrationRunError.fromMetadata(firstFailed)
         : undefined;
   const error =
-    unlockError ?? firstError ?? lockError ?? (abortSignal?.aborted ? toError(abortSignal.reason) : undefined);
+    optionError ??
+    unlockError ??
+    firstError ??
+    lockError ??
+    (abortSignal?.aborted ? toError(abortSignal.reason) : undefined);
 
   await reporter.onFinished?.(finishedMigrations, error);