3 namespace Drupal\migrate_tools\Commands;
5 use Consolidation\OutputFormatters\StructuredData\RowsOfFields;
6 use Drupal\Component\Utility\Unicode;
7 use Drupal\Core\Datetime\DateFormatter;
8 use Drupal\Core\Entity\EntityTypeManagerInterface;
9 use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
10 use Drupal\migrate\Exception\RequirementsException;
11 use Drupal\migrate\Plugin\MigrationInterface;
12 use Drupal\migrate\Plugin\MigrationPluginManager;
13 use Drupal\migrate\Plugin\RequirementsInterface;
14 use Drupal\migrate_tools\Drush9LogMigrateMessage;
15 use Drupal\migrate_tools\MigrateExecutable;
16 use Drush\Commands\DrushCommands;
19 * Migrate Tools drush commands.
21 class MigrateToolsCommands extends DrushCommands {
24 * Migration plugin manager service.
26 * @var \Drupal\migrate\Plugin\MigrationPluginManager
28 protected $migrationPluginManager;
31 * Date formatter service.
33 * @var \Drupal\Core\Datetime\DateFormatter
35 protected $dateFormatter;
38 * Entity type manager.
40 * @var \Drupal\Core\Entity\EntityTypeManagerInterface
42 protected $entityTypeManager;
45 * Key-value store service.
47 * @var \Drupal\Core\KeyValueStore\KeyValueFactoryInterface
52 * Migrate message logger.
54 * @var \Drupal\migrate_tools\Drush9LogMigrateMessage
56 protected $migrateMessage;
59 * MigrateToolsCommands constructor.
61 * @param \Drupal\migrate\Plugin\MigrationPluginManager $migrationPluginManager
62 * Migration Plugin Manager service.
63 * @param \Drupal\Core\Datetime\DateFormatter $dateFormatter
64 * Date formatter service.
65 * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entityTypeManager
66 * Entity type manager service.
67 * @param \Drupal\Core\KeyValueStore\KeyValueFactoryInterface $keyValue
68 * Key-value store service.
70 public function __construct(MigrationPluginManager $migrationPluginManager, DateFormatter $dateFormatter, EntityTypeManagerInterface $entityTypeManager, KeyValueFactoryInterface $keyValue) {
71 parent::__construct();
72 $this->migrationPluginManager = $migrationPluginManager;
73 $this->dateFormatter = $dateFormatter;
74 $this->entityTypeManager = $entityTypeManager;
75 $this->keyValue = $keyValue;
79 * List all migrations with current status.
81 * @param string $migration_names
82 * Restrict to a comma-separated list of migrations (Optional).
83 * @param array $options
84 * Additional options for the command.
86 * @command migrate:status
88 * @option group A comma-separated list of migration groups to list
89 * @option tag Name of the migration tag to list
90 * @option names-only Only return names, not all the details (faster)
92 * @usage migrate:status
93 * Retrieve status for all migrations
94 * @usage migrate:status --group=beer
95 * Retrieve status for all migrations in a given group
96 * @usage migrate:status --tag=user
97 * Retrieve status for all migrations with a given tag
98 * @usage migrate:status --group=beer --tag=user
99 * Retrieve status for all migrations in the beer group
100 * and with the user tag.
101 * @usage migrate:status beer_term,beer_node
102 * Retrieve status for specific migrations
104 * @validate-module-enabled migrate_tools
106 * @aliases ms, migrate-status
114 * unprocessed: Unprocessed
115 * last_imported: Last Imported
116 * @default-fields group,id,status,total,imported,unprocessed,last_imported
118 * @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
119 * Migrations status formatted as table.
121 public function status($migration_names = '', array $options = ['group' => NULL, 'tag' => NULL, 'names-only' => NULL]) {
122 $names_only = $options['names-only'];
124 $migrations = $this->migrationsList($migration_names, $options);
127 // Take it one group at a time, listing the migrations within each group.
128 foreach ($migrations as $group_id => $migration_list) {
129 /** @var \Drupal\migrate_plus\Entity\MigrationGroup $group */
130 $group = $this->entityTypeManager->getStorage('migration_group')->load($group_id);
131 $group_name = !empty($group) ? "{$group->label()} ({$group->id()})" : $group_id;
133 foreach ($migration_list as $migration_id => $migration) {
136 'group' => dt('Group: @name', ['@name' => $group_name]),
137 'id' => $migration_id,
142 $map = $migration->getIdMap();
143 $imported = $map->importedCount();
144 $source_plugin = $migration->getSourcePlugin();
146 catch (\Exception $e) {
147 $this->logger()->error(
149 'Failure retrieving information on @migration: @message',
150 ['@migration' => $migration_id, '@message' => $e->getMessage()]
157 $source_rows = $source_plugin->count();
158 // -1 indicates uncountable sources.
159 if ($source_rows == -1) {
160 $source_rows = dt('N/A');
161 $unprocessed = dt('N/A');
164 $unprocessed = $source_rows - $map->processedCount();
167 catch (\Exception $e) {
168 $this->logger()->error(
170 'Could not retrieve source count from @migration: @message',
171 ['@migration' => $migration_id, '@message' => $e->getMessage()]
174 $source_rows = dt('N/A');
175 $unprocessed = dt('N/A');
178 $status = $migration->getStatusLabel();
179 $migrate_last_imported_store = $this->keyValue->get(
180 'migrate_last_imported'
182 $last_imported = $migrate_last_imported_store->get(
186 if ($last_imported) {
187 $last_imported = $this->dateFormatter->format(
188 $last_imported / 1000,
197 'group' => $group_name,
198 'id' => $migration_id,
200 'total' => $source_rows,
201 'imported' => $imported,
202 'unprocessed' => $unprocessed,
203 'last_imported' => $last_imported,
208 // Add empty row to separate groups, for readability.
210 if ($group_id !== key($migrations)) {
215 return new RowsOfFields($table);
219 * Perform one or more migration processes.
221 * @param string $migration_names
222 * ID of migration(s) to import. Delimit multiple using commas.
223 * @param array $options
224 * Additional options for the command.
226 * @command migrate:import
228 * @option all Process all migrations.
229 * @option group A comma-separated list of migration groups to import
230 * @option tag Name of the migration tag to import
231 * @option limit Limit on the number of items to process in each migration
232 * @option feedback Frequency of progress messages, in items processed
233 * @option idlist Comma-separated list of IDs to import
234 * @option update In addition to processing unprocessed items from the
235 * source, update previously-imported items with the current data
236 * @option force Force an operation to run, even if all dependencies are not
238 * @option execute-dependencies Execute all dependent migrations first.
240 * @usage migrate:import --all
241 * Perform all migrations
242 * @usage migrate:import --group=beer
243 * Import all migrations in the beer group
244 * @usage migrate:import --tag=user
245 * Import all migrations with the user tag
246 * @usage migrate:import --group=beer --tag=user
247 * Import all migrations in the beer group and with the user tag
248 * @usage migrate:import beer_term,beer_node
249 * Import new terms and nodes
250 * @usage migrate:import beer_user --limit=2
251 * Import no more than 2 users
252 * @usage migrate:import beer_user --idlist=5
253 * Import the user record with source ID 5
255 * @validate-module-enabled migrate_tools
257 * @aliases mim, migrate-import
260 * If there are not enough parameters to the command.
262 public function import($migration_names = '', array $options = ['all' => NULL, 'group' => NULL, 'tag' => NULL, 'limit' => NULL, 'feedback' => NULL, 'idlist' => NULL, 'update' => NULL, 'force' => NULL, 'execute-dependencies' => NULL]) {
263 $group_names = $options['group'];
264 $tag_names = $options['tag'];
265 $all = $options['all'];
266 $additional_options = [];
267 if (!$all && !$group_names && !$migration_names && !$tag_names) {
268 throw new \Exception(dt('You must specify --all, --group, --tag or one or more migration names separated by commas'));
271 foreach (['limit', 'feedback', 'idlist', 'update', 'force', 'execute-dependencies'] as $option) {
272 if ($options[$option]) {
273 $additional_options[$option] = $options[$option];
277 $migrations = $this->migrationsList($migration_names, $options);
278 if (empty($migrations)) {
279 $this->logger->error(dt('No migrations found.'));
282 // Take it one group at a time, importing the migrations within each group.
283 foreach ($migrations as $group_id => $migration_list) {
286 [$this, 'executeMigration'],
293 * Rollback one or more migrations.
295 * @param string $migration_names
296 * Name of migration(s) to rollback. Delimit multiple using commas.
297 * @param array $options
298 * Additional options for the command.
300 * @command migrate:rollback
302 * @option all Process all migrations.
303 * @option group A comma-separated list of migration groups to rollback
304 * @option tag ID of the migration tag to rollback
305 * @option feedback Frequency of progress messages, in items processed
307 * @usage migrate:rollback --all
308 * Perform all migrations
309 * @usage migrate:rollback --group=beer
310 * Rollback all migrations in the beer group
311 * @usage migrate:rollback --tag=user
312 * Rollback all migrations with the user tag
313 * @usage migrate:rollback --group=beer --tag=user
314 * Rollback all migrations in the beer group and with the user tag
315 * @usage migrate:rollback beer_term,beer_node
316 * Rollback imported terms and nodes
317 * @validate-module-enabled migrate_tools
319 * @aliases mr, migrate-rollback
322 * If there are not enough parameters to the command.
324 public function rollback($migration_names = '', array $options = ['all' => NULL, 'group' => NULL, 'tag' => NULL, 'feedback' => NULL]) {
325 $group_names = $options['group'];
326 $tag_names = $options['tag'];
327 $all = $options['all'];
328 $additional_options = [];
329 if (!$all && !$group_names && !$migration_names && !$tag_names) {
330 throw new \Exception(dt('You must specify --all, --group, --tag, or one or more migration names separated by commas'));
333 if ($options['feedback']) {
334 $additional_options['feedback'] = $options['feedback'];
337 $migrations = $this->migrationsList($migration_names, $options);
338 if (empty($migrations)) {
339 $this->logger()->error(dt('No migrations found.'));
342 // Take it one group at a time,
343 // rolling back the migrations within each group.
344 foreach ($migrations as $group_id => $migration_list) {
345 // Roll back in reverse order.
346 $migration_list = array_reverse($migration_list);
347 foreach ($migration_list as $migration_id => $migration) {
348 $executable = new MigrateExecutable(
350 $this->getMigrateMessage(),
353 // drush_op() provides --simulate support.
354 drush_op([$executable, 'rollback']);
360 * Stop an active migration operation.
362 * @param string $migration_id
363 * ID of migration to stop.
365 * @command migrate:stop
367 * @validate-module-enabled migrate_tools
368 * @aliases mst, migrate-stop
370 public function stop($migration_id = '') {
371 /** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
372 $migration = $this->migrationPluginManager->createInstance(
376 $status = $migration->getStatus();
378 case MigrationInterface::STATUS_IDLE:
379 $this->logger()->warning(
380 dt('Migration @id is idle', ['@id' => $migration_id])
384 case MigrationInterface::STATUS_DISABLED:
385 $this->logger()->warning(
386 dt('Migration @id is disabled', ['@id' => $migration_id])
390 case MigrationInterface::STATUS_STOPPING:
391 $this->logger()->warning(
392 dt('Migration @id is already stopping', ['@id' => $migration_id])
397 $migration->interruptMigration(MigrationInterface::RESULT_STOPPED);
398 $this->logger()->notice(
399 dt('Migration @id requested to stop', ['@id' => $migration_id])
405 $this->logger()->error(
406 dt('Migration @id does not exist', ['@id' => $migration_id])
412 * Reset a active migration's status to idle.
414 * @param string $migration_id
415 * ID of migration to reset.
417 * @command migrate:reset-status
419 * @validate-module-enabled migrate_tools
420 * @aliases mrs, migrate-reset-status
422 public function resetStatus($migration_id = '') {
423 /** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
424 $migration = $this->migrationPluginManager->createInstance(
428 $status = $migration->getStatus();
429 if ($status == MigrationInterface::STATUS_IDLE) {
430 $this->logger()->warning(
431 dt('Migration @id is already Idle', ['@id' => $migration_id])
435 $migration->setStatus(MigrationInterface::STATUS_IDLE);
436 $this->logger()->notice(
437 dt('Migration @id reset to Idle', ['@id' => $migration_id])
442 $this->logger()->error(
443 dt('Migration @id does not exist', ['@id' => $migration_id])
449 * View any messages associated with a migration.
451 * @param string $migration_id
452 * ID of the migration.
453 * @param array $options
454 * Additional options for the command.
456 * @command migrate:messages
458 * @option csv Export messages as a CSV
460 * @usage migrate:messages MyNode
461 * Show all messages for the MyNode migration
463 * @validate-module-enabled migrate_tools
465 * @aliases mmsg,migrate-messages
468 * source_ids_hash: Source IDs Hash
471 * @default-fields source_ids_hash,level,message
473 * @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
474 * Source fields of the given migration formatted as a table.
476 public function messages($migration_id, array $options = ['csv' => NULL]) {
477 /** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
478 $migration = $this->migrationPluginManager->createInstance(
482 $this->logger()->error(
483 dt('Migration @id does not exist', ['@id' => $migration_id])
488 $map = $migration->getIdMap();
490 foreach ($map->getMessageIterator() as $row) {
492 $table[] = (array) $row;
495 $this->logger()->notice(dt('No messages for this migration'));
499 if ($options['csv']) {
500 fputcsv(STDOUT, array_keys($table[0]));
501 foreach ($table as $row) {
502 fputcsv(STDOUT, $row);
506 return new RowsOfFields($table);
510 * List the fields available for mapping in a source.
512 * @param string $migration_id
513 * ID of the migration.
515 * @command migrate:fields-source
517 * @usage migrate:fields-source my_node
518 * List fields for the source in the my_node migration
520 * @validate-module-enabled migrate_tools
522 * @aliases mfs, migrate-fields-source
525 * machine_name: Machine Name
526 * description: Description
527 * @default-fields machine_name,description
529 * @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
530 * Source fields of the given migration formatted as a table.
532 public function fieldsSource($migration_id) {
533 /** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
534 $migration = $this->migrationPluginManager->createInstance(
538 $source = $migration->getSourcePlugin();
540 foreach ($source->fields() as $machine_name => $description) {
542 'machine_name' => $machine_name,
543 'description' => strip_tags($description),
546 return new RowsOfFields($table);
549 $this->logger()->error(
550 dt('Migration @id does not exist', ['@id' => $migration_id])
556 * Retrieve a list of active migrations.
558 * @param string $migration_ids
559 * Comma-separated list of migrations -
560 * if present, return only these migrations.
561 * @param array $options
564 * @return \Drupal\migrate\Plugin\MigrationInterface[][]
565 * An array keyed by migration group, each value containing an array of
566 * migrations or an empty array if no migrations match the input criteria.
568 protected function migrationsList($migration_ids = '', array $options = []) {
569 // Filter keys must match the migration configuration property name.
570 $filter['migration_group'] = $options['group'] ? explode(
574 $filter['migration_tags'] = $options['tag'] ? explode(
579 $manager = $this->migrationPluginManager;
580 $plugins = $manager->createInstances([]);
581 $matched_migrations = [];
583 // Get the set of migrations that may be filtered.
584 if (empty($migration_ids)) {
585 $matched_migrations = $plugins;
588 // Get the requested migrations.
589 $migration_ids = explode(',', Unicode::strtolower($migration_ids));
590 foreach ($plugins as $id => $migration) {
591 if (in_array(Unicode::strtolower($id), $migration_ids)) {
592 $matched_migrations[$id] = $migration;
597 // Do not return any migrations which fail to meet requirements.
598 /** @var \Drupal\migrate\Plugin\Migration $migration */
599 foreach ($matched_migrations as $id => $migration) {
600 if ($migration->getSourcePlugin() instanceof RequirementsInterface) {
602 $migration->getSourcePlugin()->checkRequirements();
604 catch (RequirementsException $e) {
605 unset($matched_migrations[$id]);
610 // Filters the matched migrations if a group or a tag has been input.
611 if (!empty($filter['migration_group']) || !empty($filter['migration_tags'])) {
612 // Get migrations in any of the specified groups and with any of the
614 foreach ($filter as $property => $values) {
615 if (!empty($values)) {
616 $filtered_migrations = [];
617 foreach ($values as $search_value) {
618 foreach ($matched_migrations as $id => $migration) {
619 // Cast to array because migration_tags can be an array.
620 $configured_values = (array) $migration->get($property);
621 $configured_id = (in_array(
624 )) ? $search_value : 'default';
625 if (empty($search_value) || $search_value == $configured_id) {
626 if (empty($migration_ids) || in_array(
627 Unicode::strtolower($id),
630 $filtered_migrations[$id] = $migration;
635 $matched_migrations = $filtered_migrations;
640 // Sort the matched migrations by group.
641 if (!empty($matched_migrations)) {
642 foreach ($matched_migrations as $id => $migration) {
643 $configured_group_id = empty($migration->get('migration_group')) ? 'default' : $migration->get('migration_group');
644 $migrations[$configured_group_id][$id] = $migration;
647 return isset($migrations) ? $migrations : [];
651 * Executes a single migration.
653 * If the --execute-dependencies option was given,
654 * the migration's dependencies will also be executed first.
656 * @param \Drupal\migrate\Plugin\MigrationInterface $migration
657 * The migration to execute.
658 * @param string $migration_id
659 * The migration ID (not used, just an artifact of array_walk()).
660 * @param array $options
661 * Additional options of the command.
664 * If some migrations failed during execution.
666 protected function executeMigration(MigrationInterface $migration, $migration_id, array $options = []) {
667 // Keep track of all migrations run during this command so the same
668 // migration is not run multiple times.
669 static $executed_migrations = [];
671 if (isset($options['execute-dependencies'])) {
672 $required_migrations = $migration->get('requirements');
673 $required_migrations = array_filter($required_migrations, function ($value) use ($executed_migrations) {
674 return !isset($executed_migrations[$value]);
677 if (!empty($required_migrations)) {
678 $manager = $this->migrationPluginManager;
679 $required_migrations = $manager->createInstances($required_migrations);
680 $dependency_options = array_merge($options, ['is_dependency' => TRUE]);
681 array_walk($required_migrations, [$this, __FUNCTION__], $dependency_options);
682 $executed_migrations += $required_migrations;
685 if (!empty($options['force'])) {
686 $migration->set('requirements', []);
688 if (!empty($options['update'])) {
689 $migration->getIdMap()->prepareUpdate();
691 $executable = new MigrateExecutable($migration, $this->getMigrateMessage(), $options);
692 // drush_op() provides --simulate support.
693 drush_op([$executable, 'import']);
694 $executed_migrations += [$migration_id => $migration_id];
695 if ($count = $executable->getFailedCount()) {
696 // Nudge Drush to use a non-zero exit code.
697 throw new \Exception(
699 '!name Migration - !count failed.',
700 ['!name' => $migration_id, '!count' => $count]
707 * Gets the migrate message logger.
709 * @return \Drupal\migrate\MigrateMessageInterface
710 * The migrate message service.
712 protected function getMigrateMessage() {
713 if (!isset($this->migrateMessage)) {
714 $this->migrateMessage = new Drush9LogMigrateMessage($this->logger());
716 return $this->migrateMessage;