Issue #1: support larger exports by splitting the dump to chunks.

reinstall.drush.inc

@@ -26,9 +26,12 @@ function reinstall_drush_command() {
  *
  * @param string $entity_type
  *   The entity type.
+ * @param array $bundles
+ *   An array of bundle names to export.
  */
-function drush_reinstall_export($entity_type = NULL) {
+function drush_reinstall_export($entity_type = NULL, array ...$bundles) {
+
   /** @var \Drupal\reinstall\Dumper $dumper */
   $dumper = \Drupal::service('reinstall.dumper');
-  $dumper->dump($entity_type);
+  $dumper->dump($entity_type, $bundles);
 }

reinstall.services.yml

@@ -2,6 +2,10 @@ parameters:
   reinstall.path: '../data'
 
 services:
+  logger.channel.reinstall:
+    parent: logger.channel_base
+    arguments: ['reinstall']
+
   reinstall.dumper:
     class: 'Drupal\reinstall\Dumper'
     arguments:
@@ -12,6 +16,7 @@ services:
       - '@serializer'
       - '@event_dispatcher'
       - '%reinstall.path%'
+      - '@logger.channel.reinstall'
 
   reinstall.post_dump.file:
     class: 'Drupal\reinstall\EventSubscriber\FilePostDump'

src/Dumper.php

@@ -4,10 +4,13 @@ namespace Drupal\reinstall;
 
 use Drupal\Core\Entity\ContentEntityStorageInterface;
 use Drupal\Core\Entity\ContentEntityTypeInterface;
+use Drupal\Core\Entity\EntityStorageInterface;
 use Drupal\Core\Entity\EntityTypeBundleInfo;
+use Drupal\Core\Entity\EntityTypeInterface;
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Session\AccountSwitcherInterface;
 use Drupal\Core\Session\UserSession;
+use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Symfony\Component\Serializer\Serializer;
 use Symfony\Component\Yaml\Yaml;
@@ -19,6 +22,14 @@ use Symfony\Component\Yaml\Yaml;
  */
 class Dumper {
 
+  /**
+   * Should probably be a service parameter in the future.
+   */
+  const BATCH_SIZE = 50;
+
+  /**
+   * The interface used to ensure the dump request is for content entities.
+   */
   const CONTENT_INTERFACE = 'Drupal\Core\Entity\ContentEntityInterface';
 
   /**
@@ -69,6 +80,13 @@ class Dumper {
    */
   protected $importPath = self::IMPORT_PATH;
 
+  /**
+   * The reinstall logger channel service.
+   *
+   * @var \Psr\Log\LoggerInterface
+   */
+  protected $logger;
+
   /**
    * The app.root parameter service.
    *
@@ -100,6 +118,8 @@ class Dumper {
    *   The event_dispatcher service.
    * @param string $path
    *   The import path.
+   * @param \Psr\Log\LoggerInterface $logger
+   *   The reinstall logger channel service.
    */
   public function __construct(
     AccountSwitcherInterface $accountSwitcher,
@@ -108,11 +128,13 @@ class Dumper {
     string $root,
     Serializer $serializer,
     EventDispatcherInterface $eventDispatcher,
-    string $path
+    string $path,
+    LoggerInterface $logger
     ) {
     $this->accountSwitcher = $accountSwitcher;
     $this->entityTypeBundleInfo = $bundleInfo;
     $this->entityTypeManager = $entityTypeManager;
+    $this->logger = $logger;
     $this->root = $root;
     $this->serializer = $serializer;
     $this->eventDispatcher = $eventDispatcher;
@@ -148,6 +170,95 @@ class Dumper {
     return $entityTypes;
   }
 
+  /**
+   * Dump the entities in the selected bundle of the entity type.
+   *
+   * @param string $typeName
+   *   The name of the entity type from which to dump a bundle.
+   * @param \Drupal\Core\Entity\EntityTypeInterface $entityType
+   *   The entity type instance.
+   * @param string $bundle
+   *   The name of the bundle to dump. Assumed to be valid.
+   *
+   * @see \Drupal\reinstall\Dumper::validateBundles()
+   */
+  protected function dumpEntitiesBundle($typeName, EntityTypeInterface $entityType, string $bundle) {
+    $t0 = microtime(TRUE);
+    $storage = $this->entityTypeManager->getStorage($typeName);
+    $bundleKey = $this->entityTypeManager->getDefinition($typeName)->getKey('bundle');
+    $countQuery = $storage->getQuery();
+    if ($bundleKey) {
+      $countQuery = $countQuery->condition($bundleKey, $bundle);
+    }
+    $count = $countQuery
+      ->count()
+      ->execute();
+
+    $query = $storage->getQuery();
+    if ($bundleKey) {
+      $query = $query->condition($bundleKey, $bundle);
+    }
+    $ids = $query->execute();
+    $chunks = array_chunk($ids, static::BATCH_SIZE);
+
+    $path = $this->prepareDestination($typeName, $bundle);
+    $fp = fopen($path, "w");
+
+    foreach ($chunks as $chunk) {
+      $this->dumpEntitiesChunk($fp, $typeName, $bundle, $storage, $chunk);
+    }
+
+    // Ensure files always contain at least an empty array.
+    if (empty($chunks)) {
+      fwrite($fp, '{  }');
+    }
+
+    fclose($fp);
+    $t1 = microtime(TRUE);
+
+    $this->logger->info('Dumped @count entities for @type/@bundle in @sec seconds.', [
+      '@count' => $count,
+      '@type' => $typeName,
+      '@bundle' => $bundle,
+      '@sec' => sprintf('%.1f', $t1 - $t0),
+    ]);
+
+  }
+
+  /**
+   * Dump a chunk of entities from a common bundle in an existing opened file.
+   *
+   * @param resource $fp
+   *   A file pointer to which to write.
+   * @param string $typeName
+   *   The name of the entity type from which to dump a chunk of entities.
+   * @param string $bundle
+   *   The bundle name.
+   * @param \Drupal\Core\Entity\EntityStorageInterface $storage
+   *   The entity storage for the entity type.
+   * @param array $chunk
+   *   An array of entities to dump.
+   */
+  protected function dumpEntitiesChunk(
+    $fp,
+    string $typeName,
+    string $bundle,
+    EntityStorageInterface $storage,
+    array $chunk
+  ) {
+    $entities = $storage->loadMultiple($chunk);
+
+    // Allow adding data to entities before exporting, like term parents.
+    $eventPre = new DumperEvent($storage, $bundle, $entities);
+    $this->eventDispatcher->dispatch(ReinstallEvents::PRE_DUMP, $eventPre);
+
+    $this->dumpEntities($fp, $typeName, $bundle, $entities);
+
+    // Allow extra work after exporting, like copying files.
+    $eventPost = new DumperEvent($storage, $bundle, $entities);
+    $this->eventDispatcher->dispatch(ReinstallEvents::POST_DUMP, $eventPost);
+  }
+
   /**
    * Load entities for a given entity bundle.
    *
@@ -177,40 +288,32 @@ class Dumper {
   /**
    * Dump entities to YAML files.
    *
-   * @param string $requestedTypeName
-   *   Optional. The name of the entity type for which to export entities. If
-   *   absent, all entities in all types are dumped.
+   * @param string $typeName
+   *   The name of the entity type for which to export entities.
+   * @param array $bundles
+   *   Optional. An array of bundles to export. Only used if an entity type is
+   *   specified, since $requestedTypeName is itself optional.
    *
    * @see https://www.drupal.org/node/218104
    */
-  public function dump($requestedTypeName = NULL) {
+  public function dump($typeName, array $bundles = []) {
     $this->accountSwitcher->switchTo(new UserSession(['uid' => 1]));
 
-    $entityTypes = $this->contentEntityTypes($requestedTypeName);
-    foreach ($entityTypes as $entityTypeName => $entityType) {
-      $storage = $this->entityTypeManager->getStorage($entityTypeName);
-      $entities = $entityType->hasKey('bundle')
-        ? $this->loadMultiBundleEntities($entityType, $storage)
-        : [$entityTypeName => $storage->loadMultiple()];
-
-      foreach ($entities as $bundleName => $bundleEntities) {
-        // Allow adding data to entities before exporting, like term parents.
-        $eventPre = new DumperEvent($storage, $bundleName, $bundleEntities);
-        $this->eventDispatcher->dispatch(ReinstallEvents::PRE_DUMP, $eventPre);
+    $entityType = current($this->contentEntityTypes($typeName));
+    $validBundles = $this->validateBundles($typeName, $bundles);
 
-        $this->dumpEntities($entityTypeName, $bundleName, $bundleEntities);
-
-        // Allow extra work after exporting, like copying files.
-        $eventPost = new DumperEvent($storage, $bundleName, $bundleEntities);
-        $this->eventDispatcher->dispatch(ReinstallEvents::POST_DUMP, $eventPost);
-      }
+    foreach ($validBundles as $bundle) {
+      $this->dumpEntitiesBundle($typeName, $entityType, $bundle);
     }
+
     $this->accountSwitcher->switchBack();
   }
 
   /**
    * Generate import YAML for entities.
    *
+   * @param resource $fp
+   *   The file into which to write.
    * @param string $entityTypeName
    *   The entity type.
    * @param string $bundleName
@@ -218,10 +321,10 @@ class Dumper {
    * @param array $entities
    *   The entities.
    */
-  public function dumpEntities(string $entityTypeName, string $bundleName, array $entities) {
+  public function dumpEntities($fp, string $entityTypeName, string $bundleName, array $entities) {
     $array = $this->toArray($entities);
-    $path = $this->prepareDestination($entityTypeName, $bundleName);
-    file_put_contents($path, Yaml::dump($array, static::INLINE_DEPTH, 2));
+    fwrite($fp, Yaml::dump($array, static::INLINE_DEPTH, 2));
+    fflush($fp);
   }
 
   /**
@@ -250,6 +353,7 @@ class Dumper {
    * Store the absolute path to the import directory.
    *
    * @param string $path
+   *
    *   The Drupal-root-relative import path.
    *
    * @throws \InvalidArgumentException
@@ -283,4 +387,39 @@ class Dumper {
     return $hash;
   }
 
+  /**
+   * Deduplicate the bundles list and remove invalid bundle names.
+   *
+   * @param string $entityTypeName
+   *   The name of the entity type for which to validate bundle names.
+   * @param array $bundles
+   *   Bundle names to be validated.
+   *
+   * @return array
+   *   Valid bundles to dump.
+   */
+  protected function validateBundles(string $entityTypeName, array $bundles) {
+    sort($bundles);
+    $bundleInfo = $this->entityTypeBundleInfo->getBundleInfo($entityTypeName);
+
+    // Note: array_flip will ensure requested bundles are only asked once.
+    $flippedBundles = array_flip($bundles);
+
+    $validFlippedBundles = array_intersect_key($flippedBundles, $bundleInfo);
+    $uniqueValidBundles = array_flip($validFlippedBundles);
+
+    sort($uniqueValidBundles);
+    if ($bundles !== $uniqueValidBundles) {
+      throw new \InvalidArgumentException(
+          "\nRequested bundles: " . implode(', ', $bundles)
+        . "\nValid bundles: " . implode(', ', $uniqueValidBundles) . "\n");
+    }
+
+    if (empty($uniqueValidBundles)) {
+      $uniqueValidBundles = array_keys($bundleInfo);
+    }
+
+    return $uniqueValidBundles;
+  }
+
 }

src/EventSubscriber/UserPreImport.php

@@ -6,14 +6,26 @@ use Drupal\reinstall\ReinstallEvents;
 use Drupal\reinstall\SourceEvent;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 
+/**
+ * Class UserPreImport is an EventSubscriber filtering user sources.
+ */
 class UserPreImport implements EventSubscriberInterface {
 
+  /**
+   * {@inheritdoc}
+   */
   public static function getSubscribedEvents() {
     return [
       ReinstallEvents::POST_SOURCE_PARSE => 'onPreImport',
     ];
   }
 
+  /**
+   * Event callback for POST_SOURCE_PARSE.
+   *
+   * @param \Drupal\reinstall\SourceEvent $event
+   *   The event.
+   */
   public function onPreImport(SourceEvent $event) {
     $source = $event->source;
     if ($source->getConfiguration()['type'] !== 'user') {
@@ -21,7 +33,6 @@ class UserPreImport implements EventSubscriberInterface {
     }
 
     $event->source->records = array_filter($event->source->records, [$this, 'filter01']);
-    return;
   }
 
   /**

src/Plugin/migrate/source/ReinstallSourceBase.php

@@ -12,7 +12,6 @@ use Drupal\reinstall\ReinstallEvents;
 use Drupal\reinstall\SourceEvent;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
-use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 use Symfony\Component\Yaml\Exception\ParseException;
 use Symfony\Component\Yaml\Yaml;
 
@@ -42,11 +41,25 @@ class ReinstallSourceBase extends SourcePluginBase implements ContainerFactoryPl
    */
   public $records;
 
+  /**
+   * ReinstallSourceBase constructor.
+   *
+   * @param array $configuration
+   *   The plugin configuration.
+   * @param string $plugin_id
+   *   The plugin id.
+   * @param mixed $plugin_definition
+   *   The plugin definition.
+   * @param \Drupal\migrate\Plugin\MigrationInterface $migration
+   *   The migration on which the plugin is invoked.
+   * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $eventDispatcher
+   *   The event_dispatcher service.
+   */
   public function __construct(
     array $configuration,
     $plugin_id,
     $plugin_definition,
-    \Drupal\migrate\Plugin\MigrationInterface $migration,
+    MigrationInterface $migration,
     EventDispatcherInterface $eventDispatcher
   ) {
     parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
@@ -238,6 +251,7 @@ class ReinstallSourceBase extends SourcePluginBase implements ContainerFactoryPl
    * @return array
    *   An array of dependencies grouped by type (config, content, module,
    *   theme). For example:
+   *
    * @code
    *   array(
    *     'config' => array('user.role.anonymous', 'user.role.authenticated'),
@@ -253,4 +267,5 @@ class ReinstallSourceBase extends SourcePluginBase implements ContainerFactoryPl
   public function calculateDependencies() {
     return [];
   }
+
 }

src/Plugin/migrate/source/SimpleSourceTrait.php

@@ -2,25 +2,35 @@
 
 namespace Drupal\reinstall\Plugin\migrate\source;
 
-
+/**
+ * Class SimpleSourceTrait provides shared features for entity field handling.
+ */
 trait SimpleSourceTrait {
 
   /**
+   * The entity_type.bundle.info service.
+   *
    * @var \Drupal\Core\Entity\EntityTypeBundleInfoInterface
    */
   protected $sstEntityTypeBundleInfo;
 
   /**
+   * The entity_field.manager service.
+   *
    * @var \Drupal\Core\Entity\EntityFieldManagerInterface
    */
   protected $sstEntityFieldManager;
 
   /**
+   * The name of the entity type being handled.
+   *
    * @var string
    */
   protected $sstEntityType;
 
   /**
+   * The entity_type.manager service.
+   *
    * @var \Drupal\Core\Entity\EntityTypeManagerInterface
    */
   protected $sstEntityTypeManager;
@@ -66,6 +76,12 @@ trait SimpleSourceTrait {
     return $fields;
   }
 
+  /**
+   * Getter for the entity_type.bundle.info service.
+   *
+   * @return \Drupal\Core\Entity\EntityTypeBundleInfoInterface
+   *   The service.
+   */
   protected function getEntityTypeBundleInfo() {
     if (!isset($this->sstEntityTypeBundleInfo)) {
       $this->sstEntityTypeBundleInfo = \Drupal::service('entity_type.bundle.info');
@@ -74,6 +90,12 @@ trait SimpleSourceTrait {
     return $this->sstEntityTypeBundleInfo;
   }
 
+  /**
+   * Getter for the entity_field.manager service.
+   *
+   * @return \Drupal\Core\Entity\EntityFieldManagerInterface|mixed
+   *   The service.
+   */
   protected function getEntityFieldManager() {
     if (!isset($this->sstEntityFieldManager)) {
       $this->sstEntityFieldManager = \Drupal::service('entity_field.manager');
@@ -82,11 +104,23 @@ trait SimpleSourceTrait {
     return $this->sstEntityFieldManager;
   }
 
+  /**
+   * Getter for the current entity type.
+   *
+   * @return string
+   *   The machine name of the type.
+   */
   protected function getEntityType() {
     assert(isset($this->sstEntityType));
     return $this->sstEntityType;
   }
 
+  /**
+   * Getter for the entity_type.manager service.
+   *
+   * @return \Drupal\Core\Entity\EntityTypeManagerInterface|mixed
+   *   The service.
+   */
   protected function getEntityTypeManager() {
     if (!isset($this->sstEntityTypeManager)) {
       $this->sstEntityTypeManager = \Drupal::service('entity_type.manager');
@@ -94,6 +128,7 @@ trait SimpleSourceTrait {
 
     return $this->sstEntityTypeManager;
   }
+
   /**
    * {@inheritdoc}
    */

src/ReinstallEvents.php

@@ -51,4 +51,5 @@ final class ReinstallEvents {
   const PRE_DUMP = 'reinstall.dump.pre';
 
   const POST_SOURCE_PARSE = 'reinstall.source_parse.post';
+
 }

src/SourceEvent.php

@@ -2,11 +2,12 @@
 
 namespace Drupal\reinstall;
 
-use Drupal\migrate\Plugin\MigrateSourceInterface;
 use Drupal\reinstall\Plugin\migrate\source\ReinstallSourceBase;
 use Symfony\Component\EventDispatcher\Event;
 
 /**
+ * And event being triggered during dumps.
+ *
  * @see \Drupal\reinstall\ReinstallEvents::POST_SOURCE_PARSE
  */
 class SourceEvent extends Event {
@@ -14,7 +15,7 @@ class SourceEvent extends Event {
   /**
    * The source for the migration.
    *
-   * @var MigrateSourceInterface
+   * @var \Drupal\migrate\Plugin\MigrateSourceInterface
    */
   public $source;
 
@@ -22,6 +23,7 @@ class SourceEvent extends Event {
    * DumperEvent constructor.
    *
    * @param \Drupal\reinstall\Plugin\migrate\source\ReinstallSourceBase $source
+   *   The data source.
    */
   public function __construct(ReinstallSourceBase $source) {
     $this->source = $source;