WIP on reinstall with raw source option.

reinstall.services.yml

@@ -35,3 +35,8 @@ services:
     class: 'Drupal\reinstall\EventSubscriber\UserPreImport'
     tags:
       - { name: 'event_subscriber' }
+
+  reinstall.migrate.post_row_save:
+    class: 'Drupal\reinstall\EventSubscriber\IdPostRowSave'
+    tags:
+      - { name: 'event_subscriber' }

src/EventSubscriber/IdPostRowSave.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Drupal\reinstall\EventSubscriber;
+
+use Drupal\Component\Utility\NestedArray;
+use Drupal\migrate\Event\MigrateEvents;
+use Drupal\migrate\Event\MigratePostRowSaveEvent;
+use Drupal\migrate\Row;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Class IdPostRowSave normalizes ids after the destination saved the row.
+ *
+ * This is needed because raw reinstall sources do not flatten source values,
+ * and Sql::saveIdMap() assumes property fields are scalars of the described
+ * type for the source, so we need to overwrite the source value for this last
+ * step.
+ *
+ * Example: a node has an integer "nid" key, but is dumped like:
+ * <nid>:
+ *   nid:
+ *     -
+ *       value: 2
+ *   ...
+ */
+class IdPostRowSave implements EventSubscriberInterface {
+
+  public static function getSubscribedEvents() {
+    $events = [
+      MigrateEvents::POST_ROW_SAVE => 'postRowSave',
+    ];
+
+    return $events;
+  }
+
+  public function postRowSave(MigratePostRowSaveEvent $event) {
+    $row = $event->getRow();
+    $idValues = $row->getSourceIdValues();
+    foreach ($idValues as $idName => $idValue) {
+      $current = $idValue;
+      while (is_array($current)) {
+        $current = current($current);
+      }
+      // Unavailable because source is frozen at this point.
+      // $row->setSourceProperty($idName, $current);
+      // So let's use brute force.
+      NestedArray::setValue($row->getSource(), explode(Row::PROPERTY_SEPARATOR, $idName), $current, TRUE);
+    }
+
+    return;
+  }
+
+}

src/Plugin/migrate/process/FieldProcess.php

@@ -0,0 +1,298 @@
+<?php
+
+namespace Drupal\reinstall\Plugin\migrate\process;
+
+use Drupal\Core\Url;
+use Drupal\field\Entity\FieldStorageConfig;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\Row;
+
+/**
+ * Class FieldFilterProcess handles general field transforms.
+ *
+ * @MigrateProcessPlugin(
+ *   id = "reinstall_field",
+ *   handle_multiples = true
+ * )
+ */
+class FieldProcess extends ProcessPluginBase {
+
+  protected $isMultiple;
+
+  public function __construct(
+    array $configuration,
+    $plugin_id,
+    $plugin_definition
+  ) {
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+    if (isset($this->configuration['field'])) {
+      $this->fieldValues = array_flip(FieldStorageConfig::load($this->configuration['field'])->getSetting('allowed_values'));
+    }
+  }
+
+  /**
+   * Wrap an incoming value in a HTML comment on a new line.
+   *
+   * Optional: add prefix and suffix within the comment.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformComment(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    if (empty($value)) {
+      return NULL;
+    }
+
+    $prefix = $this->configuration['prefix'] ?? NULL;
+    $suffix = $this->configuration['suffix'] ?? NULL;
+    $ret = "\n<!--${prefix}${value}${suffix}-->\n";
+    return $ret;
+  }
+
+  /**
+   * Remove everything except the target it to avoid extra ID issues.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   *
+   */
+  public function transformEntityReference(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    // Force multiple handling on since ER which arrive as scalars.
+    $multiple = $this->configuration['multiple'] ?? NULL;
+
+    if (!is_array($value)) {
+      if ($multiple) {
+        $this->isMultiple = $multiple;
+        $value = [$value];
+      }
+      else {
+        $this->isMultiple = FALSE;
+      }
+
+      return $value;
+    }
+
+    $ret = [];
+    $isTrueArray = TRUE;
+    if (key($value) !== 0) {
+      $isTrueArray = FALSE;
+      $value = [$value];
+    }
+    foreach ($value as $delta => $item) {
+      $ret[$delta] = $item['target_id'];
+    }
+
+    $this->isMultiple = TRUE;
+
+    if (count($ret) == 1 && !$isTrueArray) {
+      $this->isMultiple = FALSE;
+      $ret = reset($ret);
+    }
+    return $ret;
+  }
+
+  /**
+   * Remove empty items from a list, just like array_filter
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformFilter(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    $this->isMultiple = TRUE;
+    return array_filter($value);
+  }
+
+  /**
+   * Trim an array to its first element.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformFirst(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    $this->isMultiple = FALSE;
+    $ret = reset($value);
+    if ($ret === FALSE) {
+      $ret = 0;
+    }
+    return $ret;
+  }
+
+  /**
+   * Extract a sub-key from a D6-style image field. Useless on > D7 sources.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformImageData(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    $key = $this->configuration['key'];
+    $arrayValue = unserialize($value);
+    $ret = $arrayValue[$key] ?? NULL;
+    return $ret;
+  }
+
+  /**
+   * Helper: build a search URL for a value.
+   *
+   * @param string $bundle
+   * @param string $query
+   * @return \Drupal\Core\Url
+   *
+   * @see \Drupal\reinstall\Plugin\migrate\process\FieldProcess::transformLink()
+   */
+  protected function buildNodeSearchLink(string $bundle, string $query) {
+    $url = Url::fromRoute('search.view_node_search', [], [
+      'query' => [
+        // XXX unsafe, don't do this outside a migration.
+        'keys' => $query,
+        'f[0]' => "type:${bundle}",
+      ],
+    ]);
+    return $url;
+  }
+
+  /**
+   * Transform a link field. Beware limitations.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformLink(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    if (empty($value)) {
+      $this->isMultiple = FALSE;
+      return NULL;
+    }
+    // CAVEAT: assumes it is used in a "multiple" case, and no attributes.
+    $ret = [];
+    foreach ($value as $item) {
+      $uri = $item['url'] ?? NULL;
+      $title = $item['title'] ?? NULL;
+
+      if (empty($uri)) {
+        if (empty($title)) {
+          continue;
+        }
+
+        $bundle = $this->configuration['search_bundle'];
+        $link = [
+          'uri' => $this->buildNodeSearchLink($bundle, $title)->toUriString(),
+          'title' => $title,
+          'options' => [],
+        ];
+      }
+      else {
+        // Build URL on the previous site scheme, not on the current one.
+        if (strpos($uri, 'http') !== 0) {
+          $uri = 'http://mag.plantes-et-jardins.com/' . $uri;
+        }
+
+        $link = [
+          'uri' => $uri,
+          'title' => $title,
+          'options' => [],
+        ];
+      }
+
+      $ret[] = $link;
+    }
+
+    $this->isMultiple = TRUE;
+    return $ret;
+  }
+
+  /**
+   * Like static_map, but based on the destination and not doing queries.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformLookupList(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    $ret = $this->fieldValues[$value] ?? NULL;
+    return $ret;
+  }
+
+  /**
+   * For multiple use only: push a configured value to the value array.
+   *
+   * @param mixed $value
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   * @param \Drupal\migrate\Row $row
+   * @param mixed $destination_property
+   * @return null|string
+   */
+  public function transformPush(
+    $value,
+    MigrateExecutableInterface $migrate_executable,
+    Row $row,
+    $destination_property
+  ) {
+    $this->isMultiple = TRUE;
+    $ret = $value;
+    $ret[] = $this->configuration['push'];
+    return $ret;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function multiple() {
+    return $this->isMultiple;
+  }
+
+}

src/Plugin/migrate/source/ReinstallSourceBase.php

@@ -21,6 +21,16 @@ use Symfony\Component\Yaml\Yaml;
  * @MigrateSource(
  *   id = "reinstall_base"
  * )
+ *
+ * Expected configuration in migration:source
+ * - bundle: the bundle name of the migrated entities
+ * - plugin: "reinstall_base", this plugin.
+ * - type: the type of the migrated entities
+ * - keys: the entity keys to use as the entities source keys when migrating.
+ *   - Defaults to ['id'].
+ *   - Revisionable content should use ['id', 'revision'].
+ *   - Maximum stability needs may be satisfied using ['uuid']
+ *   - Other variants might be useful.
  */
 class ReinstallSourceBase extends SourcePluginBase implements ContainerFactoryPluginInterface, ConfigurablePluginInterface {
   use SimpleSourceTrait;
@@ -64,7 +74,10 @@ class ReinstallSourceBase extends SourcePluginBase implements ContainerFactoryPl
   ) {
     parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
     $this->eventDispatcher = $eventDispatcher;
-    $this->records = array_map([$this, 'flattenRecord'], $this->initialParse($configuration));
+    $parsedRecords = $this->initialParse($configuration);
+    $this->records = empty($this->configuration['raw'])
+      ? array_map([$this, 'flattenRecord'], $parsedRecords)
+      : $parsedRecords;
     $eventDispatcher->dispatch(ReinstallEvents::POST_SOURCE_PARSE, new SourceEvent($this));
   }
 

src/Plugin/migrate/source/SimpleSourceTrait.php

@@ -134,20 +134,21 @@ trait SimpleSourceTrait {
    */
   public function getIds() {
     $typeName = $this->getEntityType();
-
     $typeDefinition = $this->getEntityTypeManager()->getDefinition($typeName);
-    $idName = $typeDefinition->getKey('id');
-    assert(!empty($idName));
-
-    $definitions = $this->getEntityFieldManager()->getBaseFieldDefinitions($typeName);
-    assert(isset($definitions[$idName]));
-    $idDefinition = $definitions[$idName];
-    $idType = $idDefinition->getType();
-    $ids = [
-      $idName => [
-        'type' => $idType,
-      ],
-    ];
+
+    $fieldManager = $this->getEntityFieldManager();
+    $definitions = $fieldManager->getBaseFieldDefinitions($typeName);
+
+    $keys = $this->configuration['keys'] ?? ['id'];
+    $ids = [];
+    foreach ($keys as $key) {
+      $keyName = $typeDefinition->getKey($key);
+      assert(!empty($keyName));
+      assert(isset($definitions[$keyName]));
+      $idDefinition = $definitions[$keyName];
+      $idType = $idDefinition->getType();
+      $ids[$keyName] = ['type' => $idType];
+    }
 
     return $ids;
   }