first commit

modules/token/tests/modules/token_module_test/src/Controller/TokenTreeBrowseController.php

@@ -0,0 +1,68 @@
+<?php
+
+namespace Drupal\token_module_test\Controller;
+
+use Drupal\Core\Controller\ControllerBase;
+use Drupal\token\TokenInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+use Symfony\Component\HttpFoundation\Request;
+
+/**
+ * Controller for token module test.
+ */
+class TokenTreeBrowseController extends ControllerBase {
+
+  /**
+   * Service to retrieve token information.
+   *
+   * @var \Drupal\token\TokenInterface
+   */
+  protected $token;
+
+  /**
+   * The construct method.
+   *
+   * @param \Drupal\token\TokenInterface $token
+   *   The token.
+   */
+  public function __construct(TokenInterface $token) {
+    $this->token = $token;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container) {
+    return new static(
+      $container->get('token')
+    );
+  }
+
+  /**
+   * Page callback to output a link.
+   */
+  public function outputLink(Request $request) {
+    $build['tree']['#theme'] = 'token_tree_link';
+    $build['tokenarea'] = [
+      '#markup' => $this->token->replace('[current-page:title]'),
+      '#type' => 'markup',
+    ];
+    return $build;
+  }
+
+  /**
+   * Title callback for the page outputting a link.
+   *
+   * We are using a title callback instead of directly defining the title in the
+   * routing YML file. This is so that we could return an array instead of a
+   * simple string. This allows us to test if [current-page:title] works with
+   * render arrays and other objects as titles.
+   */
+  public function getTitle() {
+    return [
+      '#type' => 'markup',
+      '#markup' => 'Available Tokens',
+    ];
+  }
+
+}

modules/token/tests/modules/token_module_test/test_views/views.view.token_views_test.yml

@@ -0,0 +1,130 @@
+langcode: en
+status: true
+dependencies:
+  module:
+    - user
+id: token_views_test
+label: token_views_test
+module: views
+description: ''
+tag: ''
+base_table: users_field_data
+base_field: uid
+core: 8.x
+display:
+  default:
+    display_plugin: default
+    id: default
+    display_title: Master
+    position: 0
+    display_options:
+      access:
+        type: none
+        options: {  }
+      cache:
+        type: tag
+        options: {  }
+      query:
+        type: views_query
+        options:
+          disable_sql_rewrite: false
+          distinct: false
+          replica: false
+          query_comment: ''
+          query_tags: {  }
+      exposed_form:
+        type: basic
+        options:
+          submit_button: Apply
+          reset_button: false
+          reset_button_label: Reset
+          exposed_sorts_label: 'Sort by'
+          expose_sort_order: true
+          sort_asc_label: Asc
+          sort_desc_label: Desc
+      pager:
+        type: some
+        options:
+          items_per_page: 5
+          offset: 0
+      style:
+        type: default
+      row:
+        type: fields
+        options:
+          default_field_elements: true
+          inline: {  }
+          separator: ''
+          hide_empty: false
+      fields:
+        name:
+          id: name
+          table: users_field_data
+          field: name
+          entity_type: user
+          entity_field: name
+          label: ''
+          alter:
+            alter_text: false
+            make_link: false
+            absolute: false
+            trim: false
+            word_boundary: false
+            ellipsis: false
+            strip_tags: false
+            html: false
+          hide_empty: false
+          empty_zero: false
+          plugin_id: field
+          relationship: none
+          group_type: group
+          admin_label: ''
+          exclude: false
+          element_type: ''
+          element_class: ''
+          element_label_type: ''
+          element_label_class: ''
+          element_label_colon: true
+          element_wrapper_type: ''
+          element_wrapper_class: ''
+          element_default_classes: true
+          empty: ''
+          hide_alter_empty: true
+          click_sort_column: value
+          type: user_name
+          settings: {  }
+          group_column: value
+          group_columns: {  }
+          group_rows: true
+          delta_limit: 0
+          delta_offset: 0
+          delta_reversed: false
+          delta_first_last: false
+          multi_type: separator
+          separator: ', '
+          field_api_classes: false
+      filters: {  }
+      sorts: {  }
+      title: token_views_test
+      header: {  }
+      footer: {  }
+      empty: {  }
+      relationships: {  }
+      arguments: {  }
+      display_extenders: {  }
+      filter_groups:
+        operator: AND
+        groups: {  }
+    cache_metadata:
+      max-age: -1
+      contexts:
+        - 'languages:language_content'
+        - 'languages:language_interface'
+      tags: {  }
+  block_1:
+    display_plugin: block
+    id: block_1
+    display_title: Block
+    position: 1
+    display_options:
+      display_extenders: {  }

modules/token/tests/modules/token_module_test/token_module_test.info.yml

@@ -0,0 +1,10 @@
+type: module
+name: Token Module Test
+description: Testing module for token functionality.
+package: Testing
+hidden: TRUE
+
+# Information added by Drupal.org packaging script on 2024-05-18
+version: '8.x-1.14+6-dev'
+project: 'token'
+datestamp: 1716022861

modules/token/tests/modules/token_module_test/token_module_test.module

@@ -0,0 +1,33 @@
+<?php
+
+/**
+ * @file
+ * Helper module for token tests.
+ */
+
+use Drupal\node\NodeInterface;
+
+/**
+ * Implements hook_page_attachments().
+ */
+function token_module_test_page_attachments() {
+  if ($debug = \Drupal::state()->get('token_page_tokens', [])) {
+    $debug += ['tokens' => [], 'data' => [], 'options' => []];
+    foreach (array_keys($debug['tokens']) as $token) {
+      $debug['values'][$token] = \Drupal::token()->replace($token, $debug['data'], $debug['options']);
+    }
+    \Drupal::state()->set('token_page_tokens', $debug);
+  }
+}
+
+/**
+ * Implements hook_ENTITY_TYPE_presave for Node entities.
+ */
+function token_module_test_node_presave(NodeInterface $node) {
+  // Transform tokens in the body.
+  // @see \Drupal\token\Tests\TokenMenuTest::testMenuTokens()
+  if ($node->hasField('body') && $node->get('body')->value) {
+    $node->body->value = \Drupal::token()
+      ->replace($node->body->value, ['node' => $node]);
+  }
+}

modules/token/tests/modules/token_module_test/token_module_test.routing.yml

@@ -0,0 +1,7 @@
+token_module_test.browse:
+  path: '/token_module_test/browse'
+  defaults:
+    _controller: '\Drupal\token_module_test\Controller\TokenTreeBrowseController::outputLink'
+    _title_callback: '\Drupal\token_module_test\Controller\TokenTreeBrowseController::getTitle'
+  requirements:
+    _access: 'TRUE'

modules/token/tests/modules/token_module_test/token_module_test.tokens.inc

@@ -0,0 +1,13 @@
+<?php
+
+/**
+ * Implements hook_token_info()
+ */
+function token_module_test_token_info() {
+  $info['tokens']['node']['colons:in:name'] = [
+    'name' => t('A test token with colons in the name'),
+    'description' => NULL,
+  ];
+
+  return $info;
+}

modules/token/tests/modules/token_user_picture/token_user_picture.info.yml

@@ -0,0 +1,12 @@
+type: module
+name: Token User picture
+description: Testing module that provides user pictures field.
+package: Testing
+hidden: TRUE
+dependencies:
+  - drupal:image
+
+# Information added by Drupal.org packaging script on 2024-05-18
+version: '8.x-1.14+6-dev'
+project: 'token'
+datestamp: 1716022861

modules/token/tests/src/Functional/TokenBlockTest.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\block_content\Entity\BlockContent;
+use Drupal\block_content\Entity\BlockContentType;
+
+/**
+ * Tests block tokens.
+ *
+ * @group token
+ */
+class TokenBlockTest extends TokenTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['block', 'node', 'views', 'block_content'];
+
+  /**
+   * Admin user.
+   *
+   * @var \Drupal\user\UserInterface
+   */
+  protected $adminUser;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->adminUser = $this->drupalCreateUser(['access content', 'administer blocks']);
+    $this->drupalLogin($this->adminUser);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function testBlockTitleTokens(): void {
+    $label = 'tokenblock';
+    $bundle = BlockContentType::create([
+      'id' => $label,
+      'label' => $label,
+      'revision' => FALSE
+    ]);
+    $bundle->save();
+
+    $block_content = BlockContent::create([
+      'type' => $label,
+      'label' => '[current-page:title] block title',
+      'info' => 'Test token title block',
+      'body[value]' => 'This is the test token title block.',
+    ]);
+    $block_content->save();
+
+    $block = $this->drupalPlaceBlock('block_content:' . $block_content->uuid(), [
+      'label' => '[user:name]',
+    ]);
+    $this->drupalGet($block->toUrl());
+    // Ensure that the link to available tokens is present and correctly
+    // positioned.
+    $this->assertSession()->linkExists('Browse available tokens.');
+    $this->assertSession()->pageTextContains('This field supports tokens. Browse available tokens.');
+    $this->submitForm([], 'Save block');
+    // Ensure token validation is working on the block.
+    $this->assertSession()->pageTextContains('Title is using the following invalid tokens: [user:name].');
+
+    // Create the block for real now with a valid title.
+    $settings = $block->get('settings');
+    $settings['label'] = '[current-page:title] block title';
+    $block->set('settings', $settings);
+    $block->save();
+
+    // Ensure that tokens are not double-escaped when output as a block title.
+    $this->drupalCreateContentType(['type' => 'page']);
+    $node = $this->drupalCreateNode(['title' => "Site's first node"]);
+    $this->drupalGet('node/' . $node->id());
+    // The apostraphe should only be escaped once.
+    $this->assertSession()->responseContains("Site&#039;s first node block title");
+  }
+}

modules/token/tests/src/Functional/TokenCurrentPageTest.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Core\Url;
+
+/**
+ * Test the [current-page:*] tokens.
+ *
+ * @group token
+ */
+class TokenCurrentPageTest extends TokenTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node'];
+
+  function testCurrentPageTokens() {
+    // Cache clear is necessary because the frontpage was already cached by an
+    // initial request.
+    $this->rebuildAll();
+    $tokens = [
+      '[current-page:title]' => 'Log in',
+      '[current-page:url]' => Url::fromRoute('user.login', [], ['absolute' => TRUE])->toString(),
+      '[current-page:url:absolute]' => Url::fromRoute('user.login', [], ['absolute' => TRUE])->toString(),
+      '[current-page:url:relative]' => Url::fromRoute('user.login')->toString(),
+      '[current-page:url:path]' => '/user/login',
+      '[current-page:url:args:value:0]' => 'user',
+      '[current-page:url:args:value:1]' => 'login',
+      '[current-page:url:args:value:2]' => NULL,
+      '[current-page:url:unaliased]' => Url::fromRoute('user.login', [], ['absolute' => TRUE, 'alias' => TRUE])->toString(),
+      '[current-page:page-number]' => 1,
+      '[current-page:query:foo]' => NULL,
+      '[current-page:query:bar]' => NULL,
+      // Deprecated tokens
+      '[current-page:arg:0]' => 'user',
+      '[current-page:arg:1]' => 'login',
+      '[current-page:arg:2]' => NULL,
+    ];
+    $this->assertPageTokens('user/login', $tokens);
+
+    $this->drupalCreateContentType(['type' => 'page']);
+    $node = $this->drupalCreateNode(['title' => 'Node title', 'path' => ['alias' => '/node-alias']]);
+    $tokens = [
+      '[current-page:title]' => 'Node title',
+      '[current-page:url]' => $node->toUrl('canonical', ['absolute' => TRUE])->toString(),
+      '[current-page:url:absolute]' => $node->toUrl('canonical', ['absolute' => TRUE])->toString(),
+      '[current-page:url:relative]' => $node->toUrl()->toString(),
+      '[current-page:url:alias]' => '/node-alias',
+      '[current-page:url:args:value:0]' => 'node-alias',
+      '[current-page:url:args:value:1]' => NULL,
+      '[current-page:url:unaliased]' => $node->toUrl('canonical', ['absolute' => TRUE, 'alias' => TRUE])->toString(),
+      '[current-page:url:unaliased:args:value:0]' => 'node',
+      '[current-page:url:unaliased:args:value:1]' => $node->id(),
+      '[current-page:url:unaliased:args:value:2]' => NULL,
+      '[current-page:page-number]' => 1,
+      '[current-page:query:foo]' => 'bar',
+      '[current-page:query:bar]' => NULL,
+      // Deprecated tokens
+      '[current-page:arg:0]' => 'node',
+      '[current-page:arg:1]' => 1,
+      '[current-page:arg:2]' => NULL,
+    ];
+    $this->assertPageTokens("/node/{$node->id()}", $tokens, [], ['url_options' => ['query' => ['foo' => 'bar']]]);
+  }
+}

modules/token/tests/src/Functional/TokenFieldUiTest.php

@@ -0,0 +1,289 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\field\Entity\FieldConfig;
+use Drupal\field\Entity\FieldStorageConfig;
+use Drupal\file\Entity\File;
+use Drupal\image\Entity\ImageStyle;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+
+/**
+ * Tests field ui.
+ *
+ * @group token
+ */
+class TokenFieldUiTest extends TokenTestBase {
+
+  /**
+   * @var \Drupal\Core\Session\AccountInterface
+   */
+  protected $adminUser;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['field_ui', 'node', 'image'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->adminUser = $this->drupalCreateUser(['bypass node access', 'administer content types', 'administer node fields']);
+    $this->drupalLogin($this->adminUser);
+
+    $node_type = NodeType::create([
+      'type' => 'article',
+      'name' => 'Article',
+      'description' => "Use <em>articles</em> for time-sensitive content like news, press releases or blog posts.",
+    ]);
+    $node_type->save();
+
+    FieldStorageConfig::create([
+      'field_name' => 'field_body',
+      'entity_type' => 'node',
+      'type' => 'text_with_summary',
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'field_body',
+      'label' => 'Body',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+    ])->save();
+    FieldStorageConfig::create([
+      'field_name' => 'field_image',
+      'entity_type' => 'node',
+      'type' => 'image',
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'field_image',
+      'label' => 'Image',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+    ])->save();
+    FieldStorageConfig::create([
+      'field_name' => 'field_image_2',
+      'entity_type' => 'node',
+      'type' => 'image',
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'field_image_2',
+      'label' => 'Image 2',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+    ])->save();
+    FieldStorageConfig::create([
+      'field_name' => 'multivalued_field_image',
+      'entity_type' => 'node',
+      'type' => 'image',
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'multivalued_field_image',
+      'label' => 'Multivalued field image',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+    ])->save();
+
+    \Drupal::service('entity_display.repository')->getFormDisplay('node', 'article', 'default')
+      ->setComponent('field_body', [
+        'type' => 'text_textarea_with_summary',
+        'settings' => [
+          'rows' => '9',
+          'summary_rows' => '3',
+        ],
+        'weight' => 5,
+      ])
+      ->save();
+  }
+
+  public function testFileFieldUi() {
+    $this->drupalGet('admin/structure/types/manage/article/fields/node.article.field_image');
+
+    // Ensure the 'Browse available tokens' link is present and correct.
+    $this->assertSession()->linkExists('Browse available tokens.');
+    $this->assertSession()->linkByHrefExists('token/tree');
+
+    // Ensure that the default file directory value validates correctly.
+    $this->submitForm([], 'Save settings');
+    $this->assertSession()->pageTextContains('Saved Image configuration.');
+  }
+
+  public function testFieldDescriptionTokens() {
+    $edit = [
+      'description' => 'The site is called [site:name].',
+    ];
+    $this->drupalGet('admin/structure/types/manage/article/fields/node.article.field_body');
+    $this->submitForm($edit, 'Save settings');
+
+    $this->drupalGet('node/add/article');
+    $this->assertSession()->pageTextContains('The site is called Drupal.');
+  }
+
+  /**
+   * Test that tokens are correctly provided and replaced for the image fields.
+   */
+  public function testImageFieldTokens() {
+    // Generate 2 different test images.
+    $file_system = \Drupal::service('file_system');
+    $file_system->copy(\Drupal::root() . '/core/misc/druplicon.png', 'public://example1.png');
+    $file_system->copy(\Drupal::root() . '/core/misc/loading.gif', 'public://example2.gif');
+
+    // Resize the test images so that they will be scaled down during token
+    // replacement.
+    $image1 = \Drupal::service('image.factory')->get('public://example1.png');
+    $image1->resize(500, 500);
+    $image1->save();
+    $image2 = \Drupal::service('image.factory')->get('public://example2.gif');
+    $image2->resize(500, 500);
+    $image2->save();
+
+    /** @var \Drupal\file\Entity\File $image1 */
+    $image1 = File::create(['uri' => 'public://example1.png']);
+    $image1->save();
+    /** @var \Drupal\file\Entity\File $image2 */
+    $image2 = File::create(['uri' => 'public://example2.gif']);
+    $image2->save();
+
+    $node = Node::create([
+      'title' => 'Test node title',
+      'type' => 'article',
+      'field_image' => [
+        [
+          'target_id' => $image1->id(),
+        ],
+      ],
+      'field_image_2' => [
+        [
+          'target_id' => $image2->id(),
+        ],
+      ],
+      'multivalued_field_image' => [
+        ['target_id' => $image1->id()],
+        ['target_id' => $image2->id()],
+      ],
+    ]);
+    $node->save();
+
+    // Obtain the file size and dimension of the images that will be scaled
+    // down during token replacement by applying the styles here.
+    $style_thumbnail = ImageStyle::load('thumbnail');
+    $style_thumbnail->createDerivative('public://example1.png', 'public://styles/thumbnail/public/example1-test.png');
+    $style_thumbnail->createDerivative('public://example2.gif', 'public://styles/thumbnail/public/example2-test.gif');
+    $image_1_thumbnail = \Drupal::service('image.factory')->get('public://styles/thumbnail/public/example1-test.png');
+    $image_2_thumbnail = \Drupal::service('image.factory')->get('public://styles/thumbnail/public/example2-test.gif');
+    $style_medium = ImageStyle::load('medium');
+    $style_medium->createDerivative('public://example1.png', 'public://styles/medium/public/example1-test.png');
+    $style_medium->createDerivative('public://example2.gif', 'public://styles/medium/public/example2-test.gif');
+    $image_1_medium = \Drupal::service('image.factory')->get('public://styles/medium/public/example1-test.png');
+    $image_2_medium = \Drupal::service('image.factory')->get('public://styles/medium/public/example2-test.gif');
+    $style_large = ImageStyle::load('large');
+    $style_large->createDerivative('public://example1.png', 'public://styles/large/public/example1-test.png');
+    $style_large->createDerivative('public://example2.gif', 'public://styles/large/public/example2-test.gif');
+    $image_1_large = \Drupal::service('image.factory')->get('public://styles/large/public/example1-test.png');
+    $image_2_large = \Drupal::service('image.factory')->get('public://styles/large/public/example2-test.gif');
+
+    // Delete the image derivatives, to make sure they are re-created.
+    unlink('public://styles/thumbnail/public/example1-test.png');
+    unlink('public://styles/medium/public/example1-test.png');
+    unlink('public://styles/large/public/example1-test.png');
+    unlink('public://styles/thumbnail/public/example2-test.gif');
+    unlink('public://styles/medium/public/example2-test.gif');
+    unlink('public://styles/large/public/example2-test.gif');
+
+    $style_mimetype_png = 'image/webp';
+    $style_mimetype_gif = 'image/webp';
+    if (version_compare(\Drupal::VERSION, '10.3', '<')) {
+      $style_mimetype_png = 'image/png';
+      $style_mimetype_gif = 'image/gif';
+    }
+
+    $tokens = [
+      // field_image
+      'field_image:thumbnail:mimetype' => $style_mimetype_png,
+      'field_image:medium:mimetype' => $style_mimetype_png,
+      'field_image:large:mimetype' => $style_mimetype_png,
+      'field_image:thumbnail:filesize' => $image_1_thumbnail->getFileSize(),
+      'field_image:medium:filesize' => $image_1_medium->getFileSize(),
+      'field_image:large:filesize' => $image_1_large->getFileSize(),
+      'field_image:thumbnail:height' => '100',
+      'field_image:medium:height' => '220',
+      'field_image:large:height' => '480',
+      'field_image:thumbnail:width' => '100',
+      'field_image:medium:width' => '220',
+      'field_image:large:width' => '480',
+      'field_image:thumbnail:uri' => $style_thumbnail->buildUri('public://example1.png'),
+      'field_image:medium:uri' => $style_medium->buildUri('public://example1.png'),
+      'field_image:large:uri' => $style_large->buildUri('public://example1.png'),
+      'field_image:thumbnail:url' => $style_thumbnail->buildUrl('public://example1.png'),
+      'field_image:medium:url' => $style_medium->buildUrl('public://example1.png'),
+      'field_image:large:url' => $style_large->buildUrl('public://example1.png'),
+      'field_image:thumbnail' => $style_thumbnail->buildUrl('public://example1.png'),
+      'field_image:medium' => $style_medium->buildUrl('public://example1.png'),
+      'field_image:large' => $style_large->buildUrl('public://example1.png'),
+      // field_image_2
+      'field_image_2:thumbnail:mimetype' => $style_mimetype_gif,
+      'field_image_2:medium:mimetype' => $style_mimetype_gif,
+      'field_image_2:large:mimetype' => $style_mimetype_gif,
+      'field_image_2:thumbnail:filesize' => $image_2_thumbnail->getFileSize(),
+      'field_image_2:medium:filesize' => $image_2_medium->getFileSize(),
+      'field_image_2:large:filesize' => $image_2_large->getFileSize(),
+      'field_image_2:thumbnail:height' => '100',
+      'field_image_2:medium:height' => '220',
+      'field_image_2:large:height' => '480',
+      'field_image_2:thumbnail:width' => '100',
+      'field_image_2:medium:width' => '220',
+      'field_image_2:large:width' => '480',
+      'field_image_2:thumbnail:uri' => $style_thumbnail->buildUri('public://example2.gif'),
+      'field_image_2:medium:uri' => $style_medium->buildUri('public://example2.gif'),
+      'field_image_2:large:uri' => $style_large->buildUri('public://example2.gif'),
+      'field_image_2:thumbnail:url' => $style_thumbnail->buildUrl('public://example2.gif'),
+      'field_image_2:medium:url' => $style_medium->buildUrl('public://example2.gif'),
+      'field_image_2:large:url' => $style_large->buildUrl('public://example2.gif'),
+      'field_image_2:thumbnail' => $style_thumbnail->buildUrl('public://example2.gif'),
+      'field_image_2:medium' => $style_medium->buildUrl('public://example2.gif'),
+      'field_image_2:large' => $style_large->buildUrl('public://example2.gif'),
+      // multivalued_field_image:0, test for thumbnail image style only.
+      'multivalued_field_image:0:thumbnail:mimetype' => $style_mimetype_png,
+      'multivalued_field_image:0:thumbnail:filesize' => $image_1_thumbnail->getFileSize(),
+      'multivalued_field_image:0:thumbnail:height' => '100',
+      'multivalued_field_image:0:thumbnail:width' => '100',
+      'multivalued_field_image:0:thumbnail:uri' => $style_thumbnail->buildUri('public://example1.png'),
+      'multivalued_field_image:0:thumbnail:url' => $style_thumbnail->buildUrl('public://example1.png'),
+      'multivalued_field_image:0:thumbnail' => $style_thumbnail->buildUrl('public://example1.png'),
+      // multivalued_field_image:1, test for medium image style only.
+      'multivalued_field_image:1:medium:mimetype' => $style_mimetype_gif,
+      'multivalued_field_image:1:medium:filesize' => $image_2_medium->getFileSize(),
+      'multivalued_field_image:1:medium:height' => '220',
+      'multivalued_field_image:1:medium:width' => '220',
+      'multivalued_field_image:1:medium:uri' => $style_medium->buildUri('public://example2.gif'),
+      'multivalued_field_image:1:medium:url' => $style_medium->buildUrl('public://example2.gif'),
+      'multivalued_field_image:1:medium' => $style_medium->buildUrl('public://example2.gif'),
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens);
+
+    /** @var \Drupal\token\Token $token_service */
+    $token_service = \Drupal::service('token');
+
+    // Test one of the image style's token info for cardinality 1 image field.
+    $token_info = $token_service->getTokenInfo('node-field_image', 'thumbnail');
+    $this->assertEquals('Thumbnail (100×100)', $token_info['name']);
+    $this->assertEquals('Represents the image in the given image style.', $token_info['description']);
+
+    // Test one of the image style's token info for a multivalued image field.
+    $token_info = $token_service->getTokenInfo('node-multivalued_field_image', 'medium');
+    $this->assertEquals('Medium (220×220)', $token_info['name']);
+    $this->assertEquals('Represents the image in the given image style.', $token_info['description']);
+
+    // Test few of the image styles' properties token info.
+    $token_info = $token_service->getTokenInfo('image_with_image_style', 'mimetype');
+    $this->assertEquals('MIME type', $token_info['name']);
+    $this->assertEquals('The MIME type (image/png, image/bmp, etc.) of the image.', $token_info['description']);
+
+    $token_info = $token_service->getTokenInfo('image_with_image_style', 'filesize');
+    $this->assertEquals('File size', $token_info['name']);
+    $this->assertEquals('The file size of the image.', $token_info['description']);
+  }
+
+}

modules/token/tests/src/Functional/TokenMenuTest.php

@@ -0,0 +1,496 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Behat\Mink\Element\NodeElement;
+use Drupal\Core\Url;
+use Drupal\language\Entity\ConfigurableLanguage;
+use Drupal\menu_link_content\Entity\MenuLinkContent;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+use Drupal\system\Entity\Menu;
+
+/**
+ * Tests menu tokens.
+ *
+ * @group token
+ */
+class TokenMenuTest extends TokenTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'menu_ui',
+    'node',
+    'block',
+    'language',
+    'block_content',
+    'content_translation',
+  ];
+
+  function testMenuTokens() {
+
+    $admin = $this->drupalCreateUser(['administer menu', 'link to any page']);
+    $this->drupalLogin($admin);
+
+    // Make sure we have a body field on the node type.
+    $this->drupalCreateContentType(['type' => 'page']);
+    // Add a menu.
+    $menu = Menu::create([
+      'id' => 'main-menu',
+      'label' => 'Main menu',
+      'description' => 'The <em>Main</em> menu is used on many sites to show the major sections of the site, often in a top navigation bar.',
+    ]);
+    $menu->save();
+
+    // Place the menu block.
+    $this->drupalPlaceBlock('system_menu_block:main-menu');
+
+    // Add a root link.
+    /** @var \Drupal\menu_link_content\Plugin\Menu\MenuLinkContent $root_link */
+    $root_link = MenuLinkContent::create([
+      'link' => ['uri' => 'internal:/admin'],
+      'title' => 'Administration',
+      'menu_name' => 'main-menu',
+    ]);
+    $root_link->save();
+
+    // Add another link with the root link as the parent.
+    /** @var \Drupal\menu_link_content\Plugin\Menu\MenuLinkContent $parent_link */
+    $parent_link = MenuLinkContent::create([
+      'link' => ['uri' => 'internal:/admin/config'],
+      'title' => 'Configuration',
+      'menu_name' => 'main-menu',
+      'parent' => $root_link->getPluginId(),
+    ]);
+    $parent_link->save();
+
+    // Test menu link tokens.
+    $tokens = [
+      'id' => $parent_link->getPluginId(),
+      'title' => 'Configuration',
+      'menu' => 'Main menu',
+      'menu:name' => 'Main menu',
+      'menu:machine-name' => $menu->id(),
+      'menu:description' => 'The <em>Main</em> menu is used on many sites to show the major sections of the site, often in a top navigation bar.',
+      'menu:menu-link-count' => '2',
+      'menu:edit-url' => Url::fromRoute('entity.menu.edit_form', ['menu' => 'main-menu'], ['absolute' => TRUE])->toString(),
+      'url' => Url::fromRoute('system.admin_config', [], ['absolute' => TRUE])->toString(),
+      'url:absolute' => Url::fromRoute('system.admin_config', [], ['absolute' => TRUE])->toString(),
+      'url:relative' => Url::fromRoute('system.admin_config', [], ['absolute' => FALSE])->toString(),
+      'url:path' => '/admin/config',
+      'url:alias' => '/admin/config',
+      'edit-url' => Url::fromRoute('entity.menu_link_content.canonical', ['menu_link_content' => $parent_link->id()], ['absolute' => TRUE])->toString(),
+      'parent' => 'Administration',
+      'parent:id' => $root_link->getPluginId(),
+      'parent:title' => 'Administration',
+      'parent:menu' => 'Main menu',
+      'parent:parent' => NULL,
+      'parents' => 'Administration',
+      'parents:count' => 1,
+      'parents:keys' => $root_link->getPluginId(),
+      'root' => 'Administration',
+      'root:id' => $root_link->getPluginId(),
+      'root:parent' => NULL,
+      'root:root' => NULL,
+    ];
+    $this->assertTokens('menu-link', ['menu-link' => $parent_link], $tokens);
+
+    // Add a node.
+    $node = $this->drupalCreateNode();
+
+    // Allow main menu for this node type.
+    //$this->config('menu.entity.node.' . $node->getType())->set('available_menus', ['main-menu'])->save();
+
+    // Add a node menu link.
+    /** @var \Drupal\menu_link_content\MenuLinkContentInterface $node_link */
+    $node_link = MenuLinkContent::create([
+      'link' => ['uri' => 'entity:node/' . $node->id()],
+      'title' => 'Node link',
+      'parent' => $parent_link->getPluginId(),
+      'menu_name' => 'main-menu',
+    ]);
+    $node_link->save();
+
+    // Test [node:menu] tokens.
+    $tokens = [
+      'menu-link' => 'Node link',
+      'menu-link:id' => $node_link->getPluginId(),
+      'menu-link:title' => 'Node link',
+      'menu-link:menu' => 'Main menu',
+      'menu-link:url' => $node->toUrl('canonical', ['absolute' => TRUE])->toString(),
+      'menu-link:url:path' => '/node/' . $node->id(),
+      'menu-link:edit-url' => $node_link->toUrl('edit-form', ['absolute' => TRUE])->toString(),
+      'menu-link:parent' => 'Configuration',
+      'menu-link:parent:id' => $parent_link->getPluginId(),
+      'menu-link:parents' => 'Administration, Configuration',
+      'menu-link:parents:count' => 2,
+      'menu-link:parents:keys' => $root_link->getPluginId() . ', ' . $parent_link->getPluginId(),
+      'menu-link:root' => 'Administration',
+      'menu-link:root:id' => $root_link->getPluginId(),
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens);
+
+    // Reload the node which will not have $node->menu defined and re-test.
+    $loaded_node = Node::load($node->id());
+    $this->assertTokens('node', ['node' => $loaded_node], $tokens);
+
+    // Regression test for http://drupal.org/node/1317926 to ensure the
+    // original node object is not changed when calling menu_node_prepare().
+    $this->assertTrue(!isset($loaded_node->menu), 'The $node->menu property was not modified during token replacement.', 'Regression');
+
+    // Now add a node with a menu-link from the UI and ensure it works.
+    $this->drupalLogin($this->drupalCreateUser([
+      'create page content',
+      'edit any page content',
+      'administer menu',
+      'administer nodes',
+      'administer content types',
+      'access administration pages',
+      'administer site configuration'
+    ]));
+    // Setup node type menu options.
+    $edit = [
+      'menu_options[main-menu]' => 1,
+      'menu_options[main]' => 1,
+      'menu_parent' => 'main-menu:',
+    ];
+    $this->drupalGet('admin/structure/types/manage/page');
+    $this->submitForm($edit, 'Save');
+
+    // Use a menu-link token in the body.
+    $this->drupalGet('node/add/page');
+    $this->submitForm([
+      // This should get replaced on save.
+      // @see token_module_test_node_presave()
+      'title[0][value]' => 'Node menu title test',
+      'body[0][value]' => 'This is a [node:menu-link:title] token to the menu link title',
+      'menu[enabled]' => 1,
+      'menu[title]' => 'Test preview',
+    ], 'Save');
+    $node = $this->drupalGetNodeByTitle('Node menu title test');
+    $this->assertEquals('This is a Test preview token to the menu link title', $node->body->value);
+
+    // Disable the menu link, save the node and verify that the menu link is
+    // no longer displayed.
+    $link = menu_ui_get_menu_link_defaults($node);
+    $this->drupalGet('admin/structure/menu/manage/main-menu');
+    $this->submitForm(['links[menu_plugin_id:' . $link['id'] . '][enabled]' => FALSE], 'Save');
+    $this->assertSession()->pageTextContains('Menu Main menu has been updated.');
+    $this->drupalGet('node/' . $node->id() . '/edit');
+    $this->submitForm([], 'Save');
+    $this->assertSession()->linkNotExists('Test preview');
+
+    // Now test a parent link and token.
+    $this->drupalGet('node/add/page');
+    // Make sure that the previous node save didn't result in two menu-links
+    // being created by the computed menu-link ER field.
+    // @see token_entity_base_field_info()
+    // @see token_node_menu_link_submit()
+    $selects = $this->cssSelect('select[name="menu[menu_parent]"]');
+    $select = reset($selects);
+    $options = $select->findAll('css', 'option');
+    // Filter to items with title containing 'Test preview'.
+    $options = array_filter($options, function (NodeElement $element) {
+      return strpos($element->getText(), 'Test preview') !== FALSE;
+    });
+    $this->assertCount(1, $options);
+    $this->submitForm([
+      'title[0][value]' => 'Node menu title parent path test',
+      'body[0][value]' => 'This is a [node:menu-link:parent:url:path] token to the menu link parent',
+      'menu[enabled]' => 1,
+      'menu[title]' => 'Child link',
+      'menu[menu_parent]' => 'main-menu:' . $parent_link->getPluginId(),
+    ], 'Save');
+    $node = $this->drupalGetNodeByTitle('Node menu title parent path test');
+    $this->assertEquals('This is a /admin/config token to the menu link parent', $node->body->value);
+
+    // Now edit the node and update the parent and title.
+    $this->drupalGet('node/' . $node->id() . '/edit');
+    $this->submitForm([
+      'menu[menu_parent]' => 'main-menu:' . $node_link->getPluginId(),
+      'title[0][value]' => 'Node menu title edit parent path test',
+      'body[0][value]' => 'This is a [node:menu-link:parent:url:path] token to the menu link parent',
+    ], 'Save');
+    $node = $this->drupalGetNodeByTitle('Node menu title edit parent path test', TRUE);
+    $this->assertEquals(sprintf('This is a /node/%d token to the menu link parent', $loaded_node->id()), $node->body->value);
+
+    // Make sure that the previous node edit didn't result in two menu-links
+    // being created by the computed menu-link ER field.
+    // @see token_entity_base_field_info()
+    // @see token_node_menu_link_submit()
+    $this->drupalGet('node/add/page');
+    $selects = $this->cssSelect('select[name="menu[menu_parent]"]');
+    $select = reset($selects);
+    $options = $select->findAll('css', 'option');
+    // Filter to items with title containing 'Test preview'.
+    $options = array_filter($options, function (NodeElement $item) {
+      return strpos($item->getText(), 'Child link') !== FALSE;
+    });
+    $this->assertCount(1, $options);
+
+    // Now add a new node with no menu.
+    $this->drupalGet('node/add/page');
+    $this->submitForm([
+      'title[0][value]' => 'Node menu adding menu later test',
+      'body[0][value]' => 'Going to add a menu link on edit',
+      'menu[enabled]' => 0,
+    ], 'Save');
+    $node = $this->drupalGetNodeByTitle('Node menu adding menu later test');
+    // Now edit it and add a menu item.
+    $this->drupalGet('node/' . $node->id() . '/edit');
+    $this->submitForm([
+      'title[0][value]' => 'Node menu adding menu later test',
+      'body[0][value]' => 'This is a [node:menu-link:parent:url:path] token to the menu link parent',
+      'menu[enabled]' => 1,
+      'menu[title]' => 'Child link',
+      'menu[menu_parent]' => 'main-menu:' . $parent_link->getPluginId(),
+    ], 'Save');
+    $node = $this->drupalGetNodeByTitle('Node menu adding menu later test', TRUE);
+    $this->assertEquals('This is a /admin/config token to the menu link parent', $node->body->value);
+    // And make sure the menu link exists with the right URI.
+    $link = menu_ui_get_menu_link_defaults($node);
+    $this->assertNotEmpty($link['entity_id']);
+    $query = \Drupal::entityQuery('menu_link_content')
+      ->condition('link.uri', 'entity:node/' . $node->id())
+      ->accessCheck(TRUE)
+      ->sort('id', 'ASC')
+      ->range(0, 1);
+    $result = $query->execute();
+    $this->assertNotEmpty($result);
+
+    // Create a node with a menu link and create 2 menu links linking to this
+    // node after. Verify that the menu link provided by the node has priority.
+    $node_title = $this->randomMachineName();
+    $edit = [
+      'title[0][value]' => $node_title,
+      'menu[enabled]' => 1,
+      'menu[title]' => 'menu link provided by node',
+    ];
+    $this->drupalGet('node/add/page');
+    $this->submitForm($edit, 'Save');
+    $this->assertSession()->pageTextContains('page ' . $node_title . ' has been created');
+    $node = $this->drupalGetNodeByTitle($node_title);
+
+    $menu_ui_link1 = MenuLinkContent::create([
+      'link' => ['uri' => 'entity:node/' . $node->id()],
+      'title' => 'menu link 1 provided by menu ui',
+      'menu_name' => 'main-menu',
+    ]);
+    $menu_ui_link1->save();
+
+    $menu_ui_link2 = MenuLinkContent::create([
+      'link' => ['uri' => 'entity:node/' . $node->id()],
+      'title' => 'menu link 2 provided by menu ui',
+      'menu_name' => 'main-menu',
+    ]);
+    $menu_ui_link2->save();
+
+    $tokens = [
+      'menu-link' => 'menu link provided by node',
+      'menu-link:title' => 'menu link provided by node',
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens);
+  }
+
+  /**
+   * Tests that the module doesn't affect integrity of the menu, when
+   * translating them and that menu links tokens are correct.
+   */
+  function testMultilingualMenu() {
+    // Place the menu block.
+    $this->drupalPlaceBlock('system_menu_block:main');
+
+    // Add a second language.
+    $language = ConfigurableLanguage::create([
+      'id' => 'de',
+      'label' => 'German',
+    ]);
+    $language->save();
+
+    // Create the article content type.
+    $node_type = NodeType::create([
+      'type' => 'article',
+      'name' => 'Article',
+    ]);
+    $node_type->save();
+
+    $permissions = [
+      'access administration pages',
+      'administer content translation',
+      'administer content types',
+      'administer languages',
+      'create content translations',
+      'create article content',
+      'edit any article content',
+      'translate any entity',
+      'administer menu',
+    ];
+    $this->drupalLogin($this->drupalCreateUser($permissions));
+
+    // Enable translation for articles and menu links.
+    $this->drupalGet('admin/config/regional/content-language');
+    $edit = [
+      'entity_types[node]' => TRUE,
+      'entity_types[menu_link_content]' => TRUE,
+      'settings[node][article][translatable]' => TRUE,
+      'settings[node][article][fields][title]' => TRUE,
+      'settings[menu_link_content][menu_link_content][translatable]' => TRUE,
+    ];
+    $this->submitForm($edit, 'Save configuration');
+    $this->assertSession()->pageTextContains('Settings successfully updated.');
+
+    // Create an english node with an english menu.
+    $this->drupalGet('/node/add/article');
+    $edit = [
+      'title[0][value]' => 'English test node with menu',
+      'menu[enabled]' => TRUE,
+      'menu[title]' => 'English menu title',
+    ];
+    $this->drupalGet('/node/add/article');
+    $this->submitForm($edit, 'Save');
+    $this->assertSession()->pageTextContains('English test node with menu has been created.');
+
+    // Add a german translation.
+    $this->drupalGet('node/1/translations');
+    $this->clickLink('Add');
+    $edit = [
+      'title[0][value]' => 'German test node with menu',
+      'menu[enabled]' => TRUE,
+      'menu[title]' => 'German menu title',
+    ];
+    $this->submitForm($edit, 'Save (this translation)');
+    $this->assertSession()->pageTextContains('German test node with menu has been updated.');
+
+    // Verify that the menu links are correct.
+    $this->drupalGet('node/1');
+    $this->assertSession()->linkExists('English menu title');
+    $this->drupalGet('de/node/1');
+    $this->assertSession()->linkExists('German menu title');
+
+    // Verify that tokens are correct.
+    $node = Node::load(1);
+    $this->assertTokens('node', ['node' => $node], ['menu-link' => 'English menu title']);
+    $this->assertTokens('node', ['node' => $node], [
+      'menu-link' => 'German menu title',
+      'menu-link:title' => 'German menu title',
+    ], ['langcode' => 'de']);
+
+    // Get the menu link and create a child menu link to assert parent and root
+    // tokens.
+    $url = $node->toUrl();
+    /** @var \Drupal\Core\Menu\MenuLinkManagerInterface $menu_link_manager */
+    $menu_link_manager = \Drupal::service('plugin.manager.menu.link');
+    $links = $menu_link_manager->loadLinksByRoute($url->getRouteName(), $url->getRouteParameters());
+    $link = reset($links);
+
+    $base_options = [
+      'provider' => 'menu_test',
+      'menu_name' => 'menu_test',
+    ];
+    $child_1 = $base_options + [
+        'title' => 'child_1 title EN',
+        'link' => ['uri' => 'internal:/menu-test/hierarchy/parent/child_1'],
+        'parent' => $link->getPluginId(),
+        'langcode' => 'en',
+      ];
+    $child_1 = MenuLinkContent::create($child_1);
+    $child_1->save();
+
+    // Add the german translation.
+    $child_1->addTranslation('de', ['title' => 'child_1 title DE'] + $child_1->toArray());
+    $child_1->save();
+
+    $this->assertTokens('menu-link', ['menu-link' => $child_1], [
+      'title' => 'child_1 title EN',
+      'parents' => 'English menu title',
+      'root' => 'English menu title',
+    ]);
+    $this->assertTokens('menu-link', ['menu-link' => $child_1], [
+      'title' => 'child_1 title DE',
+      'parents' => 'German menu title',
+      'root' => 'German menu title',
+    ], ['langcode' => 'de']);
+  }
+
+  /**
+   * Tests menu link parents token.
+   */
+  public function testMenuLinkParentsToken() {
+    // Create a menu with a simple link hierarchy :
+    // - parent
+    //   - child-1
+    //      - child-1-1
+    Menu::create([
+      'id' => 'menu_test',
+      'label' => 'Test menu',
+    ])->save();
+    $base_options = [
+      'provider' => 'menu_test',
+      'menu_name' => 'menu_test',
+    ];
+    $parent = $base_options + [
+        'title' => 'parent title',
+        'link' => ['uri' => 'internal:/menu-test/hierarchy/parent'],
+    ];
+    $parent = MenuLinkContent::create($parent);
+    $parent->save();
+    $child_1 = $base_options + [
+        'title' => 'child_1 title',
+        'link' => ['uri' => 'internal:/menu-test/hierarchy/parent/child_1'],
+        'parent' => $parent->getPluginId(),
+    ];
+    $child_1 = MenuLinkContent::create($child_1);
+    $child_1->save();
+    $child_1_1 = $base_options + [
+        'title' => 'child_1_1 title',
+        'link' => ['uri' => 'internal:/menu-test/hierarchy/parent/child_1/child_1_1'],
+        'parent' => $child_1->getPluginId(),
+    ];
+    $child_1_1 = MenuLinkContent::create($child_1_1);
+    $child_1_1->save();
+
+    $this->assertTokens('menu-link', ['menu-link' => $child_1_1], ['parents' => 'parent title, child_1 title']);
+
+    // Change the parent of child_1_1 to 'parent' at the entity level.
+    $child_1_1->parent->value = $parent->getPluginId();
+    $child_1_1->save();
+
+    $this->assertTokens('menu-link', ['menu-link' => $child_1_1], ['parents' => 'parent title']);
+
+    // Change the parent of child_1_1 to 'main', at the entity level.
+    $child_1_1->parent->value = '';
+    $child_1_1->save();
+
+    // The token shouldn't have been generated; the menu link has no parent.
+    $this->assertNoTokens('menu-link', ['menu-link' => $child_1_1], ['parents']);
+  }
+
+  /**
+   * Tests that no menu link is generated when the node gets previewed.
+   */
+  public function testPreviewMenuLink() {
+    $this->drupalCreateContentType(['type' => 'article']);
+    $permissions = [
+      'access administration pages',
+      'administer content types',
+      'create article content',
+      'edit any article content',
+      'administer menu',
+    ];
+    $this->drupalLogin($this->drupalCreateUser($permissions));
+    // Create an english node with an english menu.
+    $this->drupalGet('/node/add/article');
+    $edit = [
+      'title[0][value]' => 'English test node with menu',
+      'menu[enabled]' => TRUE,
+      'menu[title]' => 'English menu title',
+    ];
+    $this->drupalGet('node/add/article');
+    $this->submitForm($edit, 'Preview');
+    $menu_links = \Drupal::entityTypeManager()->getStorage('menu_link_content')->loadByProperties(['menu_name' => 'main']);
+    $this->assertEmpty($menu_links);
+  }
+
+}

modules/token/tests/src/Functional/TokenMenuUiContentModerationTest.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Tests\menu_ui\Functional\MenuUiContentModerationTest;
+
+/**
+ * Tests Menu UI and Content Moderation integration.
+ *
+ * @group token
+ */
+class TokenMenuUiContentModerationTest extends MenuUiContentModerationTest {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['token'];
+
+}

modules/token/tests/src/Functional/TokenTestBase.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Tests\BrowserTestBase;
+
+/**
+ * Helper test class with some added functions for testing.
+ */
+abstract class TokenTestBase extends BrowserTestBase {
+
+  use TokenTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['path', 'token', 'token_module_test'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected $defaultTheme = 'stark';
+
+}

modules/token/tests/src/Functional/TokenTestTrait.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Core\Entity\EntityInterface;
+use Drupal\Core\Language\Language;
+use Drupal\Core\Render\BubbleableMetadata;
+use Drupal\Tests\Traits\Core\PathAliasTestTrait;
+
+/**
+ * Helper test trait with some added functions for testing.
+ */
+trait TokenTestTrait {
+
+  use PathAliasTestTrait;
+
+  function assertToken($type, array $data, $token, $expected, array $options = []) {
+    return $this->assertTokens($type, $data, [$token => $expected], $options);
+  }
+
+  function assertTokens($type, array $data, array $tokens, array $options = []) {
+    $input = $this->mapTokenNames($type, array_keys($tokens));
+    $bubbleable_metadata = new BubbleableMetadata();
+    $replacements = \Drupal::token()->generate($type, $input, $data, $options, $bubbleable_metadata);
+    foreach ($tokens as $name => $expected) {
+      $token = $input[$name];
+      if (!isset($expected)) {
+        $this->assertArrayNotHasKey($token, $replacements, t("Token value for @token was not generated.", ['@token' => $token]));
+      }
+      elseif (!isset($replacements[$token])) {
+        $this->fail(t("Token value for @token was not generated.", ['@type' => $type, '@token' => $token]));
+      }
+      elseif (!empty($options['regex'])) {
+        $this->assertEquals(1, preg_match('/^' . $expected . '$/', $replacements[$token]), t("Token value for @token was '@actual', matching regular expression pattern '@expected'.", ['@type' => $type, '@token' => $token, '@actual' => $replacements[$token], '@expected' => $expected]));
+      }
+      else {
+        $this->assertEquals($expected, $replacements[$token], t("Token value for @token was '@actual', expected value '@expected'.", ['@type' => $type, '@token' => $token, '@actual' => $replacements[$token], '@expected' => $expected]));
+      }
+    }
+
+    return $replacements;
+  }
+
+  function mapTokenNames($type, array $tokens = []) {
+    $return = [];
+    foreach ($tokens as $token) {
+      $return[$token] = "[$type:$token]";
+    }
+    return $return;
+  }
+
+  function assertNoTokens($type, array $data, array $tokens, array $options = []) {
+    $input = $this->mapTokenNames($type, $tokens);
+    $bubbleable_metadata = new BubbleableMetadata();
+    $replacements = \Drupal::token()->generate($type, $input, $data, $options, $bubbleable_metadata);
+    foreach ($tokens as $name) {
+      $token = $input[$name];
+      $this->assertTrue(!isset($replacements[$token]), t("Token value for @token was not generated.", ['@type' => $type, '@token' => $token]));
+    }
+  }
+
+  function saveAlias($source, $alias, $language = Language::LANGCODE_NOT_SPECIFIED) {
+    return $this->createPathAlias($source, $alias, $language);
+  }
+
+  function saveEntityAlias($entity_type, EntityInterface $entity, $alias, $language = Language::LANGCODE_NOT_SPECIFIED) {
+    $uri = $entity->toUrl()->toArray();
+    return $this->saveAlias($uri['path'], $alias, $language);
+  }
+
+  /**
+   * Make a page request and test for token generation.
+   */
+  function assertPageTokens($url, array $tokens, array $data = [], array $options = []) {
+    if (empty($tokens)) {
+      return TRUE;
+    }
+
+    $token_page_tokens = [
+      'tokens' => $tokens,
+      'data' => $data,
+      'options' => $options,
+    ];
+    \Drupal::state()->set('token_page_tokens', $token_page_tokens);
+
+    $options += ['url_options' => []];
+    $this->drupalGet($url, $options['url_options']);
+    $this->refreshVariables();
+    $result = \Drupal::state()->get('token_page_tokens', []);
+
+    if (!isset($result['values']) || !is_array($result['values'])) {
+      return $this->fail('Failed to generate tokens.');
+    }
+
+    foreach ($tokens as $token => $expected) {
+      if (!isset($expected)) {
+        $this->assertTrue(!isset($result['values'][$token]) || $result['values'][$token] === $token, t("Token value for @token was not generated.", ['@token' => $token]));
+      }
+      elseif (!isset($result['values'][$token])) {
+        $this->fail(t('Failed to generate token @token.', ['@token' => $token]));
+      }
+      else {
+        $this->assertSame($result['values'][$token], (string) $expected, t("Token value for @token was '@actual', expected value '@expected'.", ['@token' => $token, '@actual' => $result['values'][$token], '@expected' => $expected]));
+      }
+    }
+  }
+
+}

modules/token/tests/src/Functional/TokenURLTest.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Core\Url;
+
+/**
+ * Tests url tokens.
+ *
+ * @group token
+ */
+class TokenURLTest extends TokenTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->saveAlias('/node/1', '/first-node');
+  }
+
+  function testURLTokens() {
+    $url = new Url('entity.node.canonical', ['node' => 1]);
+    $tokens = [
+      'absolute' => $url->setAbsolute()->toString(),
+      'relative' => $url->setAbsolute(FALSE)->toString(),
+      'path' => '/first-node',
+      'brief' => preg_replace(['!^https?://!', '!/$!'], '', $url->setAbsolute()->toString()),
+      'args:value:0' => 'first-node',
+      'args:value:1' => NULL,
+      'args:value:N' => NULL,
+      'unaliased' => $url->setAbsolute()->setOption('alias', TRUE)->toString(),
+      'unaliased:relative' => $url->setAbsolute(FALSE)->setOption('alias', TRUE)->toString(),
+      'unaliased:path' => '/node/1',
+      'unaliased:brief' => preg_replace(['!^https?://!', '!/$!'], '', $url->setAbsolute()->setOption('alias', TRUE)->toString()),
+      'unaliased:args:value:0' => 'node',
+      'unaliased:args:value:1' => '1',
+      'unaliased:args:value:2' => NULL,
+      // Deprecated tokens.
+      'alias' => '/first-node',
+    ];
+    $this->assertTokens('url', ['url' => new Url('entity.node.canonical', ['node' => 1])], $tokens);
+  }
+}

modules/token/tests/src/Functional/TokenUserTest.php

@@ -0,0 +1,130 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Session\AccountInterface;
+use Drupal\Core\Session\AnonymousUserSession;
+use Drupal\field\Entity\FieldStorageConfig;
+use Drupal\Tests\TestFileCreationTrait;
+
+/**
+ * Tests user tokens.
+ *
+ * @group token
+ */
+class TokenUserTest extends TokenTestBase {
+
+  use TestFileCreationTrait;
+
+  /**
+   * The user account.
+   *
+   * @var \Drupal\Core\Session\AccountInterface
+   */
+  protected $account = NULL;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['token_user_picture'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+
+    $this->account = $this->drupalCreateUser([
+      'administer users',
+      'administer account settings',
+      'access content',
+    ]);
+    $this->drupalLogin($this->account);
+  }
+
+  /**
+   * Tests the user related tokens.
+   */
+  public function testUserTokens() {
+    // Enable user pictures.
+    \Drupal::state()->set('user_pictures', 1);
+    \Drupal::state()->set('user_picture_file_size', '');
+
+    // Set up the pictures directory.
+    $picture_path = 'public://' . \Drupal::state()->get('user_picture_path', 'pictures');
+    if (!\Drupal::service('file_system')->prepareDirectory($picture_path, FileSystemInterface::CREATE_DIRECTORY)) {
+      $this->fail('Could not create directory ' . $picture_path . '.');
+    }
+
+    // Add a user picture to the account.
+    $image = current($this->getTestFiles('image'));
+    $edit = ['files[user_picture_0]' => \Drupal::service('file_system')->realpath($image->uri)];
+    $this->drupalGet('user/' . $this->account->id() . '/edit');
+    $this->submitForm($edit, 'Save');
+
+    $storage = \Drupal::entityTypeManager()->getStorage('user');
+
+    // Load actual user data from database.
+    $storage->resetCache();
+    $this->account = $storage->load($this->account->id());
+    $this->assertNotEmpty($this->account->user_picture->target_id, 'User picture uploaded.');
+
+    $picture = [
+      '#theme' => 'user_picture',
+      '#account' => $this->account,
+    ];
+    /** @var \Drupal\Core\Render\RendererInterface $renderer */
+    $renderer = \Drupal::service('renderer');
+    $user_tokens = [
+      'picture' => $renderer->renderPlain($picture),
+      'picture:fid' => $this->account->user_picture->target_id,
+      'picture:size-raw' => 125,
+      'ip-address' => NULL,
+      'roles' => implode(', ', $this->account->getRoles()),
+    ];
+    $this->assertTokens('user', ['user' => $this->account], $user_tokens);
+
+    // Remove the simpletest-created user role.
+    $roles = $this->account->getRoles();
+    $this->account->removeRole(end($roles));
+    $this->account->save();
+
+    // Remove the user picture field and reload the user.
+    FieldStorageConfig::loadByName('user', 'user_picture')->delete();
+    $storage->resetCache();
+    $this->account = $storage->load($this->account->id());
+
+    $user_tokens = [
+      'picture' => NULL,
+      'picture:fid' => NULL,
+      'ip-address' => NULL,
+      'roles' => 'authenticated',
+      'roles:keys' => AccountInterface::AUTHENTICATED_ROLE,
+    ];
+    $this->assertTokens('user', ['user' => $this->account], $user_tokens);
+
+    // The ip address token should work for the current user token type.
+    $tokens = [
+      'ip-address' => \Drupal::request()->getClientIp(),
+    ];
+    $this->assertTokens('current-user', [], $tokens);
+
+    $anonymous = new AnonymousUserSession();
+    $tokens = [
+      'roles' => 'anonymous',
+      'roles:keys' => AccountInterface::ANONYMOUS_ROLE,
+    ];
+    $this->assertTokens('user', ['user' => $anonymous], $tokens);
+  }
+
+  /**
+   * Test user account settings.
+   */
+  public function testUserAccountSettings() {
+    $this->drupalGet('admin/config/people/accounts');
+    $this->assertSession()->pageTextContains('The list of available tokens that can be used in e-mails is provided below.');
+    $this->assertSession()->linkExists('Browse available tokens.');
+    $this->assertSession()->linkByHrefExists('token/tree');
+  }
+}

modules/token/tests/src/Functional/Tree/HelpPageTest.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace Drupal\Tests\token\Functional\Tree;
+
+use Drupal\Tests\token\Functional\TokenTestBase;
+
+/**
+ * Tests token tree on help page.
+ *
+ * @group token
+ */
+class HelpPageTest extends TokenTestBase {
+
+  use TokenTreeTestTrait;
+
+  /**
+   * @var \Drupal\Core\Session\AccountInterface
+   */
+  protected $account;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['help'];
+
+  public function setUp(): void {
+    parent::setUp();
+
+    // On Drupal 10.2 there is new permission to access help pages.
+    $permissions = ['access help pages'];
+    if (version_compare(\Drupal::VERSION, '10.2', '<')) {
+      $permissions = ['access administration pages'];
+    }
+    $this->account = $this->drupalCreateUser($permissions);
+    $this->drupalLogin($this->account);
+  }
+
+  /**
+   * Tests the token browser on the token help page.
+   */
+  public function testHelpPageTree() {
+    $this->drupalGet('admin/help/token');
+    $this->assertSession()->pageTextContains('The list of the currently available tokens on this site are shown below.');
+
+    $this->assertTokenGroup('Current date');
+    $this->assertTokenGroup('Site information');
+
+    $this->assertTokenInTree('[current-date:html_date]', 'current-date');
+    $this->assertTokenInTree('[current-date:html_week]', 'current-date');
+    $this->assertTokenInTree('[date:html_date]', 'date');
+    $this->assertTokenInTree('[date:html_week]', 'date');
+
+    $this->assertTokenInTree('[current-user:account-name]', 'current-user');
+    $this->assertTokenInTree('[user:account-name]', 'user');
+
+    $this->assertTokenInTree('[current-page:url:unaliased]', 'current-page--url');
+    $this->assertTokenInTree('[current-page:url:unaliased:args]', 'current-page--url--unaliased');
+    $this->assertTokenInTree('[user:original:account-name]', 'user--original');
+
+    // Assert some of the restricted tokens to ensure they are shown.
+    $this->assertTokenInTree('[user:one-time-login-url]', 'user');
+    $this->assertTokenInTree('[user:original:cancel-url]', 'user--original');
+
+    // The Array token is marked as nested, so it should not show up as a top
+    // level token, only nested under another token. For instance, user:roles
+    // is of type Array and tokens of type Array have 'nested' setting true.
+    $this->assertTokenNotGroup('Array');
+    $this->assertTokenNotGroup('user:roles');
+    $this->assertTokenInTree('[user:roles]', 'user');
+  }
+
+}

modules/token/tests/src/Functional/Tree/TokenTreeTestTrait.php

@@ -0,0 +1,121 @@
+<?php
+
+namespace Drupal\Tests\token\Functional\Tree;
+
+use Behat\Mink\Element\NodeElement;
+
+/**
+ * Helper trait to assert tokens in token tree browser.
+ */
+trait TokenTreeTestTrait {
+
+  /**
+   * Get an array of token groups from the last retrieved page.
+   *
+   * @return array
+   *   Array of token group names.
+   */
+  protected function getTokenGroups() {
+    $groups = $this->xpath('//tr[contains(@class, "token-group")]/td[1]');
+    return array_map(function (NodeElement $item) {
+      return (string) $item->getText();
+    }, $groups);
+  }
+
+  /**
+   * Check to see if the specified token group is present in the token browser.
+   *
+   * @param string $token_group
+   *   The name of the token group.
+   * @param string $message
+   *   (optional) A message to display with the assertion.
+   */
+  protected function assertTokenGroup($token_group, $message = '') {
+    $groups = $this->getTokenGroups();
+
+    if (!$message) {
+      $message = "Token group $token_group found.";
+    }
+
+    $this->assertContains($token_group, $groups, $message);
+  }
+
+  /**
+   * Check to see if the specified token group is not present in the token
+   * browser.
+   *
+   * @param string $token_group
+   *   The name of the token group.
+   * @param string $message
+   *   (optional) A message to display with the assertion.
+   */
+  protected function assertTokenNotGroup($token_group, $message = '') {
+    $groups = $this->getTokenGroups();
+
+    if (!$message) {
+      $message = "Token group $token_group not found.";
+    }
+
+    $this->assertNotContains($token_group, $groups, $message);
+  }
+
+  /**
+   * Check to see if the specified token is present in the token browser.
+   *
+   * @param $token
+   *   The token name with the surrounding square brackets [].
+   * @param string $parent
+   *   (optional) The parent CSS identifier of this token.
+   * @param string $message
+   *   (optional) A message to display with the assertion.
+   */
+  protected function assertTokenInTree($token, $parent = '', $message = '') {
+    $xpath = $this->getXpathForTokenInTree($token, $parent);
+
+    if (!$message) {
+      $message = "Token $token found.";
+    }
+
+    $this->assertCount(1, $this->xpath($xpath), $message);
+  }
+
+  /**
+   * Check to see if the specified token is present in the token browser.
+   *
+   * @param $token
+   *   The token name with the surrounding square brackets [].
+   * @param string $parent
+   *   (optional) The parent CSS identifier of this token.
+   * @param string $message
+   *   (optional) A message to display with the assertion.
+   */
+  protected function assertTokenNotInTree($token, $parent = '', $message = '') {
+    $xpath = $this->getXpathForTokenInTree($token, $parent);
+
+    if (!$message) {
+      $message = "Token $token not found.";
+    }
+
+    $this->assertCount(0, $this->xpath($xpath), $message);
+  }
+
+  /**
+   * Get xpath to check for token in tree.
+   *
+   * @param $token
+   *   The token name with the surrounding square brackets [].
+   * @param string $parent
+   *   (optional) The parent CSS identifier of this token.
+   *
+   * @return string
+   *   The xpath to check for the token and parent.
+   */
+  protected function getXpathForTokenInTree($token, $parent = '') {
+    $xpath = "//tr";
+    if ($parent) {
+      $xpath .= '[@data-tt-parent-id="token-' . $parent . '"]';
+    }
+    $xpath .= '/td[contains(@class, "token-key") and text() = "' . $token . '"]';
+    return $xpath;
+  }
+}

modules/token/tests/src/Functional/Tree/TreeTest.php

@@ -0,0 +1,146 @@
+<?php
+
+namespace Drupal\Tests\token\Functional\Tree;
+
+use Drupal\Component\Serialization\Json;
+use Drupal\Tests\token\Functional\TokenTestBase;
+
+/**
+ * Tests token tree page.
+ *
+ * @group token
+ */
+class TreeTest extends TokenTestBase {
+
+  use TokenTreeTestTrait;
+
+  /**
+   * @var \Drupal\Core\Session\AccountInterface
+   */
+  protected $account;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node'];
+
+  public function setUp(): void {
+    parent::setUp();
+
+    $this->account = $this->drupalCreateUser(['administer account settings']);
+    $this->drupalLogin($this->account);
+  }
+
+  /**
+   * Test various tokens that are possible on the site.
+   */
+  public function testAllTokens() {
+    $this->drupalGet($this->getTokenTreeUrl(['token_types' => 'all']));
+
+    $this->assertTokenGroup('Current date');
+    $this->assertTokenGroup('Site information');
+
+    $this->assertTokenInTree('[current-date:html_date]', 'current-date');
+    $this->assertTokenInTree('[current-date:html_week]', 'current-date');
+    $this->assertTokenInTree('[date:html_date]', 'date');
+    $this->assertTokenInTree('[date:html_week]', 'date');
+
+    $this->assertTokenInTree('[current-user:account-name]', 'current-user');
+    $this->assertTokenInTree('[user:account-name]', 'user');
+
+    $this->assertTokenInTree('[current-page:url:unaliased]', 'current-page--url');
+    $this->assertTokenInTree('[current-page:url:unaliased:args]', 'current-page--url--unaliased');
+    $this->assertTokenInTree('[user:original:account-name]', 'user--original');
+  }
+
+  /**
+   * Test various tokens that are possible on the site.
+   */
+  public function testGlobalTokens() {
+    $this->drupalGet($this->getTokenTreeUrl());
+
+    $this->assertTokenGroup('Current date');
+    $this->assertTokenGroup('Site information');
+
+    // Assert that non-global tokens are not listed.
+    $this->assertTokenNotInTree('[user:account-name]', 'user');
+    $this->assertTokenNotInTree('[user:original:account-name]', 'user--original');
+
+    // Assert some of the global tokens, just to be sure.
+    $this->assertTokenInTree('[current-date:html_date]', 'current-date');
+    $this->assertTokenInTree('[current-date:html_week]', 'current-date');
+
+    $this->assertTokenInTree('[current-user:account-name]', 'current-user');
+
+    $this->assertTokenInTree('[current-page:url:unaliased]', 'current-page--url');
+    $this->assertTokenInTree('[current-page:url:unaliased:args]', 'current-page--url--unaliased');
+  }
+
+  /**
+   * Tests if the token browser displays the user tokens.
+   */
+  public function testUserTokens() {
+    $this->drupalGet($this->getTokenTreeUrl(['token_types' => ['user']]));
+
+    $this->assertTokenGroup('Users');
+
+    $this->assertTokenInTree('[user:account-name]', 'user');
+    $this->assertTokenInTree('[user:original:account-name]', 'user--original');
+
+    // Assert some of the restricted tokens to ensure they are not shown.
+    $this->assertTokenNotInTree('[user:one-time-login-url]', 'user');
+    $this->assertTokenNotInTree('[user:original:cancel-url]', 'user--original');
+
+    // Request with show_restricted set to TRUE to show restricted tokens and
+    // check for them.
+    $this->drupalGet($this->getTokenTreeUrl(['token_types' => ['user'], 'show_restricted' => TRUE]));
+    $this->assertEquals('MISS', $this->getSession()->getResponseHeader('x-drupal-dynamic-cache'), 'Cache was not hit');
+    $this->assertTokenInTree('[user:one-time-login-url]', 'user');
+    $this->assertTokenInTree('[user:original:cancel-url]', 'user--original');
+  }
+
+  /**
+   * Tests if the token browser displays the node tokens.
+   */
+  public function testNodeTokens() {
+    $this->drupalGet($this->getTokenTreeUrl(['token_types' => ['node']]));
+
+    $this->assertTokenGroup('Nodes');
+
+    $this->assertTokenInTree('[node:body]', 'node');
+    $this->assertTokenInTree('[node:author:original:account-name]', 'node--author--original');
+  }
+
+  /**
+   * Get the URL for the token tree based on the specified options.
+   *
+   * The token tree route's URL requires CSRF and cannot be generated in the
+   * test code. The CSRF token generated using the test runner's session is
+   * different from the session inside the test environment. This is why the
+   * link has to be generated inside the environment.
+   *
+   * This function calls a page in token_module_test module which generates the
+   * link and the token. This then replaces the options query parameter with the
+   * specified options.
+   *
+   * The page also uses a title callback to set title to a render array, which
+   * allows us to test if [current-page:title] works properly.
+   *
+   * @param array $options
+   *   The options for the token tree browser.
+   *
+   * @return string
+   *   The complete URL of the token tree browser with the CSRF token.
+   */
+  protected function getTokenTreeUrl($options = []) {
+    $this->drupalGet('token_module_test/browse');
+    $this->assertSession()->titleEquals('Available Tokens | Drupal');
+    $links = $this->xpath('//a[contains(@href, :href)]/@href', [':href' => 'token/tree']);
+    $link = $this->getAbsoluteUrl(current($links)->getText());
+    if (!empty($options)) {
+      $options = Json::encode($options);
+      $link = str_replace('options=%5B%5D', 'options=' . urlencode($options), $link);
+    }
+    return $link;
+  }
+}

modules/token/tests/src/Functional/UrlTest.php

@@ -0,0 +1,140 @@
+<?php
+
+namespace Drupal\Tests\token\Functional;
+
+use Drupal\block\Entity\Block;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+use Drupal\Tests\BrowserTestBase;
+use Drupal\Tests\system\Functional\Cache\AssertPageCacheContextsAndTagsTrait;
+
+/**
+ * Tests URL tokens.
+ *
+ * @group token
+ */
+class UrlTest extends BrowserTestBase {
+
+  use AssertPageCacheContextsAndTagsTrait;
+
+  /**
+   * The first testing node.
+   *
+   * @var \Drupal\node\NodeInterface
+   */
+  protected $node1;
+
+  /**
+   * The second testing node.
+   *
+   * @var \Drupal\node\NodeInterface
+   */
+  protected $node2;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node', 'token', 'block'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected $defaultTheme = 'stark';
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $node_type = NodeType::create(['type' => 'article', 'name' => 'Article']);
+    $node_type->save();
+
+    $this->node1 = Node::create([
+      'type' => 'article',
+      'title' => 'Test Node 1',
+    ]);
+    $this->node1->save();
+
+    $this->node2 = Node::create([
+      'type' => 'article',
+      'title' => 'Test Node 2',
+    ]);
+    $this->node2->save();
+
+  }
+
+  /**
+   * Creates a block with token for title and tests cache contexts.
+   *
+   * @throws \Behat\Mink\Exception\ElementHtmlException
+   * @throws \Drupal\Core\Entity\EntityStorageException
+   */
+  public function testBlockUrlTokenReplacement() {
+
+    $node1_url = $this->node1->toUrl();
+    $node2_url = $this->node2->toUrl();
+
+    // Using a @dataprovider causes repeated database installations and takes a
+    // very long time.
+    $tests = [];
+    $tests[] = [
+      'token' => 'prefix_[current-page:url:path]_suffix',
+      'expected1' => 'prefix_/' . $node1_url->getInternalPath() . '_suffix',
+      'expected2' => 'prefix_/' . $node2_url->getInternalPath() . '_suffix',
+      // A path can only be generated from a routed path.
+      'expected3' => 'prefix_/_suffix',
+    ];
+    $tests[] = [
+      'token' => 'prefix_[current-page:url]_suffix',
+      'expected1' => 'prefix_' . $node1_url->setAbsolute()->toString() . '_suffix',
+      'expected2' => 'prefix_' . $node2_url->setAbsolute()->toString() . '_suffix',
+      'expected3' => 'prefix_' . $this->getAbsoluteUrl('does-not-exist') . '_suffix',
+    ];
+
+    // Place a standard block and use a token in the label.
+    $edit = [
+      'id' => 'token_url_test_block',
+      'label' => 'label',
+      'label_display' => TRUE,
+    ];
+    $this->placeBlock('system_powered_by_block', $edit);
+    $block = Block::load('token_url_test_block');
+
+    $assert_session = $this->assertSession();
+
+    foreach ($tests as $test) {
+      // Set the block label.
+      $block->getPlugin()->setConfigurationValue('label', $test['token']);
+      $block->save();
+
+      // Go to the first node page and test that the token is correct.
+      $this->drupalGet($node1_url);
+      $assert_session->elementContains('css', '#block-token-url-test-block', $test['expected1']);
+
+      // Go to the second node page and check that the block title has changed.
+      $this->drupalGet($node2_url);
+      $assert_session->elementContains('css', '#block-token-url-test-block', $test['expected2']);
+
+      // Test the current page url on a 404 page.
+      $this->drupalGet('does-not-exist');
+      $assert_session->statusCodeEquals(404);
+      $assert_session->elementContains('css', '#block-token-url-test-block', $test['expected3']);
+    }
+
+
+    // Can't do this test in the for loop above, it's too different.
+    $block->getPlugin()->setConfigurationValue('label', 'prefix_[current-page:query:unicorns]_suffix');
+    $block->save();
+
+    // Test the parameter token.
+    $this->drupalGet($node1_url->setOption('query', ['unicorns' => 'fluffy']));
+    $this->assertCacheContext('url.query_args');
+    $assert_session->elementContains('css', '#block-token-url-test-block', 'prefix_fluffy_suffix');
+
+    // Change the parameter on the same page.
+    $this->drupalGet($node1_url->setOption('query', ['unicorns' => 'dead']));
+    $assert_session->elementContains('css', '#block-token-url-test-block', 'prefix_dead_suffix');
+  }
+
+}

modules/token/tests/src/Kernel/ArrayTest.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+/**
+ * Tests array tokens.
+ *
+ * @group token
+ */
+class ArrayTest extends TokenKernelTestBase {
+
+  function testArrayTokens() {
+    // Test a simple array.
+    $array = [0 => 'a', 1 => 'b', 2 => 'c', 4 => 'd'];
+    $tokens = [
+      'first' => 'a',
+      'last' => 'd',
+      'value:0' => 'a',
+      'value:2' => 'c',
+      'count' => 4,
+      'keys' => '0, 1, 2, 4',
+      'keys:value:3' => '4',
+      'keys:join' => '0124',
+      'reversed' => 'd, c, b, a',
+      'reversed:keys' => '4, 2, 1, 0',
+      'join:/' => 'a/b/c/d',
+      'join' => 'abcd',
+      'join:, ' => 'a, b, c, d',
+      'join: ' => 'a b c d',
+    ];
+    $this->assertTokens('array', ['array' => $array], $tokens);
+
+    // Test a mixed simple and render array.
+    // 2 => c, 0 => a, 4 => d, 1 => b
+    $array = [
+      '#property' => 'value',
+      0 => 'a',
+      1 => ['#markup' => 'b', '#weight' => 0.01],
+      2 => ['#markup' => 'c', '#weight' => -10],
+      4 => ['#markup' => 'd', '#weight' => 0],
+    ];
+    $tokens = [
+      'first' => 'c',
+      'last' => 'b',
+      'value:0' => 'a',
+      'value:2' => 'c',
+      'count' => 4,
+      'keys' => '2, 0, 4, 1',
+      'keys:value:3' => '1',
+      'keys:join' => '2041',
+      'reversed' => 'b, d, a, c',
+      'reversed:keys' => '1, 4, 0, 2',
+      'join:/' => 'c/a/d/b',
+      'join' => 'cadb',
+      'join:, ' => 'c, a, d, b',
+      'join: ' => 'c a d b',
+    ];
+    $this->assertTokens('array', ['array' => $array], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/BookTest.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\Url;
+use Drupal\node\Entity\Node;
+
+/**
+ * Test the book tokens.
+ *
+ * @group token
+ */
+class BookTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'user',
+    'field',
+    'filter',
+    'text',
+    'node',
+    'book',
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('user');
+    $this->installEntitySchema('node');
+    $this->installSchema('book', ['book']);
+    $this->installSchema('node', ['node_access']);
+    $this->installConfig(['node', 'book', 'field']);
+  }
+
+  /**
+   *
+   */
+  public function testBookTokens() {
+    $book = Node::create([
+      'type' => 'book',
+      'title' => 'Book Main Page',
+      'book' => ['bid' => 'new'],
+    ]);
+    $book->save();
+
+    $page1 = Node::create([
+      'type' => 'book',
+      'title' => '1st Page',
+      'book' => ['bid' => $book->id(), 'pid' => $book->id()],
+    ]);
+    $page1->save();
+
+    $page2 = Node::create([
+      'type' => 'book',
+      'title' => '2nd Page',
+      'book' => ['bid' => $book->id(), 'pid' => $page1->id()],
+    ]);
+    $page2->save();
+
+    $book_title = $book->getTitle();
+
+    $tokens = [
+      'nid' => $book->id(),
+      'title' => $book_title,
+      'book:title' => $book_title,
+      'book:root' => $book_title,
+      'book:root:nid' => $book->id(),
+      'book:root:title' => $book_title,
+      'book:root:url' => Url::fromRoute('entity.node.canonical', ['node' => $book->id()], ['absolute' => TRUE])->toString(),
+      'book:root:content-type' => 'Book page',
+      'book:parent' => NULL,
+      'book:parents' => NULL,
+    ];
+    $this->assertTokens('node', ['node' => $book], $tokens);
+
+    $tokens = [
+      'nid' => $page1->id(),
+      'title' => $page1->getTitle(),
+      'book:title' => $book_title,
+      'book:root' => $book_title,
+      'book:root:nid' => $book->id(),
+      'book:root:title' => $book_title,
+      'book:root:url' => Url::fromRoute('entity.node.canonical', ['node' => $book->id()], ['absolute' => TRUE])->toString(),
+      'book:root:content-type' => 'Book page',
+      'book:parent:nid' => $book->id(),
+      'book:parent:title' => $book_title,
+      'book:parent:url' => Url::fromRoute('entity.node.canonical', ['node' => $book->id()], ['absolute' => TRUE])->toString(),
+      'book:parents:count' => 1,
+      'book:parents:join:/' => $book_title,
+    ];
+    $this->assertTokens('node', ['node' => $page1], $tokens);
+
+    $tokens = [
+      'nid' => $page2->id(),
+      'title' => $page2->getTitle(),
+      'book:title' => $book_title,
+      'book:root' => $book_title,
+      'book:root:nid' => $book->id(),
+      'book:root:title' => $book_title,
+      'book:root:url' => Url::fromRoute('entity.node.canonical', ['node' => $book->id()], ['absolute' => TRUE])->toString(),
+      'book:root:content-type' => 'Book page',
+      'book:parent:nid' => $page1->id(),
+      'book:parent:title' => $page1->getTitle(),
+      'book:parent:url' => Url::fromRoute('entity.node.canonical', ['node' => $page1->id()], ['absolute' => TRUE])->toString(),
+      'book:parents:count' => 2,
+      'book:parents:join:/' => $book_title . '/' . $page1->getTitle(),
+    ];
+    $this->assertTokens('node', ['node' => $page2], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/CommentTest.php

@@ -0,0 +1,107 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\comment\Entity\Comment;
+use Drupal\comment\Tests\CommentTestTrait;
+use Drupal\Core\Url;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+
+/**
+ * Tests comment tokens.
+ *
+ * @group token
+ */
+class CommentTest extends TokenKernelTestBase {
+
+  use CommentTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'node',
+    'comment',
+    'field',
+    'text',
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('node');
+    $this->installEntitySchema('user');
+    $this->installEntitySchema('comment');
+    $this->installSchema('comment', ['comment_entity_statistics']);
+
+    $node_type = NodeType::create(['type' => 'page', 'name' => 'Page']);
+    $node_type->save();
+
+    $this->installConfig(['comment']);
+
+    $this->addDefaultCommentField('node', 'page');
+  }
+
+  /**
+   *
+   */
+  public function testCommentTokens() {
+    $node = Node::create([
+      'type' => 'page',
+      'title' => $this->randomMachineName(),
+    ]);
+    $node->save();
+
+    $parent_comment = Comment::create([
+      'entity_id' => $node->id(),
+      'entity_type' => 'node',
+      'field_name' => 'comment',
+      'name' => 'anonymous user',
+      'mail' => 'anonymous@example.com',
+      'subject' => $this->randomMachineName(),
+      'body' => $this->randomMachineName(),
+    ]);
+    $parent_comment->save();
+
+    // Fix http://example.com/index.php/comment/1 fails 'url:path' test.
+    $parent_comment_path = $parent_comment->toUrl()->toString();
+
+    $tokens = [
+      'url' => $parent_comment->toUrl('canonical', ['fragment' => "comment-{$parent_comment->id()}"])->setAbsolute()->toString(),
+      'url:absolute' => $parent_comment->toUrl('canonical', ['fragment' => "comment-{$parent_comment->id()}"])->setAbsolute()->toString(),
+      'url:relative' => $parent_comment->toUrl('canonical', ['fragment' => "comment-{$parent_comment->id()}"])->toString(),
+      'url:path' => $parent_comment_path,
+      'parent:url:absolute' => NULL,
+    ];
+    $this->assertTokens('comment', ['comment' => $parent_comment], $tokens);
+
+    $comment = Comment::create([
+      'entity_id' => $node->id(),
+      'pid' => $parent_comment->id(),
+      'entity_type' => 'node',
+      'field_name' => 'comment',
+      'name' => 'anonymous user',
+      'mail' => 'anonymous@example.com',
+      'subject' => $this->randomMachineName(),
+      'body' => $this->randomMachineName(),
+    ]);
+    $comment->save();
+
+    // Fix http://example.com/index.php/comment/1 fails 'url:path' test.
+    $comment_path = Url::fromRoute('entity.comment.canonical', ['comment' => $comment->id()])->toString();
+
+    $tokens = [
+      'url' => $comment->toUrl('canonical', ['fragment' => "comment-{$comment->id()}"])->setAbsolute()->toString(),
+      'url:absolute' => $comment->toUrl('canonical', ['fragment' => "comment-{$comment->id()}"])->setAbsolute()->toString(),
+      'url:relative' => $comment->toUrl('canonical', ['fragment' => "comment-{$comment->id()}"])->toString(),
+      'url:path' => $comment_path,
+      'parent:url:absolute' => $parent_comment->toUrl('canonical', ['fragment' => "comment-{$parent_comment->id()}"])->setAbsolute()->toString(),
+    ];
+    $this->assertTokens('comment', ['comment' => $comment], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/DateTest.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+/**
+ * Tests date tokens.
+ *
+ * @group token
+ */
+class DateTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+    $this->installConfig(['system', 'token_module_test']);
+  }
+
+  function testDateTokens() {
+    $tokens = [
+      'token_module_test' => '1984',
+      'invalid_format' => NULL,
+    ];
+
+    $this->assertTokens('date', ['date' => 453859200], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/EntityTest.php

@@ -0,0 +1,109 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\node\Entity\Node;
+use Drupal\taxonomy\Entity\Term;
+use Drupal\taxonomy\Entity\Vocabulary;
+use Drupal\taxonomy\VocabularyInterface;
+
+/**
+ * Tests entity tokens.
+ *
+ * @group token
+ */
+class EntityTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node', 'taxonomy', 'text'];
+
+  /**
+   * Vocabulary for testing chained token support.
+   *
+   * @var \Drupal\taxonomy\VocabularyInterface
+   */
+  protected $vocabulary;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+
+    // Create the default tags vocabulary.
+    $this->vocabulary = Vocabulary::create([
+      'name' => 'Tags',
+      'vid' => 'tags',
+    ]);
+    $this->vocabulary->save();
+
+    $this->installEntitySchema('taxonomy_term');
+    $this->installEntitySchema('user');
+    $this->installEntitySchema('node');
+  }
+
+  function testEntityMapping() {
+    /** @var \Drupal\token\TokenEntityMapperInterface $mapper */
+    $mapper = \Drupal::service('token.entity_mapper');
+    $this->assertSame('node', $mapper->getEntityTypeForTokenType('node'));
+    $this->assertSame('taxonomy_term', $mapper->getEntityTypeForTokenType('term'));
+    $this->assertSame('taxonomy_vocabulary', $mapper->getEntityTypeForTokenType('vocabulary'));
+    $this->assertSame(FALSE, $mapper->getEntityTypeForTokenType('invalid'));
+    $this->assertSame('invalid', $mapper->getEntityTypeForTokenType('invalid', TRUE));
+    $this->assertSame('node', $mapper->getTokenTypeForEntityType('node'));
+    $this->assertSame('term', $mapper->getTokenTypeForEntityType('taxonomy_term'));
+    $this->assertSame('vocabulary', $mapper->getTokenTypeForEntityType('taxonomy_vocabulary'));
+    $this->assertSame(FALSE, $mapper->getTokenTypeForEntityType('invalid'));
+    $this->assertSame('invalid', $mapper->getTokenTypeForEntityType('invalid', TRUE));
+
+    // Test that when we send the mis-matched entity type into
+    // Drupal\Core\Utility\Token::replace() that we still get the tokens
+    // replaced.
+    $vocabulary = Vocabulary::load('tags');
+    $term = $this->addTerm($vocabulary);
+    $this->assertSame($vocabulary->label(), \Drupal::token()->replace('[vocabulary:name]', ['taxonomy_vocabulary' => $vocabulary]));
+    $this->assertSame($term->label() . $vocabulary->label(), \Drupal::token()->replace('[term:name][term:vocabulary:name]', ['taxonomy_term' => $term]));
+  }
+
+  function addTerm(VocabularyInterface $vocabulary, array $term = []) {
+    $term += [
+      'name' => mb_strtolower($this->randomMachineName(5)),
+      'vid' => $vocabulary->id(),
+    ];
+    $term = Term::create($term);
+    $term->save();
+    return $term;
+  }
+
+  /**
+   * Test the [entity:original:*] tokens.
+   */
+  function testEntityOriginal() {
+    $node = Node::create(['type' => 'page', 'title' => 'Original title']);
+    $node->save();
+
+    $tokens = [
+      'nid' => $node->id(),
+      'title' => 'Original title',
+      'original' => NULL,
+      'original:nid' => NULL,
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens);
+
+    // Emulate the original entity property that would be available from
+    // node_save() and change the title for the node.
+    $node->original = \Drupal::entityTypeManager()->getStorage('node')->loadUnchanged($node->id());
+    $node->title = 'New title';
+
+    $tokens = [
+      'nid' => $node->id(),
+      'title' => 'New title',
+      'original' => 'Original title',
+      'original:nid' => $node->id(),
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/FieldTest.php

@@ -0,0 +1,786 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Component\Utility\Html;
+use Drupal\Component\Utility\Unicode;
+use Drupal\contact\Entity\ContactForm;
+use Drupal\contact\Entity\Message;
+use Drupal\Core\Entity\Entity\EntityViewMode;
+use Drupal\Core\Field\FieldStorageDefinitionInterface;
+use Drupal\Core\Render\Markup;
+use Drupal\datetime\Plugin\Field\FieldType\DateTimeItem;
+use Drupal\datetime_range\Plugin\Field\FieldType\DateRangeItem;
+use Drupal\field\Entity\FieldConfig;
+use Drupal\field\Entity\FieldStorageConfig;
+use Drupal\filter\Entity\FilterFormat;
+use Drupal\language\Entity\ConfigurableLanguage;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+use Drupal\Tests\taxonomy\Traits\TaxonomyTestTrait;
+
+/**
+ * Tests field tokens.
+ *
+ * @group token
+ */
+class FieldTest extends TokenKernelTestBase {
+
+  use TaxonomyTestTrait;
+
+  /**
+   * @var \Drupal\filter\FilterFormatInterface
+   */
+  protected $testFormat;
+
+
+  /**
+   * Vocabulary for testing chained token support.
+   *
+   * @var \Drupal\taxonomy\VocabularyInterface
+   */
+  protected $vocabulary;
+
+  /**
+   * The field used in this test class.
+   *
+   * @var \Drupal\field\Entity\FieldConfig
+   */
+  protected $field;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'node',
+    'text',
+    'field',
+    'filter',
+    'contact',
+    'options',
+    'taxonomy',
+    'language',
+    'datetime',
+    'datetime_range',
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('user');
+    $this->installEntitySchema('node');
+    $this->installEntitySchema('taxonomy_term');
+
+    // Create the article content type with a text field.
+    $node_type = NodeType::create([
+      'type' => 'article',
+    ]);
+    $node_type->save();
+
+    $field_storage = FieldStorageConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'node',
+      'type' => 'text',
+    ]);
+    $field_storage->save();
+
+    $field = FieldConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+      'label' => 'Test field',
+    ]);
+    $field->save();
+
+    // Create a reference field with the same name on user.
+    $field_storage = FieldStorageConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'user',
+      'type' => 'entity_reference',
+    ]);
+    $field_storage->save();
+
+    $field = FieldConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'user',
+      'bundle' => 'user',
+      'label' => 'Test field',
+    ]);
+    $field->save();
+
+    $this->testFormat = FilterFormat::create([
+      'format' => 'test',
+      'name' => 'Test format',
+      'weight' => 1,
+      'filters' => [
+        'filter_html_escape' => ['status' => TRUE],
+      ],
+    ]);
+    $this->testFormat->save();
+
+    // Create a multi-value list_string field.
+    $field_storage = FieldStorageConfig::create([
+      'field_name' => 'test_list',
+      'entity_type' => 'node',
+      'type' => 'list_string',
+      'cardinality' => 2,
+      'settings' => [
+        'allowed_values' => [
+          'key1' => 'value1',
+          'key2' => 'value2',
+        ],
+      ],
+    ]);
+    $field_storage->save();
+
+    $this->field = FieldConfig::create([
+      'field_name' => 'test_list',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+    ])->save();
+
+    // Add an untranslatable node reference field.
+    FieldStorageConfig::create([
+      'field_name' => 'test_reference',
+      'type' => 'entity_reference',
+      'entity_type' => 'node',
+      'settings' => [
+        'target_type' => 'node',
+      ],
+      'translatable' => FALSE,
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'test_reference',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+      'label' => 'Test reference',
+    ])->save();
+
+    // Add an untranslatable taxonomy term reference field.
+    $this->vocabulary = $this->createVocabulary();
+
+    FieldStorageConfig::create([
+      'field_name' => 'test_term_reference',
+      'type' => 'entity_reference',
+      'entity_type' => 'node',
+      'settings' => [
+        'target_type' => 'taxonomy_term',
+      ],
+      'translatable' => FALSE,
+    ])->save();
+    FieldConfig::create([
+      'field_name' => 'test_term_reference',
+      'entity_type' => 'node',
+      'bundle' => 'article',
+      'label' => 'Test term reference',
+      'settings' => [
+        'handler' => 'default:taxonomy_term',
+        'handler_settings' => [
+          'target_bundles' => [
+            $this->vocabulary->id() => $this->vocabulary->id(),
+          ],
+        ],
+      ],
+    ])->save();
+
+    // Add a field to terms of the created vocabulary.
+    $storage = FieldStorageConfig::create([
+      'field_name' => 'term_field',
+      'entity_type' => 'taxonomy_term',
+      'type' => 'text',
+    ]);
+    $storage->save();
+    $field = FieldConfig::create([
+      'field_name' => 'term_field',
+      'entity_type' => 'taxonomy_term',
+      'bundle' => $this->vocabulary->id(),
+    ]);
+    $field->save();
+
+    // Add a second language.
+    $language = ConfigurableLanguage::create([
+      'id' => 'de',
+      'label' => 'German',
+    ]);
+    $language->save();
+
+    // Add a datetime field.
+    $field_datetime_storage = FieldStorageConfig::create([
+      'field_name' => 'field_datetime',
+      'type' => 'datetime',
+      'entity_type' => 'node',
+      'settings' => ['datetime_type' => DateTimeItem::DATETIME_TYPE_DATETIME],
+      'cardinality' => FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED,
+    ]);
+    $field_datetime_storage->save();
+    $field_datetime = FieldConfig::create([
+      'field_storage' => $field_datetime_storage,
+      'bundle' => 'article',
+    ]);
+    $field_datetime->save();
+
+    // Add a daterange field.
+    $field_daterange_storage = FieldStorageConfig::create([
+      'field_name' => 'field_daterange',
+      'type' => 'daterange',
+      'entity_type' => 'node',
+      'settings' => ['datetime_type' => DateRangeItem::DATETIME_TYPE_DATETIME],
+      'cardinality' => FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED,
+    ]);
+    $field_daterange_storage->save();
+    $field_daterange = FieldConfig::create([
+      'field_storage' => $field_daterange_storage,
+      'bundle' => 'article',
+    ]);
+    $field_daterange->save();
+
+    // Add a timestamp field.
+    $field_timestamp_storage = FieldStorageConfig::create([
+      'field_name' => 'field_timestamp',
+      'type' => 'timestamp',
+      'entity_type' => 'node',
+      'cardinality' => FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED,
+    ]);
+    $field_timestamp_storage->save();
+    $field_timestamp = FieldConfig::create([
+      'field_storage' => $field_timestamp_storage,
+      'bundle' => 'article',
+    ]);
+    $field_timestamp->save();
+  }
+
+  /**
+   * Tests [entity:field_name] tokens.
+   */
+  public function testEntityFieldTokens() {
+    // Create a node with a value in its fields and test its tokens.
+    $entity = Node::create([
+      'title' => 'Test node title',
+      'type' => 'article',
+      'test_field' => [
+        'value' => 'foo',
+        'format' => $this->testFormat->id(),
+      ],
+      'test_list' => [
+        'value1',
+        'value2',
+      ],
+    ]);
+    $entity->save();
+    $this->assertTokens('node', ['node' => $entity], [
+      'test_field' => Markup::create('foo'),
+      'test_field:0' => Markup::create('foo'),
+      'test_field:0:value' => 'foo',
+      'test_field:value' => 'foo',
+      'test_field:0:format' => $this->testFormat->id(),
+      'test_field:format' => $this->testFormat->id(),
+      'test_list:0' => Markup::create('value1'),
+      'test_list:1' => Markup::create('value2'),
+      'test_list:0:value' => Markup::create('value1'),
+      'test_list:value' => Markup::create('value1'),
+      'test_list:1:value' => Markup::create('value2'),
+    ]);
+
+    // Verify that no third token was generated for the list_string field.
+    $this->assertNoTokens('node', ['node' => $entity], [
+      'test_list:2',
+      'test_list:2:value',
+    ]);
+
+    // Test the test_list token metadata.
+    $tokenService = \Drupal::service('token');
+    $token_info = $tokenService->getTokenInfo('node', 'test_list');
+    $this->assertEquals('test_list', $token_info['name']);
+    $this->assertEquals('token', $token_info['module']);
+    $this->assertEquals('list<node-test_list>', $token_info['type']);
+    $typeInfo = $tokenService->getTypeInfo('list<node-test_list>');
+    $this->assertEquals('List of test_list values', $typeInfo['name']);
+    $this->assertEquals('list<node-test_list>', $typeInfo['type']);
+
+    // Create a node type that does not have test_field field.
+    $node_type = NodeType::create([
+      'type' => 'page',
+    ]);
+    $node_type->save();
+
+    $node_without_test_field = Node::create([
+      'title' => 'Node without test_field',
+      'type' => 'page',
+    ]);
+    $node_without_test_field->save();
+
+    // Ensure that trying to generate tokens for a non-existing field does not
+    // throw an exception.
+    $this->assertNoTokens('node', ['node' => $node_without_test_field], ['test_field']);
+
+    // Create a node without a value in the text field and test its token.
+    $entity = Node::create([
+      'title' => 'Test node title',
+      'type' => 'article',
+    ]);
+    $entity->save();
+
+    $this->assertNoTokens('node', ['node' => $entity], [
+      'test_field',
+    ]);
+  }
+
+  /**
+   * Tests the token metadata for a field token.
+   */
+  public function testFieldTokenInfo() {
+    /** @var \Drupal\token\Token $tokenService */
+    $tokenService = \Drupal::service('token');
+
+    // Test the token info of the text field of the artcle content type.
+    $token_info = $tokenService->getTokenInfo('node', 'test_field');
+    $this->assertEquals('Test field', $token_info['name'], 'The token info name is correct.');
+    $this->assertEquals('Text (formatted) field.', $token_info['description'], 'The token info description is correct.');
+    $this->assertEquals('token', $token_info['module'], 'The token info module is correct.');
+
+    // Now create two more content types that share the field but the last
+    // of them sets a different label. This should show an alternative label
+    // at the token info.
+    $node_type = NodeType::create([
+      'type' => 'article2',
+    ]);
+    $node_type->save();
+    $field = FieldConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'node',
+      'bundle' => 'article2',
+      'label' => 'Test field',
+    ]);
+    $field->save();
+
+    $node_type = NodeType::create([
+      'type' => 'article3',
+    ]);
+    $node_type->save();
+    $field = FieldConfig::create([
+      'field_name' => 'test_field',
+      'entity_type' => 'node',
+      'bundle' => 'article3',
+      'label' => 'Different test field',
+    ]);
+    $field->save();
+
+    $token_info = $tokenService->getTokenInfo('node', 'test_field');
+    $this->assertEquals('Test field', $token_info['name'], 'The token info name is correct.');
+    $this->assertEquals('Text (formatted) field. Also known as <em class="placeholder">Different test field</em>.', (string) $token_info['description'], 'When a field is used in several bundles with different labels, this is noted at the token info description.');
+    $this->assertEquals('token', $token_info['module'], 'The token info module is correct.');
+    $this->assertEquals('node-test_field', $token_info['type'], 'The field property token info type is correct.');
+
+    // Test field property token info.
+    $token_info = $tokenService->getTokenInfo('node-test_field', 'value');
+    $this->assertEquals('Text', $token_info['name'], 'The field property token info name is correct.');
+    // This particular field property description happens to be empty.
+    $this->assertEquals('', (string) $token_info['description'], 'The field property token info description is correct.');
+    $this->assertEquals('token', $token_info['module'], 'The field property token info module is correct.');
+  }
+
+  /**
+   * Test tokens on node with the token view mode overriding default formatters.
+   */
+  public function testTokenViewMode() {
+    $value = 'A really long string that should be trimmed by the special formatter on token view we are going to have.';
+
+    // The formatter we are going to use will eventually call Unicode::strlen.
+    // This expects that the Unicode has already been explicitly checked, which
+    // happens in DrupalKernel. But since that doesn't run in kernel tests, we
+    // explicitly call this here.
+    Unicode::check();
+
+    // Create a node with a value in the text field and test its token.
+    $entity = Node::create([
+      'title' => 'Test node title',
+      'type' => 'article',
+      'test_field' => [
+        'value' => $value,
+        'format' => $this->testFormat->id(),
+      ],
+    ]);
+    $entity->save();
+
+    $this->assertTokens('node', ['node' => $entity], [
+      'test_field' => Markup::create($value),
+    ]);
+
+    // Now, create a token view mode which sets a different format for
+    // test_field. When replacing tokens, this formatter should be picked over
+    // the default formatter for the field type.
+    // @see field_tokens().
+    $view_mode = EntityViewMode::create([
+      'id' => 'node.token',
+      'targetEntityType' => 'node',
+    ]);
+    $view_mode->save();
+    $entity_display = \Drupal::service('entity_display.repository')->getViewDisplay('node', 'article', 'token');
+    $entity_display->setComponent('test_field', [
+      'type' => 'text_trimmed',
+      'settings' => [
+        'trim_length' => 50,
+      ],
+    ]);
+    $entity_display->save();
+
+    $this->assertTokens('node', ['node' => $entity], [
+      'test_field' => Markup::create(substr($value, 0, 50)),
+    ]);
+  }
+
+  /**
+   * Test that tokens are properly created for an entity's base fields.
+   */
+  public function testBaseFieldTokens() {
+    // Create a new contact_message entity and verify that tokens are generated
+    // for its base fields. The contact_message entity type is used because it
+    // provides no tokens by default.
+    $contact_form = ContactForm::create([
+      'id' => 'form_id',
+    ]);
+    $contact_form->save();
+
+    $entity = Message::create([
+      'contact_form' => 'form_id',
+      'uuid' => '123',
+      'langcode' => 'en',
+      'name' => 'Test name',
+      'mail' => 'Test mail',
+      'subject' => 'Test subject',
+      'message' => 'Test message',
+      'copy' => FALSE,
+    ]);
+    $entity->save();
+    $this->assertTokens('contact_message', ['contact_message' => $entity], [
+      'uuid' => Markup::create('123'),
+      'langcode' => Markup::create('English'),
+      'name' => Markup::create('Test name'),
+      'mail' => Markup::create('Test mail'),
+      'subject' => Markup::create('Test subject'),
+      'message' => Markup::create('Test message'),
+      'copy' => 'Off',
+    ]);
+
+    // Test the metadata of one of the tokens.
+    $tokenService = \Drupal::service('token');
+    $token_info = $tokenService->getTokenInfo('contact_message', 'subject');
+    $this->assertEquals($token_info['name'], 'Subject');
+    $this->assertEquals($token_info['description'], 'Text (plain) field.');
+    $this->assertEquals($token_info['module'], 'token');
+
+    // Verify that node entity type doesn't have a uid token.
+    $this->assertNull($tokenService->getTokenInfo('node', 'uid'));
+  }
+
+  /**
+   * Tests chaining entity reference tokens.
+   */
+  public function testEntityReferenceTokens() {
+    $reference = Node::create([
+      'title' => 'Test node to reference',
+      'type' => 'article',
+      'test_field' => [
+        'value' => 'foo',
+        'format' => $this->testFormat->id(),
+      ],
+    ]);
+    $reference->save();
+    $term_reference_field_value = $this->randomString();
+    $term_reference = $this->createTerm($this->vocabulary, [
+      'name' => 'Term to reference',
+      'term_field' => [
+        'value' => $term_reference_field_value,
+        'format' => $this->testFormat->id(),
+      ],
+    ]);
+    $entity = Node::create([
+      'title' => 'Test entity reference',
+      'type' => 'article',
+      'test_reference' => ['target_id' => $reference->id()],
+      'test_term_reference' => ['target_id' => $term_reference->id()],
+    ]);
+    $entity->save();
+
+    $this->assertTokens('node', ['node' => $entity], [
+      'test_reference:entity:title' => Markup::create('Test node to reference'),
+      'test_reference:entity:test_field' => Markup::create('foo'),
+      'test_term_reference:entity:term_field' => Html::escape($term_reference_field_value),
+      'test_reference:target_id' => $reference->id(),
+      'test_term_reference:target_id' => $term_reference->id(),
+      'test_term_reference:entity:url:path' => '/' . $term_reference->toUrl('canonical')->getInternalPath(),
+      // Expects the entity's label to be returned for :entity tokens.
+      'test_reference:entity' => $reference->label(),
+      'test_term_reference:entity' => $term_reference->label(),
+    ]);
+
+    // Test some non existent tokens.
+    $this->assertNoTokens('node', ['node' => $entity], [
+      'test_reference:1:title',
+      'test_reference:entity:does_not_exist',
+      'test_reference:does_not:exist',
+      'test_term_reference:does_not_exist',
+      'test_term_reference:does:not:exist',
+      'test_term_reference:does_not_exist:0',
+      'non_existing_field:entity:title',
+    ]);
+
+    /** @var \Drupal\token\Token $token_service */
+    $token_service = \Drupal::service('token');
+
+    $token_info = $token_service->getTokenInfo('node', 'test_reference');
+    $this->assertEquals('Test reference', $token_info['name']);
+    $this->assertEquals('Entity reference field.', (string) $token_info['description']);
+    $this->assertEquals('token', $token_info['module']);
+    $this->assertEquals('node-test_reference', $token_info['type']);
+
+    // Test target_id field property token info.
+    $token_info = $token_service->getTokenInfo('node-test_reference', 'target_id');
+    $this->assertEquals('Content ID', $token_info['name']);
+    $this->assertEquals('token', $token_info['module']);
+    $this->assertEquals('token', $token_info['module']);
+
+    // Test entity field property token info.
+    $token_info = $token_service->getTokenInfo('node-test_reference', 'entity');
+    $this->assertEquals('Content', $token_info['name']);
+    $this->assertEquals('The referenced entity', $token_info['description']);
+    $this->assertEquals('token', $token_info['module']);
+    $this->assertEquals('node', $token_info['type']);
+
+    // Test entity field property token info of the term reference.
+    $token_info = $token_service->getTokenInfo('node-test_term_reference', 'entity');
+    $this->assertEquals('Taxonomy term', $token_info['name']);
+    $this->assertEquals('The referenced entity', $token_info['description']);
+    $this->assertEquals('token', $token_info['module']);
+    $this->assertEquals('term', $token_info['type']);
+
+  }
+
+  /**
+   * Tests support for cardinality > 1 for entity reference tokens.
+   */
+  public function testEntityReferenceTokensCardinality() {
+    /** @var \Drupal\field\FieldStorageConfigInterface $storage */
+    $storage = FieldStorageConfig::load('node.test_term_reference');
+    $storage->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED);
+    $storage->save();
+
+    // Add a few terms.
+    $terms = [];
+    $terms_value = [];
+    foreach (range(1, 3) as $i) {
+      $terms_value[$i] = $this->randomString();
+      $terms[$i] = $this->createTerm($this->vocabulary, [
+        'name' => $this->randomString(),
+        'term_field' => [
+          'value' => $terms_value[$i],
+          'format' => $this->testFormat->id(),
+        ],
+      ]);
+    }
+
+    $entity = Node::create([
+      'title' => 'Test multivalue chained tokens',
+      'type' => 'article',
+      'test_term_reference' => [
+        ['target_id' => $terms[1]->id()],
+        ['target_id' => $terms[2]->id()],
+        ['target_id' => $terms[3]->id()],
+      ],
+    ]);
+    $entity->save();
+
+    $this->assertTokens('node', ['node' => $entity], [
+      'test_term_reference:0:entity:term_field' => Html::escape($terms[1]->term_field->value),
+      'test_term_reference:1:entity:term_field' => Html::escape($terms[2]->term_field->value),
+      'test_term_reference:2:entity:term_field' => Html::escape($terms[3]->term_field->value),
+      'test_term_reference:0:target_id' => $terms[1]->id(),
+      'test_term_reference:1:target_id' => $terms[2]->id(),
+      'test_term_reference:2:target_id' => $terms[3]->id(),
+      // Expects the entity's label to be returned for :entity tokens.
+      'test_term_reference:0:entity' => $terms[1]->label(),
+      'test_term_reference:1:entity' => $terms[2]->label(),
+      'test_term_reference:2:entity' => $terms[3]->label(),
+      // To make sure tokens without an explicit delta can also be replaced in
+      // the same token replacement call.
+      'test_term_reference:entity:term_field' => Html::escape($terms[1]->term_field->value),
+      'test_term_reference:target_id' => $terms[1]->id(),
+    ]);
+
+    // Test some non existent tokens.
+    $this->assertNoTokens('node', ['node' => $entity], [
+      'test_term_reference:3:term_field',
+      'test_term_reference:0:does_not_exist',
+      'test_term_reference:1:does:not:exist',
+      'test_term_reference:1:2:does_not_exist',
+    ]);
+  }
+
+  /**
+   * Test tokens for multilingual fields and entities.
+   */
+  public function testMultilingualFields() {
+    // Create an english term and add a german translation for it.
+    $term = $this->createTerm($this->vocabulary, [
+      'name' => 'english-test-term',
+      'langcode' => 'en',
+      'term_field' => [
+        'value' => 'english-term-field-value',
+        'format' => $this->testFormat->id(),
+      ],
+    ]);
+    $term->addTranslation('de', [
+      'name' => 'german-test-term',
+      'term_field' => [
+        'value' => 'german-term-field-value',
+        'format' => $this->testFormat->id(),
+      ],
+    ])->save();
+    $german_term = $term->getTranslation('de');
+
+    // Create an english node, add a german translation for it and add the
+    // english term to the english node's entity reference field and the
+    // german term to the german's entity reference field.
+    $node = Node::create([
+      'title' => 'english-node-title',
+      'type' => 'article',
+      'test_term_reference' => [
+        'target_id' => $term->id(),
+      ],
+      'test_field' => [
+        'value' => 'test-english-field',
+        'format' => $this->testFormat->id(),
+      ],
+    ]);
+    $node->addTranslation('de', [
+      'title' => 'german-node-title',
+      'test_term_reference' => [
+        'target_id' => $german_term->id(),
+      ],
+      'test_field' => [
+        'value' => 'test-german-field',
+        'format' => $this->testFormat->id(),
+      ],
+    ])->save();
+
+    // Verify the :title token of the english node and the :name token of the
+    // english term it refers to. Also verify the value of the term's field.
+    $this->assertTokens('node', ['node' => $node], [
+      'title' => 'english-node-title',
+      'test_term_reference:entity:name' => 'english-test-term',
+      'test_term_reference:entity:term_field:value' => 'english-term-field-value',
+      'test_term_reference:entity:term_field' => 'english-term-field-value',
+      'test_field' => 'test-english-field',
+      'test_field:value' => 'test-english-field',
+    ]);
+
+    // Same test for the german node and its german term.
+    $german_node = $node->getTranslation('de');
+    $this->assertTokens('node', ['node' => $german_node], [
+      'title' => 'german-node-title',
+      'test_term_reference:entity:name' => 'german-test-term',
+      'test_term_reference:entity:term_field:value' => 'german-term-field-value',
+      'test_term_reference:entity:term_field' => 'german-term-field-value',
+      'test_field' => 'test-german-field',
+      'test_field:value' => 'test-german-field',
+    ]);
+
+    // If the langcode is specified, it should have priority over the node's
+    // active language.
+    $tokens = [
+      'test_field' => 'test-german-field',
+      'test_field:value' => 'test-german-field',
+      'test_term_reference:entity:term_field' => 'german-term-field-value',
+      'test_term_reference:entity:term_field:value' => 'german-term-field-value',
+    ];
+    $this->assertTokens('node', ['node' => $node], $tokens, ['langcode' => 'de']);
+  }
+
+  /**
+   * Tests support for a datetime fields.
+   */
+  public function testDatetimeFieldTokens() {
+
+    $node = Node::create([
+      'title' => 'Node for datetime field',
+      'type' => 'article',
+    ]);
+
+    $node->set('field_datetime', ['1925-09-28T00:00:00', '1930-10-28T00:00:00'])->save();
+    $this->assertTokens('node', ['node' => $node], [
+      'field_datetime:date:custom:Y' => '1925',
+      'field_datetime:date:html_month' => '1925-09',
+      'field_datetime:date' => $node->get('field_datetime')->date->getTimestamp(),
+      'field_datetime:0:date:custom:Y' => '1925',
+      'field_datetime:0:date:html_month' => '1925-09',
+      'field_datetime:0:date' => $node->get('field_datetime')->date->getTimestamp(),
+      'field_datetime:1:date:custom:Y' => '1930',
+      'field_datetime:1:date:html_month' => '1930-10',
+      'field_datetime:1:date' => $node->get('field_datetime')->get(1)->date->getTimestamp(),
+    ]);
+  }
+
+  /**
+   * Tests support for a daterange fields.
+   */
+  public function testDatetimeRangeFieldTokens() {
+
+    /** @var \Drupal\node\NodeInterface $node */
+    $node = Node::create([
+      'title' => 'Node for daterange field',
+      'type' => 'article',
+    ]);
+
+    $node->get('field_daterange')->value = '2013-12-22T00:00:00';
+    $node->get('field_daterange')->end_value = '2016-08-26T00:00:00';
+    $node->get('field_daterange')->appendItem([
+      'value' => '2014-08-22T00:00:00',
+      'end_value' => '2017-12-20T00:00:00',
+    ]);
+    $node->get('field_daterange')->value = '2013-12-22T00:00:00';
+    $node->get('field_daterange')->end_value = '2016-08-26T00:00:00';
+    $node->save();
+    $this->assertTokens('node', ['node' => $node], [
+      'field_daterange:start_date:html_month' => '2013-12',
+      'field_daterange:start_date:custom:Y' => '2013',
+      'field_daterange:end_date:custom:Y' => '2016',
+      'field_daterange:start_date' => $node->get('field_daterange')->start_date->getTimestamp(),
+      'field_daterange:0:start_date:html_month' => '2013-12',
+      'field_daterange:0:start_date:custom:Y' => '2013',
+      'field_daterange:0:end_date:custom:Y' => '2016',
+      'field_daterange:0:start_date' => $node->get('field_daterange')->start_date->getTimestamp(),
+      'field_daterange:1:start_date:html_month' => '2014-08',
+      'field_daterange:1:start_date:custom:Y' => '2014',
+      'field_daterange:1:end_date:custom:Y' => '2017',
+      'field_daterange:1:end_date' => $node->get('field_daterange')->get(1)->end_date->getTimestamp(),
+    ]);
+  }
+
+  /**
+   * Tests support for a timestamp fields.
+   */
+  public function testTimestampFieldTokens() {
+
+    $node = Node::create([
+      'title' => 'Node for timestamp field',
+      'type' => 'article',
+    ]);
+
+    $node->set('field_timestamp', ['1277540209', '1532593009'])->save();
+    $this->assertTokens('node', ['node' => $node], [
+      'field_timestamp:date:custom:Y' => '2010',
+      'field_timestamp:date:html_month' => '2010-06',
+      'field_timestamp:date' => $node->get('field_timestamp')->value,
+      'field_timestamp:0:date:custom:Y' => '2010',
+      'field_timestamp:0:date:html_month' => '2010-06',
+      'field_timestamp:0:date' => $node->get('field_timestamp')->value,
+      'field_timestamp:1:date:custom:Y' => '2018',
+      'field_timestamp:1:date:html_month' => '2018-07',
+      'field_timestamp:1:date' => $node->get('field_timestamp')->get(1)->value,
+    ]);
+  }
+
+}

modules/token/tests/src/Kernel/FileTest.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\file\Entity\File;
+/**
+ * Tests file tokens.
+ *
+ * @group token
+ */
+class FileTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['file'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->installEntitySchema('file');
+  }
+
+  function testFileTokens() {
+    // Create a test file object.
+    $file = File::create([
+      'fid' => 1,
+      'filename' => 'test.png',
+      'filesize' => 100,
+      'uri' => 'public://images/test.png',
+      'filemime' => 'image/png',
+    ]);
+
+    $tokens = [
+      'basename' => 'test.png',
+      'extension' => 'png',
+      'size-raw' => 100,
+    ];
+    $this->assertTokens('file', ['file' => $file], $tokens);
+
+    // Test a file with no extension and a fake name.
+    $file->filename = 'Test PNG image';
+    $file->uri = 'public://images/test';
+
+    $tokens = [
+      'basename' => 'test',
+      'extension' => '',
+      'size-raw' => 100,
+    ];
+    $this->assertTokens('file', ['file' => $file], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/KernelTestBase.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+@\trigger_error('The ' . __NAMESPACE__ . '\KernelTestBase class is deprecated in token:8.x-1.14 and is removed from token:2.0.0. Use \Drupal\Tests\token\Kernel\TokenKernelTestBase instead. See https://www.drupal.org/node/3440940', E_USER_DEPRECATED);
+
+/**
+ * Helper test class with some added functions for testing.
+ *
+ * @deprecated in token:8.x-1.14 and is removed from token:2.0.0. Use
+ *   \Drupal\Tests\token\Kernel\TokenKernelTestBase instead.
+ *
+ * @see https://www.drupal.org/node/3440940
+ */
+abstract class KernelTestBase extends TokenKernelTestBase {
+}

modules/token/tests/src/Kernel/LanguageTest.php

@@ -0,0 +1,367 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\DependencyInjection\ContainerBuilder;
+use Drupal\Core\Language\Language;
+use Drupal\Core\Language\LanguageInterface;
+use Drupal\Core\Render\BubbleableMetadata;
+use Drupal\language\Entity\ConfigurableLanguage;
+
+/**
+ * Tests language tokens.
+ *
+ * @group token
+ */
+class LanguageTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'language',
+    'token',
+  ];
+
+  /**
+   * Language codes of languages to enable during the test.
+   *
+   * @var array
+   */
+  protected $langcodes = ['bg', 'hu', 'nl', 'pt-pt'];
+
+  /**
+   * An array of languages used during the test, keyed by language code.
+   *
+   * @var \Drupal\language\Entity\ConfigurableLanguage[]
+   */
+  protected $languages = [];
+
+  /**
+   * Language prefixes used during the test.
+   *
+   * @var array
+   */
+  protected $language_prefixes = [];
+
+  /**
+   * Language domains used during the test.
+   *
+   * @var array
+   */
+  protected $language_domains = [];
+
+  /**
+   * The token replacement service.
+   *
+   * @var \Drupal\Core\Utility\Token
+   */
+  protected $token;
+
+  /**
+   * The mock language manager service.
+   *
+   * @var \Drupal\Tests\token\Kernel\MockLanguageManager
+   */
+  protected $languageManager;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function register(ContainerBuilder $container) {
+    parent::register($container);
+
+    // Use Portuguese as the default language during the test. We're not using
+    // English so we can detect if the default language is correctly honored.
+    $language = Language::$defaultValues;
+    $language['id'] = 'pt-pt';
+    $language['name'] = 'Portuguese, Portugal';
+    $container->setParameter('language.default_values', $language);
+    $this->container
+      ->register('language.default', 'Drupal\Core\Language\LanguageDefault')
+      ->addArgument('%language.default_values%');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->token = $this->container->get('token');
+
+    // Use a version of the language manager in which the various languages can
+    // be easily overridden during the test. We need to do this here instead of
+    // in ::register() since the container is being altered by
+    // LanguageServiceProvider::alter() after the services have been registered.
+    $this->languageManager = new MockLanguageManager(
+      $this->container->get('language.default'),
+      $this->container->get('config.factory'),
+      $this->container->get('module_handler'),
+      $this->container->get('language.config_factory_override'),
+      $this->container->get('request_stack')
+    );
+    $this->container->set('language_manager', $this->languageManager);
+
+    foreach ($this->langcodes as $langcode) {
+      // Enable test languages.
+      $this->languages[$langcode] = ConfigurableLanguage::createFromLangcode($langcode);
+      $this->languages[$langcode]->save();
+
+      // Populate language prefixes and domains to use in the test.
+      $this->language_prefixes[$langcode] = "$langcode-prefix";
+      $this->language_domains[$langcode] = $langcode . '.example.com';
+    }
+
+    // Set language negotiation prefixes and domains to values that are uniquely
+    // identifiable in the test.
+    $language_negotiation_config = $this->config('language.negotiation');
+    $language_negotiation_config->set('url.prefixes', $this->language_prefixes);
+    $language_negotiation_config->set('url.domains', $this->language_domains);
+    $language_negotiation_config->save();
+  }
+
+  /**
+   * Tests the language tokens.
+   *
+   * @dataProvider languageTokenReplacementDataProvider
+   */
+  public function testLanguageTokenReplacement($token, $langcode, $expected_result) {
+    $bubbleable_metadata = new BubbleableMetadata();
+    $options = $langcode ? ['langcode' => $langcode] : [];
+    // The part of the token name between the last `:` and the closing bracket
+    // is the machine name of the token.
+    preg_match('/\[.+:(.+)\]/', $token, $matches);
+    $name = $matches[1];
+    $replacements = $this->token->generate('language', [$name => $token], [], $options, $bubbleable_metadata);
+    $this->assertEquals($expected_result, $replacements[$token]);
+  }
+
+  /**
+   * Tests retrieving the interface and content language from the current page.
+   *
+   * @dataProvider currentPageLanguageTokenReplacementDataProvider
+   */
+  public function testCurrentPageLanguageTokenReplacement($token, $langcode, $expected_result) {
+    // Set the interface language to Dutch.
+    $this->languageManager->setCurrentLanguage(LanguageInterface::TYPE_INTERFACE, $this->languages['nl']);
+    // Set the content language to Hungarian.
+    $this->languageManager->setCurrentLanguage(LanguageInterface::TYPE_CONTENT, $this->languages['hu']);
+
+    $options = $langcode ? ['langcode' => $langcode] : [];
+    $result = $this->token->replace($token, [], $options);
+    $this->assertEquals($expected_result, $result);
+  }
+
+  /**
+   * Provides test data for ::testLanguageTokenReplacement().
+   *
+   * @return array
+   *   An array of test cases. Each test case is an array with the following
+   *   values:
+   *   - The token to test.
+   *   - An optional language code to pass as an option.
+   *   - The expected result of the token replacement.
+   *
+   * @see testLanguageTokenReplacement()
+   */
+  public function languageTokenReplacementDataProvider() {
+    return [
+      [
+        // Test the replacement of the name of the site default language.
+        '[language:name]',
+        // We are not overriding the language by passing a language code as an
+        // option. This means that the default language should be used which has
+        // been set to Portuguese.
+        NULL,
+        // The expected result.
+        'Portuguese, Portugal',
+      ],
+      // Test the replacement of the other properties of the default language.
+      [
+        '[language:langcode]',
+        NULL,
+        'pt-pt',
+      ],
+      [
+        '[language:direction]',
+        NULL,
+        'ltr',
+      ],
+      [
+        '[language:domain]',
+        NULL,
+        'pt-pt.example.com',
+      ],
+      [
+        '[language:prefix]',
+        NULL,
+        'pt-pt-prefix',
+      ],
+      // Now repeat the entire test but override the language to use by passing
+      // Bulgarian as an option.
+      [
+        '[language:name]',
+        'bg',
+        'Bulgarian',
+      ],
+      [
+        '[language:langcode]',
+        'bg',
+        'bg',
+      ],
+      [
+        '[language:direction]',
+        'bg',
+        'ltr',
+      ],
+      [
+        '[language:domain]',
+        'bg',
+        'bg.example.com',
+      ],
+      [
+        '[language:prefix]',
+        'bg',
+        'bg-prefix',
+      ],
+    ];
+  }
+
+  /**
+   * Provides test data for ::testCurrentPageLanguageTokenReplacement().
+   *
+   * @return array
+   *   An array of test cases. Each test case is an array with the following
+   *   values:
+   *   - The token to test.
+   *   - An optional language code to pass as an option.
+   *   - The expected result of the token replacement.
+   *
+   * @see testCurrentPageLanguageTokenReplacement()
+   */
+  public function currentPageLanguageTokenReplacementDataProvider() {
+    return [
+      [
+        // Test the replacement of the language name token, taken from the
+        // interface language of the current page.
+        '[current-page:interface-language:name]',
+        // We are not overriding the language by passing a language code as an
+        // option. This means that the language should be taken from the
+        // interface language which has been set to Dutch.
+        NULL,
+        // The expected result.
+        'Dutch',
+      ],
+      // Test the token name in the content language.
+      [
+        '[current-page:content-language:name]',
+        NULL,
+        'Hungarian',
+      ],
+      // Test the other tokens both for the content and interface languages.
+      [
+        '[current-page:interface-language:langcode]',
+        NULL,
+        'nl',
+      ],
+      [
+        '[current-page:content-language:langcode]',
+        NULL,
+        'hu',
+      ],
+      [
+        '[current-page:interface-language:direction]',
+        NULL,
+        'ltr',
+      ],
+      [
+        '[current-page:content-language:direction]',
+        NULL,
+        'ltr',
+      ],
+      [
+        '[current-page:interface-language:domain]',
+        NULL,
+        'nl.example.com',
+      ],
+      [
+        '[current-page:content-language:domain]',
+        NULL,
+        'hu.example.com',
+      ],
+      [
+        '[current-page:interface-language:prefix]',
+        NULL,
+        'nl-prefix',
+      ],
+      [
+        '[current-page:content-language:prefix]',
+        NULL,
+        'hu-prefix',
+      ],
+      // Now repeat the entire test with Bulgarian passed as an option. This
+      // should not affect the results, the language should be sourced from the
+      // current page.
+      [
+        // Test the replacement of the language name token, taken from the
+        // interface language of the current page.
+        '[current-page:interface-language:name]',
+        // We are not overriding the language by passing a language code as an
+        // option. This means that the language should be taken from the
+        // interface language which has been set to Dutch.
+        'bg',
+        // The expected result.
+        'Dutch',
+      ],
+      // Test the token name in the content language.
+      [
+        '[current-page:content-language:name]',
+        'bg',
+        'Hungarian',
+      ],
+      // Test the other tokens both for the content and interface languages.
+      [
+        '[current-page:interface-language:langcode]',
+        'bg',
+        'nl',
+      ],
+      [
+        '[current-page:content-language:langcode]',
+        'bg',
+        'hu',
+      ],
+      [
+        '[current-page:interface-language:direction]',
+        'bg',
+        'ltr',
+      ],
+      [
+        '[current-page:content-language:direction]',
+        'bg',
+        'ltr',
+      ],
+      [
+        '[current-page:interface-language:domain]',
+        'bg',
+        'nl.example.com',
+      ],
+      [
+        '[current-page:content-language:domain]',
+        'bg',
+        'hu.example.com',
+      ],
+      [
+        '[current-page:interface-language:prefix]',
+        'bg',
+        'nl-prefix',
+      ],
+      [
+        '[current-page:content-language:prefix]',
+        'bg',
+        'hu-prefix',
+      ],
+    ];
+  }
+
+}

modules/token/tests/src/Kernel/MockLanguageManager.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\Language\LanguageInterface;
+use Drupal\language\ConfigurableLanguageManager;
+
+/**
+ * A language manager that can be easily overridden for testing purposes.
+ */
+class MockLanguageManager extends ConfigurableLanguageManager {
+
+  /**
+   * List of current languages used in the test.
+   *
+   * @var \Drupal\Core\Language\LanguageInterface[]
+   */
+  protected $currentLanguages;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getCurrentLanguage($type = LanguageInterface::TYPE_INTERFACE) {
+    if (isset($this->currentLanguages[$type])) {
+      return $this->currentLanguages[$type];
+    }
+    return parent::getCurrentLanguage($type);
+  }
+
+  /**
+   * Sets the current language of the given type to use during tests.
+   *
+   * @param string $type
+   *   The language type.
+   * @param \Drupal\Core\Language\LanguageInterface $language
+   *   The language.
+   */
+  public function setCurrentLanguage($type, LanguageInterface $language) {
+    $this->currentLanguages[$type] = $language;
+  }
+
+}

modules/token/tests/src/Kernel/NodeTest.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\Url;
+use Drupal\node\Entity\Node;
+use Drupal\node\Entity\NodeType;
+
+/**
+ * Test the node and content type tokens.
+ *
+ * @group token
+ */
+class NodeTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['node', 'field', 'text'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('user');
+    $this->installEntitySchema('node');
+
+    $node_type = NodeType::create([
+      'type' => 'page',
+      'name' => 'Basic page',
+      'description' => "Use <em>basic pages</em> for your static content, such as an 'About us' page.",
+    ]);
+    $node_type->save();
+    $node_type = NodeType::create([
+      'type' => 'article',
+      'name' => 'Article',
+      'description' => "Use <em>articles</em> for time-sensitive content like news, press releases or blog posts.",
+    ]);
+    $node_type->save();
+  }
+
+  /**
+   *
+   */
+  public function testNodeTokens() {
+    $page = Node::create([
+      'type' => 'page',
+      'title' => 'Source Title',
+      'revision_log' => $this->randomMachineName(),
+      'path' => ['alias' => '/content/source-node'],
+    ]);
+    $page->save();
+    $tokens = [
+      'log' => $page->revision_log->value,
+      'url:path' => '/content/source-node',
+      'url:absolute' => Url::fromRoute('entity.node.canonical', ['node' => $page->id()], ['absolute' => TRUE])->toString(),
+      'url:relative' => Url::fromRoute('entity.node.canonical', ['node' => $page->id()], ['absolute' => FALSE])->toString(),
+      'url:unaliased:path' => "/node/{$page->id()}",
+      'content-type' => 'Basic page',
+      'content-type:name' => 'Basic page',
+      'content-type:machine-name' => 'page',
+      'content-type:description' => "Use <em>basic pages</em> for your static content, such as an 'About us' page.",
+      'content-type:node-count' => 1,
+      'content-type:edit-url' => Url::fromRoute('entity.node_type.edit_form', ['node_type' => 'page'], ['absolute' => TRUE])->toString(),
+      'source:title' => 'Source Title',
+      // Deprecated tokens.
+      'type' => 'page',
+      'type-name' => 'Basic page',
+      'url:alias' => '/content/source-node',
+      'language:name' => 'English',
+    ];
+    $this->assertTokens('node', ['node' => $page], $tokens);
+
+    $article = Node::create([
+      'type' => 'article',
+      'title' => 'Source Title',
+    ]);
+    $article->save();
+    $tokens = [
+      'log' => '',
+      'url:path' => "/node/{$article->id()}",
+      'url:absolute' => Url::fromRoute('entity.node.canonical', ['node' => $article->id()], ['absolute' => TRUE])->toString(),
+      'url:relative' => Url::fromRoute('entity.node.canonical', ['node' => $article->id()], ['absolute' => FALSE])->toString(),
+      'url:unaliased:path' => "/node/{$article->id()}",
+      'content-type' => 'Article',
+      'content-type:name' => 'Article',
+      'content-type:machine-name' => 'article',
+      'content-type:description' => "Use <em>articles</em> for time-sensitive content like news, press releases or blog posts.",
+      'content-type:node-count' => 1,
+      'content-type:edit-url' => Url::fromRoute('entity.node_type.edit_form', ['node_type' => 'article'], ['absolute' => TRUE])->toString(),
+      'source:title' => 'Source Title',
+      // Deprecated tokens.
+      'type' => 'article',
+      'type-name' => 'Article',
+      'url:alias' => "/node/{$article->id()}",
+      'language:name' => 'English',
+    ];
+    $this->assertTokens('node', ['node' => $article], $tokens);
+  }
+
+}

modules/token/tests/src/Kernel/RandomTest.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+/**
+ * Tests random tokens.
+ *
+ * @group token
+ */
+class RandomTest extends TokenKernelTestBase {
+
+  function testRandomTokens() {
+    $tokens = [
+      'number' => '[0-9]{1,}',
+      'hash:md5' => '[0-9a-f]{32}',
+      'hash:sha1' => '[0-9a-f]{40}',
+      'hash:sha256' => '[0-9a-f]{64}',
+      'hash:invalid-algo' => NULL,
+    ];
+
+    $first_set = $this->assertTokens('random', [], $tokens, ['regex' => TRUE]);
+    $second_set = $this->assertTokens('random', [], $tokens, ['regex' => TRUE]);
+    foreach ($first_set as $token => $value) {
+      $this->assertNotSame($first_set[$token], $second_set[$token]);
+    }
+  }
+
+}

modules/token/tests/src/Kernel/TaxonomyTest.php

@@ -0,0 +1,162 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\Url;
+use Drupal\language\Entity\ConfigurableLanguage;
+use Drupal\taxonomy\Entity\Term;
+use Drupal\taxonomy\Entity\Vocabulary;
+
+/**
+ * Tests taxonomy tokens.
+ *
+ * @group token
+ */
+class TaxonomyTest extends TokenKernelTestBase {
+
+  protected $vocab;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['taxonomy', 'text', 'language'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('taxonomy_term');
+
+    // Create the default tags vocabulary.
+    $vocabulary = Vocabulary::create([
+      'name' => 'Tags',
+      'vid' => 'tags',
+    ]);
+    $vocabulary->save();
+    $this->vocab = $vocabulary;
+  }
+
+  /**
+   * Test the additional taxonomy term tokens.
+   */
+  public function testTaxonomyTokens() {
+    $root_term = $this->addTerm($this->vocab, ['name' => 'Root term', 'path' => ['alias' => '/root-term']]);
+    $tokens = [
+      'url' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $root_term->id()], ['absolute' => TRUE])->toString(),
+      'url:absolute' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $root_term->id()], ['absolute' => TRUE])->toString(),
+      'url:relative' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $root_term->id()], ['absolute' => FALSE])->toString(),
+      'url:path' => '/root-term',
+      'url:unaliased:path' => "/taxonomy/term/{$root_term->id()}",
+      'edit-url' => Url::fromRoute('entity.taxonomy_term.edit_form', ['taxonomy_term' => $root_term->id()], ['absolute' => TRUE])->toString(),
+      'parents' => NULL,
+      'parents:count' => NULL,
+      'parents:keys' => NULL,
+      'root' => NULL,
+      // Deprecated tokens.
+      'url:alias' => '/root-term',
+    ];
+    $this->assertTokens('term', ['term' => $root_term], $tokens);
+
+    $parent_term = $this->addTerm($this->vocab, ['name' => 'Parent term', 'parent' => $root_term->id()]);
+    $tokens = [
+      'url' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $parent_term->id()], ['absolute' => TRUE])->toString(),
+      'url:absolute' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $parent_term->id()], ['absolute' => TRUE])->toString(),
+      'url:relative' => Url::fromRoute('entity.taxonomy_term.canonical', ['taxonomy_term' => $parent_term->id()], ['absolute' => FALSE])->toString(),
+      'url:path' => "/taxonomy/term/{$parent_term->id()}",
+      'url:unaliased:path' => "/taxonomy/term/{$parent_term->id()}",
+      'edit-url' => Url::fromRoute('entity.taxonomy_term.edit_form', ['taxonomy_term' => $parent_term->id()], ['absolute' => TRUE])->toString(),
+      'parents' => 'Root term',
+      'parents:count' => 1,
+      'parents:keys' => $root_term->id(),
+      'root' => $root_term->label(),
+      'root:tid' => $root_term->id(),
+      // Deprecated tokens.
+      'url:alias' => "/taxonomy/term/{$parent_term->id()}",
+    ];
+    $this->assertTokens('term', ['term' => $parent_term], $tokens);
+
+    $term = $this->addTerm($this->vocab, ['name' => 'Test term', 'parent' => $parent_term->id()]);
+    $tokens = [
+      'parents' => 'Root term, Parent term',
+      'parents:count' => 2,
+      'parents:keys' => implode(', ', [$root_term->id(), $parent_term->id()]),
+    ];
+    $this->assertTokens('term', ['term' => $term], $tokens);
+  }
+
+  /**
+   * Test the additional vocabulary tokens.
+   */
+  public function testVocabularyTokens() {
+    $vocabulary = $this->vocab;
+    $tokens = [
+      'machine-name' => 'tags',
+      'edit-url' => Url::fromRoute('entity.taxonomy_vocabulary.edit_form', ['taxonomy_vocabulary' => $vocabulary->id()], ['absolute' => TRUE])->toString(),
+    ];
+    $this->assertTokens('vocabulary', ['vocabulary' => $vocabulary], $tokens);
+  }
+
+  /**
+   *
+   */
+  public function addVocabulary(array $vocabulary = []) {
+    $vocabulary += [
+      'name' => mb_strtolower($this->randomMachineName(5)),
+      'nodes' => ['article' => 'article'],
+    ];
+    $vocabulary = Vocabulary::create($vocabulary)->save();
+    return $vocabulary;
+  }
+
+  /**
+   *
+   */
+  public function addTerm($vocabulary, array $term = []) {
+    $term += [
+      'name' => mb_strtolower($this->randomMachineName(5)),
+      'vid' => $vocabulary->id(),
+    ];
+    $term = Term::create($term);
+    $term->save();
+    return $term;
+  }
+
+  /**
+   * Test the multilingual terms.
+   */
+  public function testMultilingualTerms() {
+    // Add a second language.
+    $language = ConfigurableLanguage::createFromLangcode('de');
+    $language->save();
+
+    // Create an english parent term and add a german translation for it.
+    $parent_term = $this->addTerm($this->vocab, [
+      'name' => 'english-parent-term',
+      'langcode' => 'en',
+    ]);
+    $parent_term->addTranslation('de', [
+      'name' => 'german-parent-term',
+    ])->save();
+
+    // Check translation source tokens.
+    $this->assertTokens('term', ['term' => $parent_term], ['source:name' => 'english-parent-term']);
+    $this->assertTokens('term', ['term' => $parent_term], ['source:name' => 'english-parent-term'], ['langcode' => 'de']);
+
+    // Create a term related to the parent term.
+    $child_term = $this->addTerm($this->vocab, [
+      'name' => 'english-child-term',
+      'langcode' => 'en',
+      'parent' => $parent_term->id(),
+    ]);
+    $child_term->addTranslation('de', [
+      'name' => 'german-child-term',
+    ])->save();
+
+    // Expect the parent term to be in the specified language.
+    $this->assertTokens('term', ['term' => $child_term], ['parents' => 'german-parent-term'], ['langcode' => 'de']);
+    $this->assertTokens('term', ['term' => $child_term], ['root' => 'german-parent-term'], ['langcode' => 'de']);
+  }
+
+}

modules/token/tests/src/Kernel/TokenKernelTestBase.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\KernelTests\KernelTestBase;
+use Drupal\Tests\token\Functional\TokenTestTrait;
+
+/**
+ * Helper test class with some added functions for testing.
+ */
+abstract class TokenKernelTestBase extends KernelTestBase {
+
+  use TokenTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'path',
+    'token',
+    'token_module_test',
+    'system',
+    'user',
+    'path_alias',
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->installEntitySchema('path_alias');
+    \Drupal::service('router.builder')->rebuild();
+    $this->installConfig(['system']);
+  }
+
+}

modules/token/tests/src/Kernel/UnitTest.php

@@ -0,0 +1,118 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+/**
+ * Test basic, low-level token functions.
+ *
+ * @group token
+ */
+class UnitTest extends TokenKernelTestBase {
+
+  /**
+   * @var \Drupal\token\Token
+   */
+  protected $tokenService;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['file', 'node'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->tokenService = \Drupal::token();
+  }
+
+  /**
+   * Test invalid tokens.
+   */
+  public function testGetInvalidTokens() {
+    $tests = [];
+    $tests[] = [
+      'valid tokens' => [
+        '[node:title]',
+        '[node:created:short]',
+        '[node:created:custom:invalid]',
+        '[node:created:custom:mm-YYYY]',
+        '[node:colons:in:name]',
+        '[site:name]',
+        '[site:slogan]',
+        '[current-date:short]',
+        '[current-user:uid]',
+        '[current-user:ip-address]',
+      ],
+      'invalid tokens' => [
+        '[node:title:invalid]',
+        '[node:created:invalid]',
+        '[node:created:short:invalid]',
+        '[node:colons:in:name:invalid]',
+        '[invalid:title]',
+        '[site:invalid]',
+        '[user:ip-address]',
+        '[user:uid]',
+        '[comment:cid]',
+        // Deprecated tokens
+        '[node:tnid]',
+        '[node:type]',
+        '[node:type-name]',
+        '[date:short]',
+      ],
+      'types' => ['node'],
+    ];
+    $tests[] = [
+      'valid tokens' => [
+        '[node:title]',
+        '[node:created:short]',
+        '[node:created:custom:invalid]',
+        '[node:created:custom:mm-YYYY]',
+        '[node:colons:in:name]',
+        '[site:name]',
+        '[site:slogan]',
+        '[user:uid]',
+        '[current-date:short]',
+        '[current-user:uid]',
+      ],
+      'invalid tokens' => [
+        '[node:title:invalid]',
+        '[node:created:invalid]',
+        '[node:created:short:invalid]',
+        '[node:colons:in:name:invalid]',
+        '[invalid:title]',
+        '[site:invalid]',
+        '[user:ip-address]',
+        '[comment:cid]',
+        // Deprecated tokens
+        '[node:tnid]',
+        '[node:type]',
+        '[node:type-name]',
+      ],
+      'types' => ['all'],
+    ];
+
+    foreach ($tests as $test) {
+      $tokens = array_merge($test['valid tokens'], $test['invalid tokens']);
+      shuffle($tokens);
+
+      $invalid_tokens = $this->tokenService->getInvalidTokensByContext(implode(' ', $tokens), $test['types']);
+
+      sort($invalid_tokens);
+      sort($test['invalid tokens']);
+      $this->assertEquals($test['invalid tokens'], $invalid_tokens, 'Invalid tokens detected properly: ' . implode(', ', $invalid_tokens));
+    }
+  }
+
+  /**
+   * Test that tokens are generated only for content entities.
+   */
+  public function testContentEntityOnlyTokens() {
+    // Verify that type and token info for a config entity is not generated.
+    $this->assertNull($this->tokenService->getTokenInfo('user_role', 'original'));
+    $this->assertNull($this->tokenService->getTokenInfo('user_role', 'url'));
+    $this->assertNull($this->tokenService->getTypeInfo('user_role'));
+  }
+
+}

modules/token/tests/src/Kernel/UrlTest.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Core\Url;
+use Symfony\Component\HttpFoundation\Request;
+
+/**
+ * Test generic url token replacements.
+ *
+ * @group token
+ */
+class UrlTest extends TokenKernelTestBase {
+
+  /**
+   * The token service.
+   *
+   * @var \Drupal\Core\Utility\Token
+   */
+  protected $token;
+
+  /**
+   * The current request stack.
+   *
+   * @var \Symfony\Component\HttpFoundation\RequestStack
+   */
+  protected $requestStack;
+
+  /**
+   * The current route match.
+   *
+   * @var \Drupal\Core\Routing\CurrentRouteMatch
+   */
+  protected $currentRouteMatch;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+    $this->token = $this->container->get('token');
+    $this->requestStack = $this->container->get('request_stack');
+    $this->currentRouteMatch = $this->container->get('current_route_match');
+  }
+
+  /**
+   * Test the url token replacements for current requests.
+   *
+   * The method ::expectedCurrentRequestUrlResults() is not declared
+   * as a regular data provider, because it might use services from
+   * the global Drupal container, which is not initialized yet during
+   * the invocation of data providers.
+   */
+  public function testCurrentRequestUrls() {
+    foreach ($this->expectedCurrentRequestUrlResults() as $data_set) {
+      list ($request, $text, $data, $options, $expected_output) = $data_set;
+      // Set the request as the current one.
+      $this->requestStack->push($request);
+      $this->currentRouteMatch->resetRouteMatch();
+
+      $this->assertEquals($expected_output, $this->token->replace($text, $data, $options));
+      $this->requestStack->pop();
+    }
+  }
+
+  /**
+   * Provides a list of results to expect for ::testRequestUrls().
+   *
+   * Each data set of this array holds the following order:
+   *   - The request object to test for.
+   *   - The input text as string.
+   *   - The token data as array.
+   *   - Further options for the token replacement as array.
+   *   - The output to expect after token replacement.
+   *
+   * @return array
+   *   The list of results to expect.
+   */
+  public function expectedCurrentRequestUrlResults() {
+    return [
+      [Request::createFromGlobals(), '[current-page:url]', [], [], Url::createFromRequest(Request::createFromGlobals())->setAbsolute()->toString()],
+      [Request::create('/should-not-exist'), '[current-page:url:path]', [], [], '/'],
+      [Request::create('/https://drupal.org/'), '[current-page:url:absolute]', [], [], '[current-page:url:absolute]'],
+      [Request::create('/https://drupal.org/'), '[current-page:url:absolute]', [], ['clear' => TRUE], ''],
+    ];
+  }
+
+}

modules/token/tests/src/Kernel/ValidateD6MigrationStateTest.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Tests\migrate_drupal\Kernel\d6\MigrateDrupal6TestBase;
+use Drupal\Tests\migrate_drupal\Traits\ValidateMigrationStateTestTrait;
+
+/**
+ * Tests that the token test has a declared D6 migration status.
+ *
+ * ValidateMigrationStateTestTrait::testMigrationState() will succeed if the
+ * modules enabled in \Drupal\Tests\KernelTestBase::bootKernel() have a valid
+ * migration status (i.e.: finished or not_finished); but will fail if they do
+ * not have a declared migration status.
+ *
+ * @group token
+ */
+class ValidateD6MigrationStateTest extends MigrateDrupal6TestBase {
+  use ValidateMigrationStateTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['token'];
+
+}

modules/token/tests/src/Kernel/ValidateD7MigrationStateTest.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\Tests\migrate_drupal\Kernel\d7\MigrateDrupal7TestBase;
+use Drupal\Tests\migrate_drupal\Traits\ValidateMigrationStateTestTrait;
+
+/**
+ * Tests that the token test has a declared D7 migration status.
+ *
+ * ValidateMigrationStateTestTrait::testMigrationState() will succeed if the
+ * modules enabled in \Drupal\Tests\KernelTestBase::bootKernel() have a valid
+ * migration status (i.e.: finished or not_finished); but will fail if they do
+ * not have a declared migration status.
+ *
+ * @group token
+ */
+class ValidateD7MigrationStateTest extends MigrateDrupal7TestBase {
+  use ValidateMigrationStateTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['token'];
+
+}

modules/token/tests/src/Kernel/ViewsTest.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Drupal\Tests\token\Kernel;
+
+use Drupal\views\Tests\ViewTestData;
+use Drupal\views\Views;
+
+/**
+ * Test the views tokens.
+ *
+ * @group token
+ */
+class ViewsTest extends TokenKernelTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = ['views', 'block'];
+
+  /**
+   * Views used by this test.
+   *
+   * @var array
+   */
+  public static $testViews = ['token_views_test'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+    $this->installEntitySchema('user');
+    ViewTestData::createTestViews(get_class($this), ['token_module_test']);
+  }
+
+  /**
+   * Tests path token replacements generated from a view without a path.
+   */
+  public function testTokenReplacementNoPath() {
+    $token_handler = \Drupal::token();
+    $view = Views::getView('token_views_test');
+    $view->setDisplay('block_1');
+    $view->execute();
+
+    $this->assertSame('', $token_handler->replace('[view:url]', ['view' => $view]), 'Token [view:url] is empty for views without path.');
+  }
+
+}