first commit

core/scripts/js/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "rules": {
+    "strict": [2, "global"]
+  }
+}

core/scripts/js/assets/ckeditor5Files.js

@@ -0,0 +1,65 @@
+/**
+ * @file
+ * Callback returning the list of files to copy to the assets/vendor directory.
+ */
+const { globSync } = require('glob');
+// There are a lot of CKEditor 5 packages, generate the list dynamically.
+// Drupal-specific mapping between CKEditor 5 name and Drupal library name.
+const ckeditor5PluginMapping = {
+  'block-quote': 'blockquote',
+  essentials: 'internal',
+  'basic-styles': 'basic',
+};
+
+/**
+ * Build the list of assets to be copied based on what exists in the filesystem.
+ *
+ * @param {string} packageFolder
+ *   The path to node_modules folder.
+ *
+ * @return {DrupalLibraryAsset[]}
+ *  List of libraries and files to process.
+ */
+module.exports = (packageFolder) => {
+  const fileList = [];
+  // Get all the CKEditor 5 packages.
+  const ckeditor5Dirs = globSync(`{${packageFolder}/@ckeditor/ckeditor5*,${packageFolder}/ckeditor5}`).sort();
+  for (const ckeditor5package of ckeditor5Dirs) {
+    // Add all the files in the build/ directory to the process array for
+    // copying.
+    const buildFiles = globSync(`${ckeditor5package}/build/**/*.js`, {
+      nodir: true,
+    });
+    if (buildFiles.length) {
+      // Clean up the path to get the original package name.
+      const pack = ckeditor5package.replace(`${packageFolder}/`, '');
+      // Use the package name to generate the plugin name. There are some
+      // exceptions that needs to be handled. Ideally remove the special cases.
+      let pluginName = pack.replace('@ckeditor/ckeditor5-', '');
+      // Target folder in the vendor/assets folder.
+      let folder = `ckeditor5/${pluginName.replace('@ckeditor/ckeditor5-', '')}`;
+      // Transform kebab-case to CamelCase.
+      let library = pluginName.replace(/-./g, (match) => match[1].toUpperCase());
+      // Special case for Drupal implementation.
+      if (ckeditor5PluginMapping.hasOwnProperty(pluginName)) {
+        library = ckeditor5PluginMapping[pluginName];
+      }
+      if (library === 'ckeditor5') {
+        folder = 'ckeditor5/ckeditor5-dll';
+      } else {
+        library = `ckeditor5.${library}`;
+      }
+      fileList.push({
+        pack,
+        library,
+        folder,
+        files: buildFiles.map((absolutePath) => ({
+          from: absolutePath.replace(`${ckeditor5package}/`, ''),
+          to: absolutePath.replace(`${ckeditor5package}/build/`, ''),
+        })),
+      });
+    }
+  }
+
+  return fileList;
+};

core/scripts/js/assets/process/jqueryui.js

@@ -0,0 +1,47 @@
+const Terser = require('terser');
+const path = require('path');
+
+/**
+ * Process jQuery UI source files.
+ *
+ * Each file being processed creates 3 files under assets/vendor/jquery.ui/:
+ *  - The original source for audit purposes, with a `.js` suffix.
+ *  - The minified version for production use, with a `-min.js` suffix.
+ *  - The source map for debugging purposes, with a `-min.js.map` suffix.
+ *
+ * @param {object} data
+ *  Object passed to the callback.
+ * @param {object} data.file
+ *  Normalized file information object.
+ * @param {string} data.file.from
+ *  Path of the file in node_modules/ directory.
+ * @param {string} data.file.to
+ *  Path of the file in core assets/vendor/ directory.
+ * @param {string} data.contents
+ *  Content of the file being processed.
+ *
+ * @return {Promise<[{filename: string, contents: string}]>}
+ *  Return a Promise that resolves into an array of file and content to create
+ *  in the assets/vendor/ directory.
+ */
+module.exports = async ({ file: { from, to }, contents }) => {
+  const filename = `${to.slice(0, -3)}-min.js`;
+  const sourcemap = `${filename}.map`;
+
+  const { code, map } = await Terser.minify(
+    { [path.basename(from)]: contents }, {
+    sourceMap: {
+      filename: path.basename(filename),
+      url: path.basename(sourcemap),
+    },
+  });
+
+  return [
+    // Original file.
+    { filename: to, contents },
+    // Minified file.
+    { filename, contents: code },
+    // Sourcemap file.
+    { filename: sourcemap, contents: map },
+  ];
+};

core/scripts/js/assets/process/map.js

@@ -0,0 +1,20 @@
+/**
+ * Process map files.
+ *
+ * In the `sources` member, remove all "../" values at the start of the file
+ * names to avoid virtual files located outside of the library vendor folder.
+ *
+ * @param {object} data
+ *  Object passed to the callback.
+ * @param {string} data.contents
+ *  Content of the file being processed.
+ *
+ * @return {Promise<[{contents: string}]>}
+ *  Return a Promise that resolves into an array of file and content to create
+ *  in the assets/vendor/ directory.
+ */
+module.exports = ({ contents }) => {
+  const json = JSON.parse(contents);
+  json.sources = json.sources.map((source) => source.replace(/^(\.\.\/)+/, ''));
+  return [{ contents: JSON.stringify(json) }];
+};

core/scripts/js/ckeditor5-check-plugins.js

@@ -0,0 +1,39 @@
+/**
+ * @file
+ *
+ * Provides the `check:ckeditor5` command.
+ *
+ * Check that the plugins are built with the appropriate dependencies. This is
+ * only run on DrupalCI.
+ *
+ * @internal This file is part of the core JavaScript build process and is only
+ * meant to be used in that context.
+ */
+
+"use strict";
+
+const { globSync } = require("glob");
+const log = require("./log");
+const fs = require("fs").promises;
+const child_process = require("child_process");
+
+async function getContents(files) {
+  return Object.fromEntries(
+    await Promise.all(
+      files.map(async (file) => [file, (await fs.readFile(file)).toString()])
+    )
+  );
+}
+
+(async () => {
+  const files = globSync("./modules/ckeditor5/js/build/*.js").sort();
+
+  const pluginsBefore = await getContents(files);
+  // Execute the plugin build script.
+  child_process.execSync("yarn run build:ckeditor5");
+  const pluginsAfter = await getContents(files);
+
+  if (JSON.stringify(pluginsBefore) !== JSON.stringify(pluginsAfter)) {
+    process.exitCode = 1;
+  }
+})();

core/scripts/js/ckeditor5-types-documentation.js

@@ -0,0 +1,117 @@
+/**
+ * @file
+ *
+ * Provides the `build:ckeditor5-types` command.
+ *
+ * This command is used for generating documentation for mapping CKEditor 5
+ * types so that they can be parsed by IDEs.
+ *
+ * @internal This file is part of the core javascript build process and is only
+ * meant to be used in that context.
+ */
+
+'use strict';
+
+const { globSync } = require('glob');
+const log = require('./log');
+const fs = require('fs');
+
+/**
+ * A list of regex used to alias CKEditor 5 types.
+ *
+ * @type {RegExp[]}
+ */
+const regexList = [
+  // Makes sure that `export default class` code can be referenced with the
+  // class name and not the module name only.
+  / * @module \b(.*)\b[\s\S]*?export default(?: class| function)? \b(\w+)\b/g,
+
+  // Pick up CKEditor 5 own aliases to alias them too.
+  / * @module \b(.*)\b[\s\S]*?@(?:typedef|interface) (?:.*~)?(\w+)/g,
+];
+
+const globOptions = {
+  // Search within the ckeditor npm namespace.
+  cwd: process.cwd() + '/node_modules/@ckeditor/',
+  absolute: true,
+};
+
+/**
+ * Template for the generated typedef comment.
+ *
+ * @param {string} file
+ *  The path to the file containing the type definition.
+ * @param {string} module
+ *  The module name as defined by the @module jsdoc comment.
+ * @param {string} name
+ *  The name of the class being exported
+ *
+ * @return {string}
+ *  The comment aliasing the module name to the specific named exports.
+ */
+function generateTypeDef(file, module, name) {
+  const cleanModule = module.replace('module:', '');
+  return `/**
+ * Declared in file @ckeditor/${file.replace(globOptions.cwd, '')}
+ *
+ * @typedef {module:${cleanModule}} module:${cleanModule}~${name}
+ */
+`;
+}
+
+
+/**
+ * Helper to get the file contents as a string.
+ *
+ * @param {string} filePath
+ *  Absolute path to the file.
+ *
+ * @return {string}
+ */
+function getFile(filePath) {
+  try {
+    return fs.readFileSync(filePath, 'utf8');
+  } catch (err) {
+    return '';
+  }
+}
+
+/**
+ * Returns a callback function.
+ *
+ * @param {string} filePath
+ *  The CKEditor 5 source file to inspect for exports or type definitions.
+ *
+ * @return {function}
+ *  The aliased typedef string.
+ *
+ * @see generateTypeDef
+ */
+function processFile(filePath) {
+  const fileData = getFile(filePath);
+  // Use a for loop to be able to return early.
+  for (const regex of regexList) {
+    // Reset the match index of the Regex to make sure we search from the
+    // beginning of the file every time.
+    regex.lastIndex = 0;
+    const m = regex.exec(fileData);
+    if (m) {
+      return generateTypeDef(filePath, m[1], m[2]);
+    }
+  }
+  return false;
+}
+
+const definitions = globSync('./ckeditor5*/src/**/*.+(js|jsdoc)', globOptions).sort().map(processFile);
+// Filter definitions that do not match any regex.
+const existingDefinitions = definitions.filter((e) => !!e);
+
+// Write the file in the ckeditor module, use the JSDoc extension to make sure
+// the JSDoc extension is associated with the JavaScript file type and it
+// prevents core JavaScript lint rules to be run. Add it to the build folder to
+// prevent cspell checks on this file.
+fs.writeFile(`./modules/ckeditor5/js/build/ckeditor5.types.jsdoc`, existingDefinitions.join('\n'), () => {
+  log(`CKEditor 5 types have been generated: ${existingDefinitions.length} declarations aliased, ${definitions.length - existingDefinitions.length} files ignored`);
+});
+
+process.exitCode = 0;

core/scripts/js/eslint-stats-by-type.js

@@ -0,0 +1,35 @@
+module.exports = function (results) {
+  results = results || [];
+
+  const errorType = {
+    warnings: {},
+    errors: {},
+  };
+
+  results.reduce((result, current) => {
+    current.messages.forEach((msg) => {
+      if (msg.severity === 1) {
+        errorType.warnings[msg.ruleId] = errorType.warnings[msg.ruleId] + 1 || 1
+      }
+      if (msg.severity === 2) {
+        errorType.errors[msg.ruleId] = errorType.errors[msg.ruleId] + 1 || 1
+      }
+    });
+    return result;
+  });
+
+  const reduceErrorCounts = (errorType) => Object.entries(errorType).sort((a, b) => b[1] - a[1])
+    .reduce((result, current) => result.concat([`${current[0]}: ${current[1]}`]), []).join('\n');
+
+  const warnings = reduceErrorCounts(errorType.warnings);
+  const errors = reduceErrorCounts(errorType.errors);
+
+  return `
+Errors:
+${'='.repeat(30)}
+${errors}
+${'\n'.repeat(4)}
+Warnings:
+${'='.repeat(30)}
+${warnings}`;
+};

core/scripts/js/log.js

@@ -0,0 +1,4 @@
+module.exports = (message) => {
+  // Logging human-readable timestamp.
+  console.log(`[${new Date().toTimeString().slice(0, 8)}] ${message}`);
+}

core/scripts/js/vendor-update.js

@@ -0,0 +1,278 @@
+/**
+ * @file
+ * Copy files for JS vendor dependencies from node_modules to the assets/vendor
+ * folder.
+ *
+ * This script handles all dependencies except CKEditor and Modernizr, which
+ * require a custom build step.
+ */
+
+const path = require('path');
+const { copyFile, writeFile, readFile, chmod, mkdir } = require('fs').promises;
+const ckeditor5Files = require('./assets/ckeditor5Files');
+const jQueryUIProcess = require('./assets/process/jqueryui');
+const mapProcess = require('./assets/process/map');
+
+const coreFolder = path.resolve(__dirname, '../../');
+const packageFolder = `${coreFolder}/node_modules`;
+const assetsFolder = `${coreFolder}/assets/vendor`;
+
+(async () => {
+  const librariesPath = `${coreFolder}/core.libraries.yml`;
+  // Open the core.libraries.yml file to update version information
+  // automatically.
+  const libraries = (await readFile(librariesPath)).toString().split('\n\n');
+
+  function updateLibraryVersion(libraryName, { version }) {
+    const libraryIndex = libraries.findIndex((lib) =>
+      lib.startsWith(libraryName),
+    );
+    if (libraryIndex > 0) {
+      const libraryDeclaration = libraries[libraryIndex];
+      // Get the previous package version from the yaml file, versions can be
+      // declared with a yaml anchor such as `version: &yaml_anchor "xxx"`
+      const currentVersion = libraryDeclaration.match(/version:(?: [&\w_]+)? "(.*)"\n/)[1];
+      // Replace the version value and the version in the license URL.
+      libraries[libraryIndex] = libraryDeclaration.replace(
+        new RegExp(currentVersion, 'g'),
+        version,
+      );
+    }
+  }
+
+  /**
+   * Structure of the object defining a library to copy to the assets/ folder.
+   *
+   * @typedef DrupalLibraryAsset
+   *
+   * @prop {string} pack
+   *   The name of the npm package (used to get the name of the folder where
+   *   the files are situated inside of the node_modules folder). Note that we
+   *   use `pack` and not `package` because `package` is a future reserved word.
+   * @prop {string} [folder]
+   *   The folder under `assets/vendor/` where the files will be copied. If
+   *   this
+   *   is not defined the value of `pack` is used.
+   * @prop {string} [library]
+   *   The key under which the library is declared in core.libraries.yml.
+   * @prop {Array} [files]
+   *   An array of files to be copied over.
+   *     - A string if the file has the same name and is at the same level in
+   *   the source and target folder.
+   *     - An object with a `from` and `to` property if the source and target
+   *   have a different name or if the folder nesting is different.
+   * @prop {object} [process]
+   *   An object containing a file extension as a key and a callback as the
+   *   value. The callback will be called for each file matching the file
+   *   extension. It can be used to minify the file content before saving to
+   *   the target directory.
+   */
+
+  /**
+   * Declare the array that defines what needs to be copied over.
+   *
+   * @type {DrupalLibraryAsset[]}
+   */
+  const ASSET_LIST = [
+    {
+      pack: 'backbone',
+      library: 'internal.backbone',
+      files: ['backbone.js', 'backbone-min.js', 'backbone-min.js.map'],
+    },
+    {
+      pack: 'jquery',
+      files: [
+        { from: 'dist/jquery.js', to: 'jquery.js' },
+        { from: 'dist/jquery.min.js', to: 'jquery.min.js' },
+        { from: 'dist/jquery.min.map', to: 'jquery.min.map' },
+      ],
+    },
+    {
+      pack: 'js-cookie',
+      files: [{ from: 'dist/js.cookie.min.js', to: 'js.cookie.min.js' }],
+    },
+    {
+      pack: 'normalize.css',
+      folder: 'normalize-css',
+      library: 'normalize',
+      files: ['normalize.css'],
+    },
+    {
+      pack: '@drupal/once',
+      folder: 'once',
+      files: [
+        { from: 'dist/once.js', to: 'once.js' },
+        { from: 'dist/once.min.js', to: 'once.min.js' },
+        { from: 'dist/once.min.js.map', to: 'once.min.js.map' },
+      ],
+    },
+    {
+      pack: 'shepherd.js',
+      folder: 'shepherd',
+      library: 'internal.shepherd',
+      files: [
+        { from: 'dist/js/shepherd.min.js', to: 'shepherd.min.js' },
+        { from: 'dist/js/shepherd.min.js.map', to: 'shepherd.min.js.map' },
+      ],
+    },
+    { pack: 'sortablejs', folder: 'sortable', files: ['Sortable.min.js'] },
+    {
+      pack: 'tabbable',
+      files: [
+        { from: 'dist/index.umd.min.js', to: 'index.umd.min.js' },
+        { from: 'dist/index.umd.min.js.map', to: 'index.umd.min.js.map' },
+      ],
+    },
+    {
+      pack: 'underscore',
+      library: 'internal.underscore',
+      files: ['underscore-min.js', 'underscore-min.js.map'],
+    },
+    {
+      pack: 'loadjs',
+      files: [{ from: 'dist/loadjs.min.js', to: 'loadjs.min.js' }],
+    },
+    {
+      pack: 'tua-body-scroll-lock',
+      files: [
+        { from: 'dist/tua-bsl.umd.min.js', to: 'tua-bsl.umd.min.js' },
+      ],
+    },
+    {
+      pack: 'transliteration',
+      files: [
+        { from: 'dist/browser/bundle.umd.min.js', to: 'bundle.umd.min.js' },
+        { from: 'dist/browser/bundle.umd.min.js.map', to: 'bundle.umd.min.js.map' },
+      ],
+    },
+    {
+      pack: 'jquery-ui',
+      folder: 'jquery.ui',
+      process: {
+        // This will automatically minify the files and update the destination
+        // filename before saving.
+        '.js': jQueryUIProcess,
+      },
+      files: [
+        'themes/base/autocomplete.css',
+        'themes/base/button.css',
+        'themes/base/checkboxradio.css',
+        'themes/base/controlgroup.css',
+        'themes/base/core.css',
+        'themes/base/dialog.css',
+        'themes/base/draggable.css',
+        'themes/base/images/ui-bg_flat_0_aaaaaa_40x100.png',
+        'themes/base/images/ui-icons_444444_256x240.png',
+        'themes/base/images/ui-icons_555555_256x240.png',
+        'themes/base/images/ui-icons_777620_256x240.png',
+        'themes/base/images/ui-icons_777777_256x240.png',
+        'themes/base/images/ui-icons_cc0000_256x240.png',
+        'themes/base/images/ui-icons_ffffff_256x240.png',
+        'themes/base/menu.css',
+        'themes/base/resizable.css',
+        'themes/base/theme.css',
+        'ui/data.js',
+        'ui/disable-selection.js',
+        'ui/focusable.js',
+        'ui/form-reset-mixin.js',
+        'ui/form.js',
+        'ui/ie.js',
+        'ui/jquery-patch.js',
+        'ui/keycode.js',
+        'ui/labels.js',
+        'ui/plugin.js',
+        'ui/safe-active-element.js',
+        'ui/safe-blur.js',
+        'ui/scroll-parent.js',
+        'ui/unique-id.js',
+        'ui/version.js',
+        'ui/widget.js',
+        'ui/widgets/autocomplete.js',
+        'ui/widgets/button.js',
+        'ui/widgets/checkboxradio.js',
+        'ui/widgets/controlgroup.js',
+        'ui/widgets/dialog.js',
+        'ui/widgets/draggable.js',
+        'ui/widgets/menu.js',
+        'ui/widgets/mouse.js',
+        'ui/widgets/resizable.js',
+      ],
+    },
+    // CKEditor 5 builds the list of files dynamically based on what exists
+    // in the filesystem.
+    ...ckeditor5Files(packageFolder),
+  ];
+
+  /**
+   * Default callback for processing map files.
+   */
+  const defaultProcessCallbacks = {
+    '.map': mapProcess,
+  };
+
+  /**
+   * Return an object with a 'from' and 'to' member.
+   *
+   * @param {string|object} file
+   *
+   * @return {{from: string, to: string}}
+   */
+  function normalizeFile(file) {
+    let normalized = file;
+    if (typeof file === 'string') {
+      normalized = {
+        from: file,
+        to: file,
+      };
+    }
+    return normalized;
+  }
+
+  for (const { pack, files = [], folder = false, library = false, process = {} } of ASSET_LIST) {
+    const sourceFolder = pack;
+    const libraryName = library || folder || pack;
+    const destFolder = folder || pack;
+    // Add a callback for map files by default.
+    const processCallbacks = { ...defaultProcessCallbacks, ...process };
+
+    // Update the library version in core.libraries.yml with the version
+    // from the npm package.
+    try {
+      const packageInfo = JSON.parse((await readFile(`${packageFolder}/${sourceFolder}/package.json`)).toString());
+      updateLibraryVersion(libraryName, packageInfo);
+    } catch (e) {
+      // The package.json file doesn't exist, so nothing to do.
+    }
+
+    for (const file of files.map(normalizeFile)) {
+      const sourceFile = `${packageFolder}/${sourceFolder}/${file.from}`;
+      const destFile = `${assetsFolder}/${destFolder}/${file.to}`;
+      const extension = path.extname(file.from);
+
+      try {
+        await mkdir(path.dirname(destFile), { recursive: true });
+      } catch (e) {
+        // Nothing to do if the folder already exists.
+      }
+
+      // There is a callback that transforms the file contents, we are not
+      // simply copying a file from A to B.
+      if (processCallbacks[extension]) {
+        const contents = (await readFile(sourceFile)).toString();
+        const results = await processCallbacks[extension]({ file, contents });
+
+        console.log(`Process ${sourceFolder}/${file.from} and save ${results.length} files:\n  ${results.map(({ filename = file.to }) => filename).join(', ')}`);
+        for (const { filename = file.to, contents } of results) {
+          // The filename key can be used to change the name of the saved file.
+          await writeFile(`${assetsFolder}/${destFolder}/${filename}`, contents);
+        }
+      } else {
+        // There is no callback simply copy the file.
+        console.log(`Copy ${sourceFolder}/${file.from} to ${destFolder}/${file.to}`);
+        await copyFile(sourceFile, destFile);
+      }
+    }
+  }
+
+  await writeFile(librariesPath, libraries.join('\n\n'));
+})();