Improved url-utils performance for sitemap-scale workloads

packages/url-utils/README.md

@@ -11,6 +11,22 @@ or
 
 ## Usage
 
+### Freezing URLs
+
+When the site, subdirectory and admin URLs never change at runtime (e.g. in production), mark them as frozen so url-utils can skip calling the URL getters:
+
+```js
+const urlUtils = new UrlUtils({getSiteUrl, getSubdir, getAdminUrl, frozen: true});
+
+// or at any time after creation
+urlUtils.freeze();
+
+// restore the original getters (e.g. after changing config in tests)
+urlUtils.unfreeze();
+```
+
+While frozen, `getSiteUrl()`, `getSubdir()` and `getAdminUrl()` return the values captured at freeze time. Calling `freeze()` again re-captures the current values.
+
 
 ## Develop
 

packages/url-utils/package.json

@@ -39,6 +39,7 @@
   "dependencies": {
     "cheerio": "1.2.0",
     "lodash": "4.18.1",
+    "lru-cache": "^11.0.0",
     "moment": "2.31.0",
     "moment-timezone": "0.6.4",
     "remark": "11.0.2",

packages/url-utils/src/UrlUtils.ts

@@ -14,7 +14,12 @@ import type {RelativeToAbsoluteOptionsInput} from './utils/relative-to-absolute'
 import type {AbsoluteToTransformReadyOptionsInput as AbsoluteToTransformReadyOptionsInputType} from './utils/absolute-to-transform-ready';
 import type {RelativeToTransformReadyOptionsInput as RelativeToTransformReadyOptionsInputType} from './utils/relative-to-transform-ready';
 import type {ToTransformReadyOptions} from './utils/to-transform-ready';
-import type {TransformReadyToAbsoluteOptionsInput} from './utils/transform-ready-to-absolute';
+import {
+    DEFAULT_OPTIONS as TRANSFORM_READY_TO_ABSOLUTE_DEFAULTS,
+    replaceTransformReadyPlaceholders,
+    type TransformReadyToAbsoluteOptions,
+    type TransformReadyToAbsoluteOptionsInput
+} from './utils/transform-ready-to-absolute';
 import type {TransformReadyReplacementOptionsInput as TransformReadyToRelativeOptionsInput} from './utils/types';
 
 interface ExpressResponse {
@@ -60,6 +65,19 @@ interface UrlUtilsOptions {
         media?: string | null;
     };
     cardTransformers?: MobiledocCardTransformer[];
+    frozen?: boolean;
+}
+
+type AssetOptionDefaults = BaseUrlOptionsInput & {
+    staticImageUrlPrefix: string;
+    staticFilesUrlPrefix: string;
+    staticMediaUrlPrefix: string;
+};
+
+interface UrlGetters {
+    getSubdir: () => string;
+    getSiteUrl: () => string;
+    getAdminUrl: () => string;
 }
 
 // similar to Object.assign but will not override defaults if a source value is undefined
@@ -78,6 +96,10 @@ export default class UrlUtils {
     public getSubdir: () => string;
     public getSiteUrl: () => string;
     public getAdminUrl: () => string;
+    private _unfrozenGetters: UrlGetters | null = null;
+    // asset options are fixed at construction so are only built once
+    private readonly _assetOptionDefaults: Readonly<AssetOptionDefaults>;
+    private readonly _transformReadyToAbsoluteDefaults: Readonly<TransformReadyToAbsoluteOptions>;
 
     /**
      * Initialization method to pass in URL configurations
@@ -96,6 +118,7 @@ export default class UrlUtils {
      * @param {string} [options.assetBaseUrls.image] image asset CDN base URL
      * @param {string} [options.assetBaseUrls.files] files asset CDN base URL
      * @param {string} [options.assetBaseUrls.media] media asset CDN base URL
+     * @param {boolean} [options.frozen=false] freeze url getters on creation, see `freeze()`
      */
     constructor(options: UrlUtilsOptions = {}) {
         const defaultOptions: UrlUtilsConfig = {
@@ -117,28 +140,81 @@ export default class UrlUtils {
             media: assetBaseUrls.media || null
         };
 
+        this._assetOptionDefaults = Object.freeze({
+            staticImageUrlPrefix: this._config.staticImageUrlPrefix,
+            staticFilesUrlPrefix: this._config.staticFilesUrlPrefix,
+            staticMediaUrlPrefix: this._config.staticMediaUrlPrefix,
+            imageBaseUrl: this._assetBaseUrls.image,
+            filesBaseUrl: this._assetBaseUrls.files,
+            mediaBaseUrl: this._assetBaseUrls.media
+        });
+        this._transformReadyToAbsoluteDefaults = Object.freeze(
+            Object.assign({}, TRANSFORM_READY_TO_ABSOLUTE_DEFAULTS, this._assetOptionDefaults)
+        );
+
         this.getSubdir = options.getSubdir || (() => '');
         this.getSiteUrl = options.getSiteUrl || (() => '');
         this.getAdminUrl = options.getAdminUrl || (() => '');
+
+        if (options.frozen) {
+            this.freeze();
+        }
     }
 
-    private _assetOptionDefaults(): BaseUrlOptionsInput & {
-        staticImageUrlPrefix: string;
-        staticFilesUrlPrefix: string;
-        staticMediaUrlPrefix: string;
-        } {
-        return {
-            staticImageUrlPrefix: this._config.staticImageUrlPrefix,
-            staticFilesUrlPrefix: this._config.staticFilesUrlPrefix,
-            staticMediaUrlPrefix: this._config.staticMediaUrlPrefix,
-            imageBaseUrl: this._assetBaseUrls.image || null,
-            filesBaseUrl: this._assetBaseUrls.files || null,
-            mediaBaseUrl: this._assetBaseUrls.media || null
+    /**
+     * Mark the site, subdirectory and admin URLs as frozen: they won't change for
+     * the lifetime of this instance (or until `unfreeze()` is called).
+     *
+     * While frozen, `getSubdir`, `getSiteUrl` and `getAdminUrl` return a snapshot
+     * taken at freeze time rather than calling the configured getters.
+     *
+     * Only freeze when the underlying config is static, e.g. in production. If the
+     * URLs can change at runtime (tests that swap config) leave unfrozen, or call
+     * `unfreeze()`/`freeze()` again after changing config.
+     */
+    freeze(): this {
+        if (this._unfrozenGetters) {
+            this.unfreeze();
+        }
+
+        const getters: UrlGetters = {
+            getSubdir: this.getSubdir,
+            getSiteUrl: this.getSiteUrl,
+            getAdminUrl: this.getAdminUrl
         };
+
+        const subdir = getters.getSubdir();
+        const siteUrl = getters.getSiteUrl();
+        const adminUrl = getters.getAdminUrl();
+
+        this._unfrozenGetters = getters;
+        this.getSubdir = () => subdir;
+        this.getSiteUrl = () => siteUrl;
+        this.getAdminUrl = () => adminUrl;
+
+        return this;
+    }
+
+    /**
+     * Restore the original URL getters.
+     */
+    unfreeze(): this {
+        if (this._unfrozenGetters) {
+            this.getSubdir = this._unfrozenGetters.getSubdir;
+            this.getSiteUrl = this._unfrozenGetters.getSiteUrl;
+            this.getAdminUrl = this._unfrozenGetters.getAdminUrl;
+            this._unfrozenGetters = null;
+        }
+
+        return this;
+    }
+
+    get isFrozen(): boolean {
+        return this._unfrozenGetters !== null;
     }
 
     private _buildAssetOptions(additionalDefaults: Record<string, unknown> = {}, options?: Record<string, unknown>): Record<string, unknown> {
-        return assignOptions({}, this._assetOptionDefaults(), additionalDefaults, options || {});
+        return assignOptions({}, this._assetOptionDefaults, additionalDefaults, options || {});
     }
 
     getProtectedSlugs(): string[] {
@@ -362,8 +438,23 @@ export default class UrlUtils {
     }
 
     transformReadyToAbsolute(url: string, options?: TransformReadyToAbsoluteOptionsInput): string {
-        const _options = this._buildAssetOptions({}, options) as TransformReadyToAbsoluteOptionsInput;
-        return utils.transformReadyToAbsolute(url, this.getSiteUrl(), _options);
+        if (options) {
+            const _options = this._buildAssetOptions({}, options) as TransformReadyToAbsoluteOptionsInput;
+            return utils.transformReadyToAbsolute(url, this.getSiteUrl(), _options);
+        }
+
+        // hot path (called for every post url and image when rendering e.g. sitemaps),
+        // skip option merging and the site url lookup when there's nothing to replace
+        if (!url) {
+            // matches the util, which defaults a missing url to ''
+            return url === undefined ? '' : url;
+        }
+
+        if (!url.includes(TRANSFORM_READY_TO_ABSOLUTE_DEFAULTS.replacementStr)) {
+            return url;
+        }
+
+        return replaceTransformReadyPlaceholders(url, this.getSiteUrl(), this._transformReadyToAbsoluteDefaults);
     }
 
     transformReadyToRelative(url: string, options?: TransformReadyToRelativeOptionsInput): string {

packages/url-utils/src/utils/absolute-to-relative.ts

@@ -1,4 +1,5 @@
 import {URL} from 'url';
+import parseRootUrl, {type ParsedRootUrl} from './parse-root-url';
 import stripSubdirectoryFromPath from './strip-subdirectory-from-path';
 
 export interface AbsoluteToRelativeOptions {
@@ -37,11 +38,11 @@ const absoluteToRelative = function absoluteToRelative(url: string, rootUrl?: st
     }
 
     let parsedUrl: URL;
-    let parsedRoot: URL | undefined;
+    let parsedRoot: ParsedRootUrl | undefined;
 
     try {
         parsedUrl = new URL(url, 'http://relative');
-        parsedRoot = parsedUrl.origin === 'null' ? undefined : new URL(rootUrl || parsedUrl.origin);
+        parsedRoot = parsedUrl.origin === 'null' ? undefined : parseRootUrl(rootUrl || parsedUrl.origin);
 
         // return the url as-is if it was relative or non-http
         if (parsedUrl.origin === 'null' || parsedUrl.origin === 'http://relative') {

packages/url-utils/src/utils/deduplicate-subdirectory.ts

@@ -1,4 +1,12 @@
-import {URL} from 'url';
+import memoize from './memoize';
+import parseRootUrl from './parse-root-url';
+
+const buildSubdirRegex = memoize(function buildSubdirRegex(pathname: string): {subdir: string; regex: RegExp} {
+    const subdir = pathname.replace(/(^\/|\/$)+/g, '');
+    // we can have subdirs that match TLDs so we need to restrict matches to
+    // duplicates that start with a / or the beginning of the url
+    return {subdir, regex: new RegExp(`(^|/)${subdir}/${subdir}(/|$)`)};
+});
 
 /**
  * Remove duplicated directories from the start of a path or url's path
@@ -13,19 +21,16 @@ const deduplicateSubdirectory = function deduplicateSubdirectory(url: string, ro
         rootUrl = `${rootUrl}/`;
     }
 
-    const parsedRoot = new URL(rootUrl);
+    const {pathname} = parseRootUrl(rootUrl);
 
     // do nothing if rootUrl does not have a subdirectory
-    if (parsedRoot.pathname === '/') {
+    if (pathname === '/') {
         return url;
     }
 
-    const subdir = parsedRoot.pathname.replace(/(^\/|\/$)+/g, '');
-    // we can have subdirs that match TLDs so we need to restrict matches to
-    // duplicates that start with a / or the beginning of the url
-    const subdirRegex = new RegExp(`(^|/)${subdir}/${subdir}(/|$)`);
+    const {subdir, regex} = buildSubdirRegex(pathname);
 
-    return url.replace(subdirRegex, `$1${subdir}/`);
+    return url.replace(regex, `$1${subdir}/`);
 };
 
 export default deduplicateSubdirectory;

packages/url-utils/src/utils/index.ts

@@ -23,6 +23,7 @@ import mobiledocRelativeToAbsolute from './mobiledoc-relative-to-absolute';
 import mobiledocAbsoluteToTransformReady from './mobiledoc-absolute-to-transform-ready';
 import mobiledocRelativeToTransformReady from './mobiledoc-relative-to-transform-ready';
 import mobiledocToTransformReady from './mobiledoc-to-transform-ready';
+import parseRootUrl from './parse-root-url';
 import plaintextAbsoluteToTransformReady from './plaintext-absolute-to-transform-ready';
 import plaintextRelativeToTransformReady from './plaintext-relative-to-transform-ready';
 import plaintextToTransformReady from './plaintext-to-transform-ready';
@@ -61,6 +62,7 @@ export {
     mobiledocAbsoluteToTransformReady,
     mobiledocRelativeToTransformReady,
     mobiledocToTransformReady,
+    parseRootUrl,
     plaintextAbsoluteToTransformReady,
     plaintextRelativeToTransformReady,
     plaintextToTransformReady,
@@ -100,6 +102,7 @@ const utils = {
     mobiledocRelativeToAbsolute,
     mobiledocRelativeToTransformReady,
     mobiledocToTransformReady,
+    parseRootUrl,
     plaintextAbsoluteToTransformReady,
     plaintextRelativeToTransformReady,
     plaintextToTransformReady,

packages/url-utils/src/utils/memoize.ts

@@ -0,0 +1,36 @@
+import {LRUCache} from 'lru-cache';
+
+export interface Memoized<V> {
+    (key: string): V;
+    clear(): void;
+}
+
+/**
+ * Memoize a pure single-argument function keyed on its string input, keeping
+ * the `max` most recently used results so arbitrary input can't grow the cache
+ * unchecked. Errors thrown by `fn` are not cached.
+ *
+ * @param {Function} fn function to memoize, must be a pure function of its input
+ * @param {number} [max=100] maximum number of cached results
+ * @returns {Function} memoized function with a `clear()` method
+ */
+export default function memoize<V extends object | string>(fn: (key: string) => V, max: number = 100): Memoized<V> {
+    const cache = new LRUCache<string, V>({max});
+
+    const memoized = function (key: string): V {
+        let value = cache.get(key);
+
+        if (value === undefined) {
+            value = fn(key);
+            cache.set(key, value);
+        }
+
+        return value;
+    };
+
+    memoized.clear = function clear(): void {
+        cache.clear();
+    };
+
+    return memoized;
+}

packages/url-utils/src/utils/parse-root-url.ts

@@ -0,0 +1,37 @@
+import {URL} from 'url';
+import memoize from './memoize';
+
+export interface ParsedRootUrl {
+    readonly href: string;
+    readonly origin: string;
+    readonly protocol: string;
+    readonly host: string;
+    readonly hostname: string;
+    readonly pathname: string;
+}
+
+/**
+ * Parse a root URL, returning a cached immutable snapshot of the parts url-utils uses.
+ * Throws the same errors as `new URL()` for invalid input (errors are not cached).
+ *
+ * Root URLs (site url, admin url, CDN base urls) are a tiny, near-static set of
+ * strings that get re-parsed on almost every url-utils call. Parsing is a pure
+ * function of the input string so results can't go stale.
+ *
+ * @param {string} rootUrl
+ * @returns {ParsedRootUrl}
+ */
+const parseRootUrl = memoize(function parseRootUrl(rootUrl: string): ParsedRootUrl {
+    const url = new URL(rootUrl);
+
+    return Object.freeze({
+        href: url.href,
+        origin: url.origin,
+        protocol: url.protocol,
+        host: url.host,
+        hostname: url.hostname,
+        pathname: url.pathname
+    });
+});
+
+export default parseRootUrl;

packages/url-utils/src/utils/plaintext-absolute-to-transform-ready.ts

@@ -2,7 +2,7 @@ import type {AbsoluteToTransformReadyOptionsInput, BaseUrlOptionsInput} from './
 import absoluteToTransformReady from './absolute-to-transform-ready';
 import buildEarlyExitMatchModule from './build-early-exit-match';
 const {escapeRegExp} = buildEarlyExitMatchModule;
-import {URL} from 'url';
+import parseRootUrl from './parse-root-url';
 
 type PlaintextAbsoluteToTransformReadyOptions = AbsoluteToTransformReadyOptionsInput & BaseUrlOptionsInput;
 type PlaintextAbsoluteToTransformReadyOptionsInput = Partial<PlaintextAbsoluteToTransformReadyOptions>;
@@ -13,7 +13,7 @@ function buildLinkRegex(rootUrl: string, options: PlaintextAbsoluteToTransformRe
         .filter((value): value is string => Boolean(value));
 
     const patterns = baseUrls.map((baseUrl: string) => {
-        const parsed = new URL(baseUrl);
+        const parsed = parseRootUrl(baseUrl);
         const escapedUrl = escapeRegExp(`${parsed.hostname}${parsed.pathname.replace(/\/$/, '')}`);
         return escapedUrl;
     });

packages/url-utils/src/utils/relative-to-absolute.ts

@@ -1,6 +1,7 @@
 import type {SecureOptions, SecureOptionsInput} from './types';
 import {URL} from 'url';
 import urlJoin from './url-join';
+import parseRootUrl from './parse-root-url';
 
 export type RelativeToAbsoluteOptions = SecureOptions;
 export type RelativeToAbsoluteOptionsInput = SecureOptionsInput;
@@ -92,7 +93,7 @@ const relativeToAbsolute = function relativeToAbsolute(
         rootUrl = `${rootUrl}/`;
     }
 
-    const parsedRootUrl: URL = new URL(rootUrl);
+    const parsedRootUrl = parseRootUrl(rootUrl);
     const basePath = path.startsWith('/') ? '' : (finalItemPath || '');
     const fullPath = urlJoin([parsedRootUrl.pathname, basePath, path], {rootUrl});
     const absoluteUrl = new URL(fullPath, rootUrl);