diff --git a/.changeset/friendly-poets-breathe.md b/.changeset/friendly-poets-breathe.md new file mode 100644 index 000000000..f7af9171e --- /dev/null +++ b/.changeset/friendly-poets-breathe.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Add fast lookups for content collection entries when using `getEntryBySlug()`. This generates a lookup map to ensure O(1) retrieval. diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index c0fddde09..35afc71e8 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -11,8 +11,10 @@ import { unescapeHTML, } from '../runtime/server/index.js'; -type GlobResult = Record Promise>; +type LazyImport = () => Promise; +type GlobResult = Record; type CollectionToEntryMap = Record; +type GetEntryImport = (collection: string, lookupId: string) => Promise; export function createCollectionToGlobResultMap({ globResult, @@ -27,9 +29,8 @@ export function createCollectionToGlobResultMap({ const segments = keyRelativeToContentDir.split('/'); if (segments.length <= 1) continue; const collection = segments[0]; - const entryId = segments.slice(1).join('/'); collectionToGlobResultMap[collection] ??= {}; - collectionToGlobResultMap[collection][entryId] = globResult[key]; + collectionToGlobResultMap[collection][key] = globResult[key]; } return collectionToGlobResultMap; } @@ -37,10 +38,10 @@ export function createCollectionToGlobResultMap({ const cacheEntriesByCollection = new Map(); export function createGetCollection({ collectionToEntryMap, - collectionToRenderEntryMap, + getRenderEntryImport, }: { collectionToEntryMap: CollectionToEntryMap; - collectionToRenderEntryMap: CollectionToEntryMap; + getRenderEntryImport: GetEntryImport; }) { return async function getCollection(collection: string, filter?: (entry: any) => unknown) { const lazyImports = Object.values(collectionToEntryMap[collection] ?? {}); @@ -63,7 +64,7 @@ export function createGetCollection({ return render({ collection: entry.collection, id: entry.id, - collectionToRenderEntryMap, + renderEntryImport: await getRenderEntryImport(collection, entry.slug), }); }, }; @@ -80,29 +81,18 @@ export function createGetCollection({ } export function createGetEntryBySlug({ - getCollection, - collectionToRenderEntryMap, + getEntryImport, + getRenderEntryImport, }: { - getCollection: ReturnType; - collectionToRenderEntryMap: CollectionToEntryMap; + getEntryImport: GetEntryImport; + getRenderEntryImport: GetEntryImport; }) { return async function getEntryBySlug(collection: string, slug: string) { - // This is not an optimized lookup. Should look into an O(1) implementation - // as it's probably that people will have very large collections. - const entries = await getCollection(collection); - let candidate: (typeof entries)[number] | undefined = undefined; - for (let entry of entries) { - if (entry.slug === slug) { - candidate = entry; - break; - } - } + const entryImport = await getEntryImport(collection, slug); + if (typeof entryImport !== 'function') return undefined; - if (typeof candidate === 'undefined') { - return undefined; - } + const entry = await entryImport(); - const entry = candidate; return { id: entry.id, slug: entry.slug, @@ -113,7 +103,7 @@ export function createGetEntryBySlug({ return render({ collection: entry.collection, id: entry.id, - collectionToRenderEntryMap, + renderEntryImport: await getRenderEntryImport(collection, slug), }); }, }; @@ -123,21 +113,20 @@ export function createGetEntryBySlug({ async function render({ collection, id, - collectionToRenderEntryMap, + renderEntryImport, }: { collection: string; id: string; - collectionToRenderEntryMap: CollectionToEntryMap; + renderEntryImport?: LazyImport; }) { const UnexpectedRenderError = new AstroError({ ...AstroErrorData.UnknownContentCollectionError, message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`, }); - const lazyImport = collectionToRenderEntryMap[collection]?.[id]; - if (typeof lazyImport !== 'function') throw UnexpectedRenderError; + if (typeof renderEntryImport !== 'function') throw UnexpectedRenderError; - const baseMod = await lazyImport(); + const baseMod = await renderEntryImport(); if (baseMod == null || typeof baseMod !== 'object') throw UnexpectedRenderError; const { collectedStyles, collectedLinks, collectedScripts, getMod } = baseMod; diff --git a/packages/astro/src/content/template/virtual-mod.mjs b/packages/astro/src/content/template/virtual-mod.mjs index 5e04ac5e7..a649804ce 100644 --- a/packages/astro/src/content/template/virtual-mod.mjs +++ b/packages/astro/src/content/template/virtual-mod.mjs @@ -28,6 +28,18 @@ const collectionToEntryMap = createCollectionToGlobResultMap({ contentDir, }); +let lookupMap = {}; +/* @@LOOKUP_MAP_ASSIGNMENT@@ */ + +function createGlobLookup(glob) { + return async (collection, lookupId) => { + const filePath = lookupMap[collection]?.[lookupId]; + + if (!filePath) return undefined; + return glob[collection][filePath]; + }; +} + const renderEntryGlob = import.meta.glob('@@RENDER_ENTRY_GLOB_PATH@@', { query: { astroPropagatedAssets: true }, }); @@ -38,10 +50,10 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({ export const getCollection = createGetCollection({ collectionToEntryMap, - collectionToRenderEntryMap, + getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), }); export const getEntryBySlug = createGetEntryBySlug({ - getCollection, - collectionToRenderEntryMap, + getEntryImport: createGlobLookup(collectionToEntryMap), + getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), }); diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index 825f7b5d8..d63afd812 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -8,20 +8,19 @@ import type { AstroSettings, ContentEntryType } from '../@types/astro.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { info, warn, type LogOptions } from '../core/logger/core.js'; import { isRelativePath } from '../core/path.js'; -import { CONTENT_TYPES_FILE } from './consts.js'; +import { CONTENT_TYPES_FILE, VIRTUAL_MODULE_ID } from './consts.js'; import { - getContentEntryExts, getContentPaths, getEntryInfo, - getEntrySlug, getEntryType, loadContentConfig, NoCollectionError, - parseFrontmatter, type ContentConfig, type ContentObservable, type ContentPaths, type EntryInfo, + getContentEntryConfigByExtMap, + getEntrySlug, } from './utils.js'; type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'; @@ -58,7 +57,8 @@ export async function createContentTypesGenerator({ }: CreateContentGeneratorParams) { const contentTypes: ContentTypes = {}; const contentPaths = getContentPaths(settings.config, fs); - const contentEntryExts = getContentEntryExts(settings); + const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); + const contentEntryExts = [...contentEntryConfigByExt.keys()]; let events: EventWithOptions[] = []; let debounceTimeout: NodeJS.Timeout | undefined; @@ -186,14 +186,23 @@ export async function createContentTypesGenerator({ return { shouldGenerateTypes: false }; } - const { id, collection } = entryInfo; + const { id, collection, slug: generatedSlug } = entryInfo; + const contentEntryType = contentEntryConfigByExt.get(path.extname(event.entry.pathname)); + if (!contentEntryType) return { shouldGenerateTypes: false }; const collectionKey = JSON.stringify(collection); const entryKey = JSON.stringify(id); switch (event.name) { case 'add': - const addedSlug = await parseSlug({ fs, event, entryInfo }); + const addedSlug = await getEntrySlug({ + generatedSlug, + id, + collection, + fileUrl: event.entry, + contentEntryType, + fs, + }); if (!(collectionKey in contentTypes)) { addCollection(contentTypes, collectionKey); } @@ -209,7 +218,14 @@ export async function createContentTypesGenerator({ case 'change': // User may modify `slug` in their frontmatter. // Only regen types if this change is detected. - const changedSlug = await parseSlug({ fs, event, entryInfo }); + const changedSlug = await getEntrySlug({ + generatedSlug, + id, + collection, + fileUrl: event.entry, + contentEntryType, + fs, + }); if (contentTypes[collectionKey]?.[entryKey]?.slug !== changedSlug) { setEntry(contentTypes, collectionKey, entryKey, changedSlug); return { shouldGenerateTypes: true }; @@ -278,6 +294,7 @@ export async function createContentTypesGenerator({ contentConfig: observable.status === 'loaded' ? observable.config : undefined, contentEntryTypes: settings.contentEntryTypes, }); + invalidateVirtualMod(viteServer); if (observable.status === 'loaded' && ['info', 'warn'].includes(logLevel)) { warnNonexistentCollections({ logging, @@ -290,6 +307,15 @@ export async function createContentTypesGenerator({ return { init, queueEvent }; } +// The virtual module contains a lookup map from slugs to content imports. +// Invalidate whenever content types change. +function invalidateVirtualMod(viteServer: ViteDevServer) { + const virtualMod = viteServer.moduleGraph.getModuleById('\0' + VIRTUAL_MODULE_ID); + if (!virtualMod) return; + + viteServer.moduleGraph.invalidateModule(virtualMod); +} + function addCollection(contentMap: ContentTypes, collectionKey: string) { contentMap[collectionKey] = {}; } @@ -298,25 +324,6 @@ function removeCollection(contentMap: ContentTypes, collectionKey: string) { delete contentMap[collectionKey]; } -async function parseSlug({ - fs, - event, - entryInfo, -}: { - fs: typeof fsMod; - event: ContentEvent; - entryInfo: EntryInfo; -}) { - // `slug` may be present in entry frontmatter. - // This should be respected by the generated `slug` type! - // Parse frontmatter and retrieve `slug` value for this. - // Note: will raise any YAML exceptions and `slug` parse errors (i.e. `slug` is a boolean) - // on dev server startup or production build init. - const rawContents = await fs.promises.readFile(event.entry, 'utf-8'); - const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry)); - return getEntrySlug({ ...entryInfo, unvalidatedSlug: frontmatter.slug }); -} - function setEntry( contentTypes: ContentTypes, collectionKey: string, diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index f392efc55..d161b93de 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -6,7 +6,12 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import type { PluginContext } from 'rollup'; import { normalizePath, type ErrorPayload as ViteErrorPayload, type ViteDevServer } from 'vite'; import { z } from 'zod'; -import type { AstroConfig, AstroSettings, ImageInputFormat } from '../@types/astro.js'; +import type { + AstroConfig, + AstroSettings, + ContentEntryType, + ImageInputFormat, +} from '../@types/astro.js'; import { VALID_INPUT_FORMATS } from '../assets/consts.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { CONTENT_TYPES_FILE } from './consts.js'; @@ -45,14 +50,19 @@ export const msg = { `${collection} does not have a config. We suggest adding one for type safety!`, }; -export function getEntrySlug({ +export function parseEntrySlug({ id, collection, - slug, - unvalidatedSlug, -}: EntryInfo & { unvalidatedSlug?: unknown }) { + generatedSlug, + frontmatterSlug, +}: { + id: string; + collection: string; + generatedSlug: string; + frontmatterSlug?: unknown; +}) { try { - return z.string().default(slug).parse(unvalidatedSlug); + return z.string().default(generatedSlug).parse(frontmatterSlug); } catch { throw new AstroError({ ...AstroErrorData.InvalidContentEntrySlugError, @@ -126,19 +136,36 @@ export function getContentEntryExts(settings: Pick t.extensions).flat(); } +export function getContentEntryConfigByExtMap(settings: Pick) { + const map: Map = new Map(); + for (const entryType of settings.contentEntryTypes) { + for (const ext of entryType.extensions) { + map.set(ext, entryType); + } + } + return map; +} + export class NoCollectionError extends Error {} export function getEntryInfo( - params: Pick & { entry: URL; allowFilesOutsideCollection?: true } + params: Pick & { + entry: string | URL; + allowFilesOutsideCollection?: true; + } ): EntryInfo; export function getEntryInfo({ entry, contentDir, allowFilesOutsideCollection = false, -}: Pick & { entry: URL; allowFilesOutsideCollection?: boolean }): - | EntryInfo - | NoCollectionError { - const rawRelativePath = path.relative(fileURLToPath(contentDir), fileURLToPath(entry)); +}: Pick & { + entry: string | URL; + allowFilesOutsideCollection?: boolean; +}): EntryInfo | NoCollectionError { + const rawRelativePath = path.relative( + fileURLToPath(contentDir), + typeof entry === 'string' ? entry : fileURLToPath(entry) + ); const rawCollection = path.dirname(rawRelativePath).split(path.sep).shift(); const isOutsideCollection = rawCollection === '..' || rawCollection === '.'; @@ -200,7 +227,7 @@ function isImageAsset(fileExt: string) { return VALID_INPUT_FORMATS.includes(fileExt.slice(1) as ImageInputFormat); } -function hasUnderscoreBelowContentDirectoryPath( +export function hasUnderscoreBelowContentDirectoryPath( fileUrl: URL, contentDir: ContentPaths['contentDir'] ): boolean { @@ -358,3 +385,43 @@ function search(fs: typeof fsMod, srcDir: URL) { } return { exists: false, url: paths[0] }; } + +/** + * Check for slug in content entry frontmatter and validate the type, + * falling back to the `generatedSlug` if none is found. + */ +export async function getEntrySlug({ + id, + collection, + generatedSlug, + contentEntryType, + fileUrl, + fs, +}: { + fs: typeof fsMod; + id: string; + collection: string; + generatedSlug: string; + fileUrl: URL; + contentEntryType: Pick; +}) { + let contents: string; + try { + contents = await fs.promises.readFile(fileUrl, 'utf-8'); + } catch (e) { + // File contents should exist. Raise unexpected error as "unknown" if not. + throw new AstroError(AstroErrorData.UnknownContentCollectionError, { cause: e }); + } + const { slug: frontmatterSlug } = await contentEntryType.getEntryInfo({ + fileUrl, + contents: await fs.promises.readFile(fileUrl, 'utf-8'), + }); + return parseEntrySlug({ generatedSlug, frontmatterSlug, id, collection }); +} + +export function getExtGlob(exts: string[]) { + return exts.length === 1 + ? // Wrapping {...} breaks when there is only one extension + exts[0] + : `{${exts.join(',')}}`; +} diff --git a/packages/astro/src/content/vite-plugin-content-assets.ts b/packages/astro/src/content/vite-plugin-content-assets.ts index efce94e9c..7e73f9f6b 100644 --- a/packages/astro/src/content/vite-plugin-content-assets.ts +++ b/packages/astro/src/content/vite-plugin-content-assets.ts @@ -16,7 +16,6 @@ import { SCRIPTS_PLACEHOLDER, STYLES_PLACEHOLDER, } from './consts.js'; -import { getContentEntryExts } from './utils.js'; function isPropagatedAsset(viteId: string) { const flags = new URLSearchParams(viteId.split('?')[1]); @@ -31,7 +30,6 @@ export function astroContentAssetPropagationPlugin({ settings: AstroSettings; }): Plugin { let devModuleLoader: ModuleLoader; - const contentEntryExts = getContentEntryExts(settings); return { name: 'astro:content-asset-propagation', configureServer(server) { diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index 67fb45bcd..27a17b58e 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -14,11 +14,12 @@ import { getContentPaths, getEntryData, getEntryInfo, - getEntrySlug, + parseEntrySlug, getEntryType, globalContentConfigObserver, NoCollectionError, type ContentConfig, + getContentEntryConfigByExtMap, } from './utils.js'; function isContentFlagImport(viteId: string) { @@ -55,12 +56,7 @@ export function astroContentImportPlugin({ const contentPaths = getContentPaths(settings.config, fs); const contentEntryExts = getContentEntryExts(settings); - const contentEntryExtToParser: Map = new Map(); - for (const entryType of settings.contentEntryTypes) { - for (const ext of entryType.extensions) { - contentEntryExtToParser.set(ext, entryType); - } - } + const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); const plugins: Plugin[] = [ { @@ -196,7 +192,7 @@ export function astroContentImportPlugin({ const contentConfig = await getContentConfigFromGlobal(); const rawContents = await fs.promises.readFile(fileId, 'utf-8'); const fileExt = extname(fileId); - if (!contentEntryExtToParser.has(fileExt)) { + if (!contentEntryConfigByExt.has(fileExt)) { throw new AstroError({ ...AstroErrorData.UnknownContentCollectionError, message: `No parser found for content entry ${JSON.stringify( @@ -204,13 +200,13 @@ export function astroContentImportPlugin({ )}. Did you apply an integration for this file type?`, }); } - const contentEntryParser = contentEntryExtToParser.get(fileExt)!; + const contentEntryConfig = contentEntryConfigByExt.get(fileExt)!; const { rawData, body, - slug: unvalidatedSlug, + slug: frontmatterSlug, data: unvalidatedData, - } = await contentEntryParser.getEntryInfo({ + } = await contentEntryConfig.getEntryInfo({ fileUrl: pathToFileURL(fileId), contents: rawContents, }); @@ -225,7 +221,12 @@ export function astroContentImportPlugin({ const _internal = { filePath: fileId, rawData: rawData }; // TODO: move slug calculation to the start of the build // to generate a performant lookup map for `getEntryBySlug` - const slug = getEntrySlug({ id, collection, slug: generatedSlug, unvalidatedSlug }); + const slug = parseEntrySlug({ + id, + collection, + generatedSlug, + frontmatterSlug, + }); const collectionConfig = contentConfig?.collections[collection]; let data = collectionConfig diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 3a72bf1de..f36fa6187 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -1,11 +1,21 @@ +import glob, { type Options as FastGlobOptions } from 'fast-glob'; import fsMod from 'node:fs'; -import * as path from 'node:path'; import type { Plugin } from 'vite'; -import { normalizePath } from 'vite'; import type { AstroSettings } from '../@types/astro.js'; -import { appendForwardSlash, prependForwardSlash } from '../core/path.js'; import { VIRTUAL_MODULE_ID } from './consts.js'; -import { getContentEntryExts, getContentPaths } from './utils.js'; +import { + getContentEntryConfigByExtMap, + getContentPaths, + getExtGlob, + type ContentPaths, + getEntryInfo, + NoCollectionError, + getEntrySlug, + hasUnderscoreBelowContentDirectoryPath, +} from './utils.js'; +import { rootRelativePath } from '../core/util.js'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { extname } from 'node:path'; interface AstroContentVirtualModPluginParams { settings: AstroSettings; @@ -15,20 +25,12 @@ export function astroContentVirtualModPlugin({ settings, }: AstroContentVirtualModPluginParams): Plugin { const contentPaths = getContentPaths(settings.config); - const relContentDir = normalizePath( - appendForwardSlash( - prependForwardSlash( - path.relative(settings.config.root.pathname, contentPaths.contentDir.pathname) - ) - ) - ); - const contentEntryExts = getContentEntryExts(settings); + const relContentDir = rootRelativePath(settings.config.root, contentPaths.contentDir); - const extGlob = - contentEntryExts.length === 1 - ? // Wrapping {...} breaks when there is only one extension - contentEntryExts[0] - : `{${contentEntryExts.join(',')}}`; + const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); + const contentEntryExts = [...contentEntryConfigByExt.keys()]; + + const extGlob = getExtGlob(contentEntryExts); const entryGlob = `${relContentDir}**/*${extGlob}`; const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') @@ -46,12 +48,88 @@ export function astroContentVirtualModPlugin({ return astroContentVirtualModuleId; } }, - load(id) { + async load(id) { + const stringifiedLookupMap = await getStringifiedLookupMap({ + fs: fsMod, + contentPaths, + contentEntryConfigByExt, + root: settings.config.root, + }); + if (id === astroContentVirtualModuleId) { return { - code: virtualModContents, + code: virtualModContents.replace( + '/* @@LOOKUP_MAP_ASSIGNMENT@@ */', + `lookupMap = ${stringifiedLookupMap};` + ), }; } }, }; } + +/** + * Generate a map from a collection + slug to the local file path. + * This is used internally to resolve entry imports when using `getEntryBySlug()`. + * @see `src/content/virtual-mod.mjs` + */ +export async function getStringifiedLookupMap({ + contentPaths, + contentEntryConfigByExt, + root, + fs, +}: { + contentEntryConfigByExt: ReturnType; + contentPaths: Pick; + root: URL; + fs: typeof fsMod; +}) { + const { contentDir } = contentPaths; + const globOpts: FastGlobOptions = { + absolute: true, + cwd: fileURLToPath(root), + fs: { + readdir: fs.readdir.bind(fs), + readdirSync: fs.readdirSync.bind(fs), + }, + }; + + const relContentDir = rootRelativePath(root, contentDir, false); + const contentGlob = await glob( + `${relContentDir}**/*${getExtGlob([...contentEntryConfigByExt.keys()])}`, + globOpts + ); + let filePathByLookupId: { + [collection: string]: Record; + } = {}; + + await Promise.all( + contentGlob + // Ignore underscore files in lookup map + .filter((e) => !hasUnderscoreBelowContentDirectoryPath(pathToFileURL(e), contentDir)) + .map(async (filePath) => { + const info = getEntryInfo({ contentDir, entry: filePath }); + // Globbed entry outside a collection directory + // Log warning during type generation, safe to ignore in lookup map + if (info instanceof NoCollectionError) return; + const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); + if (!contentEntryType) return; + + const { id, collection, slug: generatedSlug } = info; + const slug = await getEntrySlug({ + id, + collection, + generatedSlug, + fs, + fileUrl: pathToFileURL(filePath), + contentEntryType, + }); + filePathByLookupId[collection] = { + ...filePathByLookupId[collection], + [slug]: rootRelativePath(root, filePath), + }; + }) + ); + + return JSON.stringify(filePathByLookupId); +} diff --git a/packages/astro/src/core/util.ts b/packages/astro/src/core/util.ts index 5d8868115..593f2fa7d 100644 --- a/packages/astro/src/core/util.ts +++ b/packages/astro/src/core/util.ts @@ -151,7 +151,11 @@ export function relativeToSrcDir(config: AstroConfig, idOrUrl: URL | string) { return id.slice(slash(fileURLToPath(config.srcDir)).length); } -export function rootRelativePath(root: URL, idOrUrl: URL | string) { +export function rootRelativePath( + root: URL, + idOrUrl: URL | string, + shouldPrependForwardSlash = true +) { let id: string; if (typeof idOrUrl !== 'string') { id = unwrapId(viteID(idOrUrl)); @@ -162,7 +166,7 @@ export function rootRelativePath(root: URL, idOrUrl: URL | string) { if (id.startsWith(normalizedRoot)) { id = id.slice(normalizedRoot.length); } - return prependForwardSlash(id); + return shouldPrependForwardSlash ? prependForwardSlash(id) : id; } export function emoji(char: string, fallback: string) {