[Content collections] Fast getEntryBySlug() lookup (#6916)

* feat: add generated lookup-map

* feat: wire up fast getEntryBySlug() lookup

* fix: consider frontmatter slugs

* chore: changeset

* chore: lint no-shadow

* fix: revert bad rootRelativePath change

* chore: better var name

* refactor: generated `.json` to in-memory map

* chore: removed unneeded await

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* chore: removed unneeded await

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* Revert "chore: removed unneeded await"

This reverts commit 1b0a8b00c2.

* fix: bad `GetEntryImport` type

* chore: remove unused variable

* refactor: for -> Promise.all

* refactor: replace duplicate parseSlug

* refactor: add cache layer

* Revert "refactor: add cache layer"

This reverts commit 1c3bfdc6b3.

* refactor: add comment, move to virtual-mod

* chore: add jsdocs

* refactor: fiiiiine no more `??=`

* fix: ignore underscore files in lookup map

* chore: add unknowncollectionerror on bad file contents

---------

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>
This commit is contained in:
Ben Holmes 2023-05-09 12:17:08 -04:00 committed by GitHub
parent 1b90a7a5d5
commit 630f8c8ef6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 268 additions and 107 deletions

View file

@ -0,0 +1,5 @@
---
'astro': patch
---
Add fast lookups for content collection entries when using `getEntryBySlug()`. This generates a lookup map to ensure O(1) retrieval.

View file

@ -11,8 +11,10 @@ import {
unescapeHTML,
} from '../runtime/server/index.js';
type GlobResult = Record<string, () => Promise<any>>;
type LazyImport = () => Promise<any>;
type GlobResult = Record<string, LazyImport>;
type CollectionToEntryMap = Record<string, GlobResult>;
type GetEntryImport = (collection: string, lookupId: string) => Promise<LazyImport>;
export function createCollectionToGlobResultMap({
globResult,
@ -27,9 +29,8 @@ export function createCollectionToGlobResultMap({
const segments = keyRelativeToContentDir.split('/');
if (segments.length <= 1) continue;
const collection = segments[0];
const entryId = segments.slice(1).join('/');
collectionToGlobResultMap[collection] ??= {};
collectionToGlobResultMap[collection][entryId] = globResult[key];
collectionToGlobResultMap[collection][key] = globResult[key];
}
return collectionToGlobResultMap;
}
@ -37,10 +38,10 @@ export function createCollectionToGlobResultMap({
const cacheEntriesByCollection = new Map<string, any[]>();
export function createGetCollection({
collectionToEntryMap,
collectionToRenderEntryMap,
getRenderEntryImport,
}: {
collectionToEntryMap: CollectionToEntryMap;
collectionToRenderEntryMap: CollectionToEntryMap;
getRenderEntryImport: GetEntryImport;
}) {
return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
const lazyImports = Object.values(collectionToEntryMap[collection] ?? {});
@ -63,7 +64,7 @@ export function createGetCollection({
return render({
collection: entry.collection,
id: entry.id,
collectionToRenderEntryMap,
renderEntryImport: await getRenderEntryImport(collection, entry.slug),
});
},
};
@ -80,29 +81,18 @@ export function createGetCollection({
}
export function createGetEntryBySlug({
getCollection,
collectionToRenderEntryMap,
getEntryImport,
getRenderEntryImport,
}: {
getCollection: ReturnType<typeof createGetCollection>;
collectionToRenderEntryMap: CollectionToEntryMap;
getEntryImport: GetEntryImport;
getRenderEntryImport: GetEntryImport;
}) {
return async function getEntryBySlug(collection: string, slug: string) {
// This is not an optimized lookup. Should look into an O(1) implementation
// as it's probably that people will have very large collections.
const entries = await getCollection(collection);
let candidate: (typeof entries)[number] | undefined = undefined;
for (let entry of entries) {
if (entry.slug === slug) {
candidate = entry;
break;
}
}
const entryImport = await getEntryImport(collection, slug);
if (typeof entryImport !== 'function') return undefined;
if (typeof candidate === 'undefined') {
return undefined;
}
const entry = await entryImport();
const entry = candidate;
return {
id: entry.id,
slug: entry.slug,
@ -113,7 +103,7 @@ export function createGetEntryBySlug({
return render({
collection: entry.collection,
id: entry.id,
collectionToRenderEntryMap,
renderEntryImport: await getRenderEntryImport(collection, slug),
});
},
};
@ -123,21 +113,20 @@ export function createGetEntryBySlug({
async function render({
collection,
id,
collectionToRenderEntryMap,
renderEntryImport,
}: {
collection: string;
id: string;
collectionToRenderEntryMap: CollectionToEntryMap;
renderEntryImport?: LazyImport;
}) {
const UnexpectedRenderError = new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: `Unexpected error while rendering ${String(collection)}${String(id)}.`,
});
const lazyImport = collectionToRenderEntryMap[collection]?.[id];
if (typeof lazyImport !== 'function') throw UnexpectedRenderError;
if (typeof renderEntryImport !== 'function') throw UnexpectedRenderError;
const baseMod = await lazyImport();
const baseMod = await renderEntryImport();
if (baseMod == null || typeof baseMod !== 'object') throw UnexpectedRenderError;
const { collectedStyles, collectedLinks, collectedScripts, getMod } = baseMod;

View file

@ -28,6 +28,18 @@ const collectionToEntryMap = createCollectionToGlobResultMap({
contentDir,
});
let lookupMap = {};
/* @@LOOKUP_MAP_ASSIGNMENT@@ */
function createGlobLookup(glob) {
return async (collection, lookupId) => {
const filePath = lookupMap[collection]?.[lookupId];
if (!filePath) return undefined;
return glob[collection][filePath];
};
}
const renderEntryGlob = import.meta.glob('@@RENDER_ENTRY_GLOB_PATH@@', {
query: { astroPropagatedAssets: true },
});
@ -38,10 +50,10 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({
export const getCollection = createGetCollection({
collectionToEntryMap,
collectionToRenderEntryMap,
getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
});
export const getEntryBySlug = createGetEntryBySlug({
getCollection,
collectionToRenderEntryMap,
getEntryImport: createGlobLookup(collectionToEntryMap),
getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
});

View file

@ -8,20 +8,19 @@ import type { AstroSettings, ContentEntryType } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { info, warn, type LogOptions } from '../core/logger/core.js';
import { isRelativePath } from '../core/path.js';
import { CONTENT_TYPES_FILE } from './consts.js';
import { CONTENT_TYPES_FILE, VIRTUAL_MODULE_ID } from './consts.js';
import {
getContentEntryExts,
getContentPaths,
getEntryInfo,
getEntrySlug,
getEntryType,
loadContentConfig,
NoCollectionError,
parseFrontmatter,
type ContentConfig,
type ContentObservable,
type ContentPaths,
type EntryInfo,
getContentEntryConfigByExtMap,
getEntrySlug,
} from './utils.js';
type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
@ -58,7 +57,8 @@ export async function createContentTypesGenerator({
}: CreateContentGeneratorParams) {
const contentTypes: ContentTypes = {};
const contentPaths = getContentPaths(settings.config, fs);
const contentEntryExts = getContentEntryExts(settings);
const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings);
const contentEntryExts = [...contentEntryConfigByExt.keys()];
let events: EventWithOptions[] = [];
let debounceTimeout: NodeJS.Timeout | undefined;
@ -186,14 +186,23 @@ export async function createContentTypesGenerator({
return { shouldGenerateTypes: false };
}
const { id, collection } = entryInfo;
const { id, collection, slug: generatedSlug } = entryInfo;
const contentEntryType = contentEntryConfigByExt.get(path.extname(event.entry.pathname));
if (!contentEntryType) return { shouldGenerateTypes: false };
const collectionKey = JSON.stringify(collection);
const entryKey = JSON.stringify(id);
switch (event.name) {
case 'add':
const addedSlug = await parseSlug({ fs, event, entryInfo });
const addedSlug = await getEntrySlug({
generatedSlug,
id,
collection,
fileUrl: event.entry,
contentEntryType,
fs,
});
if (!(collectionKey in contentTypes)) {
addCollection(contentTypes, collectionKey);
}
@ -209,7 +218,14 @@ export async function createContentTypesGenerator({
case 'change':
// User may modify `slug` in their frontmatter.
// Only regen types if this change is detected.
const changedSlug = await parseSlug({ fs, event, entryInfo });
const changedSlug = await getEntrySlug({
generatedSlug,
id,
collection,
fileUrl: event.entry,
contentEntryType,
fs,
});
if (contentTypes[collectionKey]?.[entryKey]?.slug !== changedSlug) {
setEntry(contentTypes, collectionKey, entryKey, changedSlug);
return { shouldGenerateTypes: true };
@ -278,6 +294,7 @@ export async function createContentTypesGenerator({
contentConfig: observable.status === 'loaded' ? observable.config : undefined,
contentEntryTypes: settings.contentEntryTypes,
});
invalidateVirtualMod(viteServer);
if (observable.status === 'loaded' && ['info', 'warn'].includes(logLevel)) {
warnNonexistentCollections({
logging,
@ -290,6 +307,15 @@ export async function createContentTypesGenerator({
return { init, queueEvent };
}
// The virtual module contains a lookup map from slugs to content imports.
// Invalidate whenever content types change.
function invalidateVirtualMod(viteServer: ViteDevServer) {
const virtualMod = viteServer.moduleGraph.getModuleById('\0' + VIRTUAL_MODULE_ID);
if (!virtualMod) return;
viteServer.moduleGraph.invalidateModule(virtualMod);
}
function addCollection(contentMap: ContentTypes, collectionKey: string) {
contentMap[collectionKey] = {};
}
@ -298,25 +324,6 @@ function removeCollection(contentMap: ContentTypes, collectionKey: string) {
delete contentMap[collectionKey];
}
async function parseSlug({
fs,
event,
entryInfo,
}: {
fs: typeof fsMod;
event: ContentEvent;
entryInfo: EntryInfo;
}) {
// `slug` may be present in entry frontmatter.
// This should be respected by the generated `slug` type!
// Parse frontmatter and retrieve `slug` value for this.
// Note: will raise any YAML exceptions and `slug` parse errors (i.e. `slug` is a boolean)
// on dev server startup or production build init.
const rawContents = await fs.promises.readFile(event.entry, 'utf-8');
const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry));
return getEntrySlug({ ...entryInfo, unvalidatedSlug: frontmatter.slug });
}
function setEntry(
contentTypes: ContentTypes,
collectionKey: string,

View file

@ -6,7 +6,12 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
import type { PluginContext } from 'rollup';
import { normalizePath, type ErrorPayload as ViteErrorPayload, type ViteDevServer } from 'vite';
import { z } from 'zod';
import type { AstroConfig, AstroSettings, ImageInputFormat } from '../@types/astro.js';
import type {
AstroConfig,
AstroSettings,
ContentEntryType,
ImageInputFormat,
} from '../@types/astro.js';
import { VALID_INPUT_FORMATS } from '../assets/consts.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { CONTENT_TYPES_FILE } from './consts.js';
@ -45,14 +50,19 @@ export const msg = {
`${collection} does not have a config. We suggest adding one for type safety!`,
};
export function getEntrySlug({
export function parseEntrySlug({
id,
collection,
slug,
unvalidatedSlug,
}: EntryInfo & { unvalidatedSlug?: unknown }) {
generatedSlug,
frontmatterSlug,
}: {
id: string;
collection: string;
generatedSlug: string;
frontmatterSlug?: unknown;
}) {
try {
return z.string().default(slug).parse(unvalidatedSlug);
return z.string().default(generatedSlug).parse(frontmatterSlug);
} catch {
throw new AstroError({
...AstroErrorData.InvalidContentEntrySlugError,
@ -126,19 +136,36 @@ export function getContentEntryExts(settings: Pick<AstroSettings, 'contentEntryT
return settings.contentEntryTypes.map((t) => t.extensions).flat();
}
export function getContentEntryConfigByExtMap(settings: Pick<AstroSettings, 'contentEntryTypes'>) {
const map: Map<string, ContentEntryType> = new Map();
for (const entryType of settings.contentEntryTypes) {
for (const ext of entryType.extensions) {
map.set(ext, entryType);
}
}
return map;
}
export class NoCollectionError extends Error {}
export function getEntryInfo(
params: Pick<ContentPaths, 'contentDir'> & { entry: URL; allowFilesOutsideCollection?: true }
params: Pick<ContentPaths, 'contentDir'> & {
entry: string | URL;
allowFilesOutsideCollection?: true;
}
): EntryInfo;
export function getEntryInfo({
entry,
contentDir,
allowFilesOutsideCollection = false,
}: Pick<ContentPaths, 'contentDir'> & { entry: URL; allowFilesOutsideCollection?: boolean }):
| EntryInfo
| NoCollectionError {
const rawRelativePath = path.relative(fileURLToPath(contentDir), fileURLToPath(entry));
}: Pick<ContentPaths, 'contentDir'> & {
entry: string | URL;
allowFilesOutsideCollection?: boolean;
}): EntryInfo | NoCollectionError {
const rawRelativePath = path.relative(
fileURLToPath(contentDir),
typeof entry === 'string' ? entry : fileURLToPath(entry)
);
const rawCollection = path.dirname(rawRelativePath).split(path.sep).shift();
const isOutsideCollection = rawCollection === '..' || rawCollection === '.';
@ -200,7 +227,7 @@ function isImageAsset(fileExt: string) {
return VALID_INPUT_FORMATS.includes(fileExt.slice(1) as ImageInputFormat);
}
function hasUnderscoreBelowContentDirectoryPath(
export function hasUnderscoreBelowContentDirectoryPath(
fileUrl: URL,
contentDir: ContentPaths['contentDir']
): boolean {
@ -358,3 +385,43 @@ function search(fs: typeof fsMod, srcDir: URL) {
}
return { exists: false, url: paths[0] };
}
/**
* Check for slug in content entry frontmatter and validate the type,
* falling back to the `generatedSlug` if none is found.
*/
export async function getEntrySlug({
id,
collection,
generatedSlug,
contentEntryType,
fileUrl,
fs,
}: {
fs: typeof fsMod;
id: string;
collection: string;
generatedSlug: string;
fileUrl: URL;
contentEntryType: Pick<ContentEntryType, 'getEntryInfo'>;
}) {
let contents: string;
try {
contents = await fs.promises.readFile(fileUrl, 'utf-8');
} catch (e) {
// File contents should exist. Raise unexpected error as "unknown" if not.
throw new AstroError(AstroErrorData.UnknownContentCollectionError, { cause: e });
}
const { slug: frontmatterSlug } = await contentEntryType.getEntryInfo({
fileUrl,
contents: await fs.promises.readFile(fileUrl, 'utf-8'),
});
return parseEntrySlug({ generatedSlug, frontmatterSlug, id, collection });
}
export function getExtGlob(exts: string[]) {
return exts.length === 1
? // Wrapping {...} breaks when there is only one extension
exts[0]
: `{${exts.join(',')}}`;
}

View file

@ -16,7 +16,6 @@ import {
SCRIPTS_PLACEHOLDER,
STYLES_PLACEHOLDER,
} from './consts.js';
import { getContentEntryExts } from './utils.js';
function isPropagatedAsset(viteId: string) {
const flags = new URLSearchParams(viteId.split('?')[1]);
@ -31,7 +30,6 @@ export function astroContentAssetPropagationPlugin({
settings: AstroSettings;
}): Plugin {
let devModuleLoader: ModuleLoader;
const contentEntryExts = getContentEntryExts(settings);
return {
name: 'astro:content-asset-propagation',
configureServer(server) {

View file

@ -14,11 +14,12 @@ import {
getContentPaths,
getEntryData,
getEntryInfo,
getEntrySlug,
parseEntrySlug,
getEntryType,
globalContentConfigObserver,
NoCollectionError,
type ContentConfig,
getContentEntryConfigByExtMap,
} from './utils.js';
function isContentFlagImport(viteId: string) {
@ -55,12 +56,7 @@ export function astroContentImportPlugin({
const contentPaths = getContentPaths(settings.config, fs);
const contentEntryExts = getContentEntryExts(settings);
const contentEntryExtToParser: Map<string, ContentEntryType> = new Map();
for (const entryType of settings.contentEntryTypes) {
for (const ext of entryType.extensions) {
contentEntryExtToParser.set(ext, entryType);
}
}
const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings);
const plugins: Plugin[] = [
{
@ -196,7 +192,7 @@ export function astroContentImportPlugin({
const contentConfig = await getContentConfigFromGlobal();
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const fileExt = extname(fileId);
if (!contentEntryExtToParser.has(fileExt)) {
if (!contentEntryConfigByExt.has(fileExt)) {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: `No parser found for content entry ${JSON.stringify(
@ -204,13 +200,13 @@ export function astroContentImportPlugin({
)}. Did you apply an integration for this file type?`,
});
}
const contentEntryParser = contentEntryExtToParser.get(fileExt)!;
const contentEntryConfig = contentEntryConfigByExt.get(fileExt)!;
const {
rawData,
body,
slug: unvalidatedSlug,
slug: frontmatterSlug,
data: unvalidatedData,
} = await contentEntryParser.getEntryInfo({
} = await contentEntryConfig.getEntryInfo({
fileUrl: pathToFileURL(fileId),
contents: rawContents,
});
@ -225,7 +221,12 @@ export function astroContentImportPlugin({
const _internal = { filePath: fileId, rawData: rawData };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug({ id, collection, slug: generatedSlug, unvalidatedSlug });
const slug = parseEntrySlug({
id,
collection,
generatedSlug,
frontmatterSlug,
});
const collectionConfig = contentConfig?.collections[collection];
let data = collectionConfig

View file

@ -1,11 +1,21 @@
import glob, { type Options as FastGlobOptions } from 'fast-glob';
import fsMod from 'node:fs';
import * as path from 'node:path';
import type { Plugin } from 'vite';
import { normalizePath } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { appendForwardSlash, prependForwardSlash } from '../core/path.js';
import { VIRTUAL_MODULE_ID } from './consts.js';
import { getContentEntryExts, getContentPaths } from './utils.js';
import {
getContentEntryConfigByExtMap,
getContentPaths,
getExtGlob,
type ContentPaths,
getEntryInfo,
NoCollectionError,
getEntrySlug,
hasUnderscoreBelowContentDirectoryPath,
} from './utils.js';
import { rootRelativePath } from '../core/util.js';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { extname } from 'node:path';
interface AstroContentVirtualModPluginParams {
settings: AstroSettings;
@ -15,20 +25,12 @@ export function astroContentVirtualModPlugin({
settings,
}: AstroContentVirtualModPluginParams): Plugin {
const contentPaths = getContentPaths(settings.config);
const relContentDir = normalizePath(
appendForwardSlash(
prependForwardSlash(
path.relative(settings.config.root.pathname, contentPaths.contentDir.pathname)
)
)
);
const contentEntryExts = getContentEntryExts(settings);
const relContentDir = rootRelativePath(settings.config.root, contentPaths.contentDir);
const extGlob =
contentEntryExts.length === 1
? // Wrapping {...} breaks when there is only one extension
contentEntryExts[0]
: `{${contentEntryExts.join(',')}}`;
const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings);
const contentEntryExts = [...contentEntryConfigByExt.keys()];
const extGlob = getExtGlob(contentEntryExts);
const entryGlob = `${relContentDir}**/*${extGlob}`;
const virtualModContents = fsMod
.readFileSync(contentPaths.virtualModTemplate, 'utf-8')
@ -46,12 +48,88 @@ export function astroContentVirtualModPlugin({
return astroContentVirtualModuleId;
}
},
load(id) {
async load(id) {
const stringifiedLookupMap = await getStringifiedLookupMap({
fs: fsMod,
contentPaths,
contentEntryConfigByExt,
root: settings.config.root,
});
if (id === astroContentVirtualModuleId) {
return {
code: virtualModContents,
code: virtualModContents.replace(
'/* @@LOOKUP_MAP_ASSIGNMENT@@ */',
`lookupMap = ${stringifiedLookupMap};`
),
};
}
},
};
}
/**
* Generate a map from a collection + slug to the local file path.
* This is used internally to resolve entry imports when using `getEntryBySlug()`.
* @see `src/content/virtual-mod.mjs`
*/
export async function getStringifiedLookupMap({
contentPaths,
contentEntryConfigByExt,
root,
fs,
}: {
contentEntryConfigByExt: ReturnType<typeof getContentEntryConfigByExtMap>;
contentPaths: Pick<ContentPaths, 'contentDir' | 'cacheDir'>;
root: URL;
fs: typeof fsMod;
}) {
const { contentDir } = contentPaths;
const globOpts: FastGlobOptions = {
absolute: true,
cwd: fileURLToPath(root),
fs: {
readdir: fs.readdir.bind(fs),
readdirSync: fs.readdirSync.bind(fs),
},
};
const relContentDir = rootRelativePath(root, contentDir, false);
const contentGlob = await glob(
`${relContentDir}**/*${getExtGlob([...contentEntryConfigByExt.keys()])}`,
globOpts
);
let filePathByLookupId: {
[collection: string]: Record<string, string>;
} = {};
await Promise.all(
contentGlob
// Ignore underscore files in lookup map
.filter((e) => !hasUnderscoreBelowContentDirectoryPath(pathToFileURL(e), contentDir))
.map(async (filePath) => {
const info = getEntryInfo({ contentDir, entry: filePath });
// Globbed entry outside a collection directory
// Log warning during type generation, safe to ignore in lookup map
if (info instanceof NoCollectionError) return;
const contentEntryType = contentEntryConfigByExt.get(extname(filePath));
if (!contentEntryType) return;
const { id, collection, slug: generatedSlug } = info;
const slug = await getEntrySlug({
id,
collection,
generatedSlug,
fs,
fileUrl: pathToFileURL(filePath),
contentEntryType,
});
filePathByLookupId[collection] = {
...filePathByLookupId[collection],
[slug]: rootRelativePath(root, filePath),
};
})
);
return JSON.stringify(filePathByLookupId);
}

View file

@ -151,7 +151,11 @@ export function relativeToSrcDir(config: AstroConfig, idOrUrl: URL | string) {
return id.slice(slash(fileURLToPath(config.srcDir)).length);
}
export function rootRelativePath(root: URL, idOrUrl: URL | string) {
export function rootRelativePath(
root: URL,
idOrUrl: URL | string,
shouldPrependForwardSlash = true
) {
let id: string;
if (typeof idOrUrl !== 'string') {
id = unwrapId(viteID(idOrUrl));
@ -162,7 +166,7 @@ export function rootRelativePath(root: URL, idOrUrl: URL | string) {
if (id.startsWith(normalizedRoot)) {
id = id.slice(normalizedRoot.length);
}
return prependForwardSlash(id);
return shouldPrependForwardSlash ? prependForwardSlash(id) : id;
}
export function emoji(char: string, fallback: string) {