feat: Expose $entry variable to Markdoc (#6588)

* wip: dream api

* deps: rollup types

* feat: get entry data in there

* fix: properly show mdoc errors in overlay

* feat: implement with cache

* fix: wait for in-flight entry resolution

* test: entry properties can be rendered

* chore: changeset

* fix: remove rollup type import

* Revert "deps: rollup types"

This reverts commit 484ccb1c81.

* docs: add README reference

* docs nit: missing space

Co-authored-by: Yan Thomas <61414485+Yan-Thomas@users.noreply.github.com>

* refactor: split config config loading to separate helper

* refactor: choose more readable variable names

* refactor: store awaiting queue in existing cache

* docs: add clear code comments

* nit: add skip module code comment

* refactor: add `idHandledByContentRenderPlugin`

* nit: store chokidar modified events in const

* fix: remove loop from content renderer

* nit: else if -> if

---------

Co-authored-by: Yan Thomas <61414485+Yan-Thomas@users.noreply.github.com>
This commit is contained in:
Ben Holmes 2023-03-21 09:20:23 -04:00 committed by GitHub
parent e8b3c886a7
commit f42f47dc6a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 425 additions and 131 deletions

View file

@ -0,0 +1,14 @@
---
'astro': patch
'@astrojs/markdoc': patch
---
Allow access to content collection entry information (including parsed frontmatter and the entry slug) from your Markdoc using the `$entry` variable:
```mdx
---
title: Hello Markdoc!
---
# {% $entry.data.title %}
```

View file

@ -10,6 +10,7 @@ import type {
import type * as babel from '@babel/core';
import type { OutgoingHttpHeaders } from 'http';
import type { AddressInfo } from 'net';
import type * as rollup from 'rollup';
import type { TsConfigJson } from 'tsconfig-resolver';
import type * as vite from 'vite';
import type { z } from 'zod';
@ -1034,12 +1035,27 @@ export interface AstroConfig extends z.output<typeof AstroConfigSchema> {
integrations: AstroIntegration[];
}
export type ContentEntryModule = {
id: string;
collection: string;
slug: string;
body: string;
data: Record<string, unknown>;
_internal: {
rawData: string;
filePath: string;
};
};
export interface ContentEntryType {
extensions: string[];
getEntryInfo(params: {
fileUrl: URL;
contents: string;
}): GetEntryInfoReturnType | Promise<GetEntryInfoReturnType>;
getRenderModule?(params: {
entry: ContentEntryModule;
}): rollup.LoadResult | Promise<rollup.LoadResult>;
contentModuleTypes?: string;
}

View file

@ -1,6 +1,8 @@
import * as devalue from 'devalue';
import type fsMod from 'node:fs';
import type { ContentEntryModule } from '../@types/astro.js';
import { extname } from 'node:path';
import type { PluginContext } from 'rollup';
import { pathToFileURL } from 'url';
import type { Plugin } from 'vite';
import type { AstroSettings, ContentEntryType } from '../@types/astro.js';
@ -16,21 +18,42 @@ import {
getEntrySlug,
getEntryType,
globalContentConfigObserver,
NoCollectionError,
patchAssets,
type ContentConfig,
} from './utils.js';
function isContentFlagImport(viteId: string, contentEntryExts: string[]) {
const { searchParams, pathname } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG) && contentEntryExts.some((ext) => pathname.endsWith(ext));
}
function getContentRendererByViteId(
viteId: string,
settings: Pick<AstroSettings, 'contentEntryTypes'>
): ContentEntryType['getRenderModule'] | undefined {
let ext = viteId.split('.').pop();
if (!ext) return undefined;
for (const contentEntryType of settings.contentEntryTypes) {
if (
Boolean(contentEntryType.getRenderModule) &&
contentEntryType.extensions.includes('.' + ext)
) {
return contentEntryType.getRenderModule;
}
}
return undefined;
}
const CHOKIDAR_MODIFIED_EVENTS = ['add', 'unlink', 'change'];
export function astroContentImportPlugin({
fs,
settings,
}: {
fs: typeof fsMod;
settings: AstroSettings;
}): Plugin {
}): Plugin[] {
const contentPaths = getContentPaths(settings.config, fs);
const contentEntryExts = getContentEntryExts(settings);
@ -41,116 +64,235 @@ export function astroContentImportPlugin({
}
}
return {
name: 'astro:content-imports',
async load(id) {
const { fileId } = getFileInfo(id, settings.config);
if (isContentFlagImport(id, contentEntryExts)) {
const observable = globalContentConfigObserver.get();
// Content config should be loaded before this plugin is used
if (observable.status === 'init') {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: 'Content config failed to load.',
const plugins: Plugin[] = [
{
name: 'astro:content-imports',
async load(viteId) {
if (isContentFlagImport(viteId, contentEntryExts)) {
const { fileId } = getFileInfo(viteId, settings.config);
const { id, slug, collection, body, data, _internal } = await setContentEntryModuleCache({
fileId,
pluginContext: this,
});
}
if (observable.status === 'error') {
// Throw here to bubble content config errors
// to the error overlay in development
throw observable.error;
}
let contentConfig: ContentConfig | undefined =
observable.status === 'loaded' ? observable.config : undefined;
if (observable.status === 'loading') {
// Wait for config to load
contentConfig = await new Promise((resolve) => {
const unsubscribe = globalContentConfigObserver.subscribe((ctx) => {
if (ctx.status === 'loaded') {
resolve(ctx.config);
unsubscribe();
} else if (ctx.status === 'error') {
resolve(undefined);
unsubscribe();
}
});
});
}
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const fileExt = extname(fileId);
if (!contentEntryExtToParser.has(fileExt)) {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: `No parser found for content entry ${JSON.stringify(
fileId
)}. Did you apply an integration for this file type?`,
});
}
const contentEntryParser = contentEntryExtToParser.get(fileExt)!;
const info = await contentEntryParser.getEntryInfo({
fileUrl: pathToFileURL(fileId),
contents: rawContents,
});
const generatedInfo = getEntryInfo({
entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir,
});
if (generatedInfo instanceof Error) return;
const _internal = { filePath: fileId, rawData: info.rawData };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug({ ...generatedInfo, unvalidatedSlug: info.slug });
const collectionConfig = contentConfig?.collections[generatedInfo.collection];
let data = collectionConfig
? await getEntryData(
{ ...generatedInfo, _internal, unvalidatedData: info.data },
collectionConfig
)
: info.data;
await patchAssets(data, this.meta.watchMode, this.emitFile, settings);
const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(generatedInfo.id)};
export const collection = ${JSON.stringify(generatedInfo.collection)};
const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(id)};
export const collection = ${JSON.stringify(collection)};
export const slug = ${JSON.stringify(slug)};
export const body = ${JSON.stringify(info.body)};
export const body = ${JSON.stringify(body)};
export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
export const _internal = {
filePath: ${JSON.stringify(_internal.filePath)},
rawData: ${JSON.stringify(_internal.rawData)},
};
`);
return { code };
}
},
configureServer(viteServer) {
viteServer.watcher.on('all', async (event, entry) => {
if (
['add', 'unlink', 'change'].includes(event) &&
getEntryType(entry, contentPaths, contentEntryExts) === 'config'
) {
// Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {
if (isContentFlagImport(modUrl, contentEntryExts)) {
const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl);
if (mod) {
viteServer.moduleGraph.invalidateModule(mod);
return { code };
}
},
configureServer(viteServer) {
viteServer.watcher.on('all', async (event, entry) => {
if (
CHOKIDAR_MODIFIED_EVENTS.includes(event) &&
getEntryType(entry, contentPaths, contentEntryExts) === 'config'
) {
// Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {
if (
isContentFlagImport(modUrl, contentEntryExts) ||
Boolean(getContentRendererByViteId(modUrl, settings))
) {
const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl);
if (mod) {
viteServer.moduleGraph.invalidateModule(mod);
}
}
}
}
});
},
async transform(code, id) {
if (isContentFlagImport(id, contentEntryExts)) {
// Escape before Rollup internal transform.
// Base on MUCH trial-and-error, inspired by MDX integration 2-step transform.
return { code: escapeViteEnvReferences(code) };
}
},
},
];
if (settings.contentEntryTypes.some((t) => t.getRenderModule)) {
plugins.push({
name: 'astro:content-render-imports',
async load(viteId) {
const contentRenderer = getContentRendererByViteId(viteId, settings);
if (!contentRenderer) return;
const { fileId } = getFileInfo(viteId, settings.config);
const entry = await getContentEntryModuleFromCache(fileId);
if (!entry) {
// Cached entry must exist (or be in-flight) when importing the module via content collections.
// This is ensured by the `astro:content-imports` plugin.
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: `Unable to render ${JSON.stringify(
fileId
)}. Did you import this module directly without using a content collection query?`,
});
}
return contentRenderer({ entry });
},
});
}
/**
* There are two content collection plugins that depend on the same entry data:
* - `astro:content-imports` - creates module containing the `getCollection()` result.
* - `astro:content-render-imports` - creates module containing the `collectionEntry.render()` result.
*
* We could run the same transforms to generate the slug and parsed data in each plugin,
* though this would run the user's collection schema _twice_ for each entry.
*
* Instead, we've implemented a cache for all content entry data. To avoid race conditions,
* this may store either the module itself or a queue of promises awaiting this module.
* See the implementations of `getContentEntryModuleFromCache` and `setContentEntryModuleCache`.
*/
const contentEntryModuleByIdCache = new Map<
string,
ContentEntryModule | AwaitingCacheResultQueue
>();
type AwaitingCacheResultQueue = {
awaitingQueue: ((val: ContentEntryModule) => void)[];
};
function isAwaitingQueue(
cacheEntry: ReturnType<typeof contentEntryModuleByIdCache.get>
): cacheEntry is AwaitingCacheResultQueue {
return typeof cacheEntry === 'object' && cacheEntry != null && 'awaitingQueue' in cacheEntry;
}
function getContentEntryModuleFromCache(id: string): Promise<ContentEntryModule | undefined> {
const cacheEntry = contentEntryModuleByIdCache.get(id);
// It's possible to request an entry while `setContentEntryModuleCache` is still
// setting that entry. In this case, queue a promise for `setContentEntryModuleCache`
// to resolve once it is complete.
if (isAwaitingQueue(cacheEntry)) {
return new Promise<ContentEntryModule>((resolve, reject) => {
cacheEntry.awaitingQueue.push(resolve);
});
} else if (cacheEntry) {
return Promise.resolve(cacheEntry);
}
return Promise.resolve(undefined);
}
async function setContentEntryModuleCache({
fileId,
pluginContext,
}: {
fileId: string;
pluginContext: PluginContext;
}): Promise<ContentEntryModule> {
// Create a queue so, if `getContentEntryModuleFromCache` is called
// while this function is running, we can resolve all requests
// in the `awaitingQueue` with the result.
contentEntryModuleByIdCache.set(fileId, { awaitingQueue: [] });
const contentConfig = await getContentConfigFromGlobal();
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const fileExt = extname(fileId);
if (!contentEntryExtToParser.has(fileExt)) {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: `No parser found for content entry ${JSON.stringify(
fileId
)}. Did you apply an integration for this file type?`,
});
}
const contentEntryParser = contentEntryExtToParser.get(fileExt)!;
const {
rawData,
body,
slug: unvalidatedSlug,
data: unvalidatedData,
} = await contentEntryParser.getEntryInfo({
fileUrl: pathToFileURL(fileId),
contents: rawContents,
});
const entryInfoResult = getEntryInfo({
entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir,
});
if (entryInfoResult instanceof NoCollectionError) throw entryInfoResult;
const { id, slug: generatedSlug, collection } = entryInfoResult;
const _internal = { filePath: fileId, rawData: rawData };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug({ id, collection, slug: generatedSlug, unvalidatedSlug });
const collectionConfig = contentConfig?.collections[collection];
let data = collectionConfig
? await getEntryData({ id, collection, slug, _internal, unvalidatedData }, collectionConfig)
: unvalidatedData;
await patchAssets(data, pluginContext.meta.watchMode, pluginContext.emitFile, settings);
const contentEntryModule: ContentEntryModule = {
id,
slug,
collection,
data,
body,
_internal,
};
const cacheEntry = contentEntryModuleByIdCache.get(fileId);
// Pass the entry to all promises awaiting this result
if (isAwaitingQueue(cacheEntry)) {
for (const resolve of cacheEntry.awaitingQueue) {
resolve(contentEntryModule);
}
}
contentEntryModuleByIdCache.set(fileId, contentEntryModule);
return contentEntryModule;
}
return plugins;
}
async function getContentConfigFromGlobal() {
const observable = globalContentConfigObserver.get();
// Content config should be loaded before being accessed from Vite plugins
if (observable.status === 'init') {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: 'Content config failed to load.',
});
}
if (observable.status === 'error') {
// Throw here to bubble content config errors
// to the error overlay in development
throw observable.error;
}
let contentConfig: ContentConfig | undefined =
observable.status === 'loaded' ? observable.config : undefined;
if (observable.status === 'loading') {
// Wait for config to load
contentConfig = await new Promise((resolve) => {
const unsubscribe = globalContentConfigObserver.subscribe((ctx) => {
if (ctx.status === 'loaded') {
resolve(ctx.config);
unsubscribe();
}
if (ctx.status === 'error') {
resolve(undefined);
unsubscribe();
}
});
},
async transform(code, id) {
if (isContentFlagImport(id, contentEntryExts)) {
// Escape before Rollup internal transform.
// Base on MUCH trial-and-error, inspired by MDX integration 2-step transform.
return { code: escapeViteEnvReferences(code) };
}
},
};
});
}
return contentConfig;
}

View file

@ -125,6 +125,7 @@ export interface AstroErrorPayload {
// Shiki does not support `mjs` or `cjs` aliases by default.
// Map these to `.js` during error highlighting.
const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs'];
const ALTERNATIVE_MD_EXTS = ['mdoc'];
/**
* Generate a payload for Vite's error overlay
@ -158,6 +159,9 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
if (ALTERNATIVE_JS_EXTS.includes(highlighterLang ?? '')) {
highlighterLang = 'js';
}
if (ALTERNATIVE_MD_EXTS.includes(highlighterLang ?? '')) {
highlighterLang = 'md';
}
const highlightedCode = err.fullCode
? highlighter.codeToHtml(err.fullCode, {
lang: highlighterLang,

View file

@ -237,6 +237,20 @@ const { Content } = await entry.render();
/>
```
### Access frontmatter and content collection information from your templates
You can access content collection information from your Markdoc templates using the `$entry` variable. This includes the entry `slug`, `collection` name, and frontmatter `data` parsed by your content collection schema (if any). This example renders the `title` frontmatter property as a heading:
```md
---
title: Welcome to Markdoc 👋
---
# {% $entry.data.title %}
```
The `$entry` object matches [the `CollectionEntry` type](https://docs.astro.build/en/reference/api-reference/#collection-entry-type), excluding the `.render()` property.
### Markdoc config
The Markdoc integration accepts [all Markdoc configuration options](https://markdoc.dev/docs/config), including [tags](https://markdoc.dev/docs/tags) and [functions](https://markdoc.dev/docs/functions).

View file

@ -3,13 +3,7 @@ import Markdoc from '@markdoc/markdoc';
import type { AstroConfig, AstroIntegration, ContentEntryType, HookParameters } from 'astro';
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import type { InlineConfig } from 'vite';
import {
getAstroConfigPath,
MarkdocError,
parseFrontmatter,
prependForwardSlash,
} from './utils.js';
import { getAstroConfigPath, MarkdocError, parseFrontmatter } from './utils.js';
type SetupHookParams = HookParameters<'astro:config:setup'> & {
// `contentEntryType` is not a public API
@ -36,36 +30,27 @@ export default function markdoc(markdocConfig: Config = {}): AstroIntegration {
addContentEntryType({
extensions: ['.mdoc'],
getEntryInfo,
getRenderModule({ entry }) {
validateRenderProperties(markdocConfig, config);
const ast = Markdoc.parse(entry.body);
const content = Markdoc.transform(ast, {
...markdocConfig,
variables: {
...markdocConfig.variables,
entry,
},
});
return {
code: `import { jsx as h } from 'astro/jsx-runtime';\nimport { Renderer } from '@astrojs/markdoc/components';\nconst transformedContent = ${JSON.stringify(
content
)};\nexport async function Content ({ components }) { return h(Renderer, { content: transformedContent, components }); }\nContent[Symbol.for('astro.needsHeadRendering')] = true;`,
};
},
contentModuleTypes: await fs.promises.readFile(
new URL('../template/content-module-types.d.ts', import.meta.url),
'utf-8'
),
});
const viteConfig: InlineConfig = {
plugins: [
{
name: '@astrojs/markdoc',
async transform(code, id) {
if (!id.endsWith('.mdoc')) return;
validateRenderProperties(markdocConfig, config);
const body = getEntryInfo({
// Can't use `pathToFileUrl` - Vite IDs are not plain file paths
fileUrl: new URL(prependForwardSlash(id), 'file://'),
contents: code,
}).body;
const ast = Markdoc.parse(body);
const content = Markdoc.transform(ast, markdocConfig);
return `import { jsx as h } from 'astro/jsx-runtime';\nimport { Renderer } from '@astrojs/markdoc/components';\nconst transformedContent = ${JSON.stringify(
content
)};\nexport async function Content ({ components }) { return h(Renderer, { content: transformedContent, components }); }\nContent[Symbol.for('astro.needsHeadRendering')] = true;`;
},
},
],
};
updateConfig({ vite: viteConfig });
},
},
};

View file

@ -0,0 +1,58 @@
import { parseHTML } from 'linkedom';
import { expect } from 'chai';
import { loadFixture } from '../../../astro/test/test-utils.js';
import markdoc from '../dist/index.js';
const root = new URL('./fixtures/entry-prop/', import.meta.url);
describe('Markdoc - Entry prop', () => {
let baseFixture;
before(async () => {
baseFixture = await loadFixture({
root,
integrations: [markdoc()],
});
});
describe('dev', () => {
let devServer;
before(async () => {
devServer = await baseFixture.startDevServer();
});
after(async () => {
await devServer.stop();
});
it('has expected entry properties', async () => {
const res = await baseFixture.fetch('/');
const html = await res.text();
const { document } = parseHTML(html);
expect(document.querySelector('h1')?.textContent).to.equal('Processed by schema: Test entry');
expect(document.getElementById('id')?.textContent?.trim()).to.equal('id: entry.mdoc');
expect(document.getElementById('slug')?.textContent?.trim()).to.equal('slug: entry');
expect(document.getElementById('collection')?.textContent?.trim()).to.equal(
'collection: blog'
);
});
});
describe('build', () => {
before(async () => {
await baseFixture.build();
});
it('has expected entry properties', async () => {
const html = await baseFixture.readFile('/index.html');
const { document } = parseHTML(html);
expect(document.querySelector('h1')?.textContent).to.equal('Processed by schema: Test entry');
expect(document.getElementById('id')?.textContent?.trim()).to.equal('id: entry.mdoc');
expect(document.getElementById('slug')?.textContent?.trim()).to.equal('slug: entry');
expect(document.getElementById('collection')?.textContent?.trim()).to.equal(
'collection: blog'
);
});
});
});

View file

@ -0,0 +1,7 @@
import { defineConfig } from 'astro/config';
import markdoc from '@astrojs/markdoc';
// https://astro.build/config
export default defineConfig({
integrations: [markdoc()],
});

View file

@ -0,0 +1,9 @@
{
"name": "@test/markdoc-entry-prop",
"version": "0.0.0",
"private": true,
"dependencies": {
"@astrojs/markdoc": "workspace:*",
"astro": "workspace:*"
}
}

View file

@ -0,0 +1,9 @@
---
title: Test entry
---
# {% $entry.data.title %}
- id: {% $entry.id %} {% #id %}
- slug: {% $entry.slug %} {% #slug %}
- collection: {% $entry.collection %} {% #collection %}

View file

@ -0,0 +1,9 @@
import { defineCollection, z } from 'astro:content';
const blog = defineCollection({
schema: z.object({
title: z.string().transform(v => 'Processed by schema: ' + v),
}),
});
export const collections = { blog }

View file

@ -0,0 +1,19 @@
---
import { getEntryBySlug } from 'astro:content';
const entry = await getEntryBySlug('blog', 'entry');
const { Content } = await entry.render();
---
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width" />
<meta name="generator" content={Astro.generator} />
<title>Astro</title>
</head>
<body>
<Content />
</body>
</html>

View file

@ -3109,6 +3109,14 @@ importers:
devDependencies:
shiki: 0.11.1
packages/integrations/markdoc/test/fixtures/entry-prop:
specifiers:
'@astrojs/markdoc': workspace:*
astro: workspace:*
dependencies:
'@astrojs/markdoc': link:../../..
astro: link:../../../../../astro
packages/integrations/mdx:
specifiers:
'@astrojs/markdown-remark': ^2.1.1