[MDX] Add getHeadings + generate anchor links (#4095)

* deps: mdx github-slugger

* feat: add getHeadings via rehype plugin

* chore: stray console.log

* test: getHeadings w/ & w/0 JSX expressions

* docs: add generated exports

* refactor: pass headings using vfile.data

* deps: vfile

* test: heading anchor IDs

* docs: add collect-headings to default rehype plugins

* chore: changeset

* deps: estree-util-value-to-estree

* refactor: inject getHeadings export the right way!

* deps: switch to acorn

* refactor: just use acorn

* docs: `getHeadings` info structuring

Co-authored-by: Chris Swithinbank <swithinbank@gmail.com>

* docs: clarify `url` example

Co-authored-by: Chris Swithinbank <swithinbank@gmail.com>

* fix: move slugger inside plugin call

* refactor: cleanup code reassignment

* chore: lint

* deps: mdast-util-mdx, test utils

* refactor: add jsToTreeNode util

* feat: expose utils for lib authors

* test: rehype plugins w/ and w/o extends

* test: fixture

* refactor: remove utils from package exports

Co-authored-by: Chris Swithinbank <swithinbank@gmail.com>

.changeset/fifty-peas-admire.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': minor
+---
+
+Add IDs to MDX headings and expose via getHeadings() export

packages/integrations/mdx/README.md

@@ -103,6 +103,24 @@ const posts = await Astro.glob('./*.mdx');
 
 See [the official "how MDX works" guide](https://mdxjs.com/docs/using-mdx/#how-mdx-works) for more on MDX variables.
 
+### Exported properties
+
+Alongside your [MDX variable exports](#variables), we generate a few helpful exports as well. These are accessible when importing an MDX file via `import` statements or [`Astro.glob`](https://docs.astro.build/en/reference/api-reference/#astroglob).
+
+#### `file`
+
+The absolute path to the MDX file (e.g. `home/user/projects/.../file.md`).
+
+#### `url`
+
+The browser-ready URL for MDX files under `src/pages/`. For example, `src/pages/en/about.mdx` will provide a `url` of `/en/about/`. For MDX files outside of `src/pages`, `url` will be `undefined`.
+
+#### `getHeadings()`
+
+**Returns:** `{ depth: number; slug: string; text: string }[]`
+
+A function that returns an array of all headings (i.e. `h1 -> h6` elements) in the MDX file. Each heading’s `slug` corresponds to the generated ID for a given heading and can be used for anchor links.
+
 ### Frontmatter
 
 Astro also supports YAML-based frontmatter out-of-the-box using the [remark-mdx-frontmatter](https://github.com/remcohaszing/remark-mdx-frontmatter) plugin. By default, all variables declared in a frontmatter fence (`---`) will be accessible via the `frontmatter` export. See the `frontmatterOptions` configuration to customize this behavior.
@@ -279,11 +297,26 @@ export default {
 <details>
   <summary><strong>rehypePlugins</strong></summary>
 
-**Default plugins:** none
+**Default plugins:** [`collect-headings`](https://github.com/withastro/astro/blob/main/packages/integrations/mdx/src/rehype-collect-headings.ts)
 
 [Rehype plugins](https://github.com/rehypejs/rehype/blob/main/doc/plugins.md) allow you to transform the HTML that your Markdown generates. We recommend checking the [Remark plugin](https://github.com/remarkjs/remark/blob/main/doc/plugins.md) catalog first _before_ considering rehype plugins, since most users want to transform their Markdown syntax instead. If HTML transforms are what you need, we encourage you to browse [awesome-rehype](https://github.com/rehypejs/awesome-rehype) for a full curated list of plugins!
 
-To apply rehype plugins, use the `rehypePlugins` configuration option like so:
+We apply our own [`collect-headings`](https://github.com/withastro/astro/blob/main/packages/integrations/mdx/src/rehype-collect-headings.ts) plugin by default. This applies IDs to all headings (i.e. `h1 -> h6`) in your MDX files to [link to headings via anchor tags](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a#linking_to_an_element_on_the_same_page).
+
+To apply rehype plugins _while preserving_ Astro's default plugins, use a nested `extends` object like so:
+
+```js
+// astro.config.mjs
+import rehypeMinifyHtml from 'rehype-minify';
+
+export default {
+  integrations: [mdx({
+    rehypePlugins: { extends: [rehypeMinifyHtml] },
+  })],
+}
+```
+
+To apply plugins _without_ Astro's defaults, you can apply a plain array:
 
 ```js
 // astro.config.mjs

packages/integrations/mdx/package.json

@@ -33,8 +33,11 @@
     "@astrojs/prism": "^0.6.1",
     "@mdx-js/mdx": "^2.1.2",
     "@mdx-js/rollup": "^2.1.1",
+    "acorn": "^8.8.0",
     "es-module-lexer": "^0.10.5",
+    "github-slugger": "^1.4.0",
     "gray-matter": "^4.0.3",
+    "mdast-util-mdx": "^2.0.0",
     "prismjs": "^1.28.0",
     "rehype-raw": "^6.1.1",
     "remark-frontmatter": "^4.0.1",
@@ -53,7 +56,9 @@
     "astro-scripts": "workspace:*",
     "chai": "^4.3.6",
     "linkedom": "^0.14.12",
+    "mdast-util-to-string": "^3.1.0",
     "mocha": "^9.2.2",
+    "reading-time": "^1.5.0",
     "remark-toc": "^8.0.1"
   },
   "engines": {

packages/integrations/mdx/src/index.ts

@@ -1,4 +1,5 @@
-import { nodeTypes } from '@mdx-js/mdx';
+import { nodeTypes, compile as mdxCompile } from '@mdx-js/mdx';
+import { VFile } from 'vfile';
 import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
 import type { AstroIntegration } from 'astro';
 import { parse as parseESM } from 'es-module-lexer';
@@ -12,6 +13,7 @@ import remarkSmartypants from 'remark-smartypants';
 import type { Plugin as VitePlugin } from 'vite';
 import remarkPrism from './remark-prism.js';
 import { getFileInfo, getFrontmatter } from './utils.js';
+import rehypeCollectHeadings from './rehype-collect-headings.js';
 
 type WithExtends<T> = T | { extends: T };
 
@@ -27,6 +29,7 @@ type MdxOptions = {
 };
 
 const DEFAULT_REMARK_PLUGINS = [remarkGfm, remarkSmartypants];
+const DEFAULT_REHYPE_PLUGINS = [rehypeCollectHeadings];
 
 function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] = []): T[] {
 	if (Array.isArray(config)) return config;
@@ -41,7 +44,7 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 			'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
 				addPageExtension('.mdx');
 				let remarkPlugins = handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS);
-				let rehypePlugins = handleExtends(mdxOptions.rehypePlugins);
+				let rehypePlugins = handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS);
 
 				if (config.markdown.syntaxHighlight === 'shiki') {
 					remarkPlugins.push([
@@ -69,7 +72,7 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 					},
 				]);
 
-				const configuredMdxPlugin = mdxPlugin({
+				const mdxPluginOpts: MdxRollupPluginOptions = {
 					remarkPlugins,
 					rehypePlugins,
 					jsx: true,
@@ -77,38 +80,47 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 					// Note: disable `.md` support
 					format: 'mdx',
 					mdExtensions: [],
-				});
+				};
 
 				updateConfig({
 					vite: {
 						plugins: [
 							{
 								enforce: 'pre',
-								...configuredMdxPlugin,
-								// Override transform to inject layouts before MDX compilation
-								async transform(this, code, id) {
-									if (!id.endsWith('.mdx')) return;
+								...mdxPlugin(mdxPluginOpts),
+								// Override transform to alter code before MDX compilation
+								// ex. inject layouts
+								async transform(code, id) {
+									if (!id.endsWith('mdx')) return;
 
-									const mdxPluginTransform = configuredMdxPlugin.transform?.bind(this);
 									// If user overrides our default YAML parser,
 									// do not attempt to parse the `layout` via gray-matter
-									if (mdxOptions.frontmatterOptions?.parsers) {
-										return mdxPluginTransform?.(code, id);
+									if (!mdxOptions.frontmatterOptions?.parsers) {
+										const frontmatter = getFrontmatter(code, id);
+										if (frontmatter.layout) {
+											const { layout, ...content } = frontmatter;
+											code += `\nexport default async function({ children }) {\nconst Layout = (await import(${JSON.stringify(
+												frontmatter.layout
+											)})).default;\nreturn <Layout content={${JSON.stringify(
+												content
+											)}}>{children}</Layout> }`;
+										}
 									}
-									const frontmatter = getFrontmatter(code, id);
-									if (frontmatter.layout) {
-										const { layout, ...content } = frontmatter;
-										code += `\nexport default async function({ children }) {\nconst Layout = (await import(${JSON.stringify(
-											frontmatter.layout
-										)})).default;\nreturn <Layout content={${JSON.stringify(
-											content
-										)}}>{children}</Layout> }`;
-									}
-									return mdxPluginTransform?.(code, id);
+
+									const compiled = await mdxCompile(
+										new VFile({ value: code, path: id }),
+										mdxPluginOpts
+									);
+
+									return {
+										code: String(compiled.value),
+										map: compiled.map,
+									};
 								},
 							},
 							{
-								name: '@astrojs/mdx',
+								name: '@astrojs/mdx-postprocess',
+								// These transforms must happen *after* JSX runtime transformations
 								transform(code, id) {
 									if (!id.endsWith('.mdx')) return;
 									const [, moduleExports] = parseESM(code);

packages/integrations/mdx/src/rehype-collect-headings.ts

@@ -0,0 +1,50 @@
+import Slugger from 'github-slugger';
+import { visit } from 'unist-util-visit';
+import { jsToTreeNode } from './utils.js';
+
+export interface MarkdownHeading {
+	depth: number;
+	slug: string;
+	text: string;
+}
+
+export default function rehypeCollectHeadings() {
+	const slugger = new Slugger();
+	return function (tree: any) {
+		const headings: MarkdownHeading[] = [];
+		visit(tree, (node) => {
+			if (node.type !== 'element') return;
+			const { tagName } = node;
+			if (tagName[0] !== 'h') return;
+			const [_, level] = tagName.match(/h([0-6])/) ?? [];
+			if (!level) return;
+			const depth = Number.parseInt(level);
+
+			let text = '';
+			visit(node, (child, __, parent) => {
+				if (child.type === 'element' || parent == null) {
+					return;
+				}
+				if (child.type === 'raw' && child.value.match(/^\n?<.*>\n?$/)) {
+					return;
+				}
+				if (new Set(['text', 'raw', 'mdxTextExpression']).has(child.type)) {
+					text += child.value;
+				}
+			});
+
+			node.properties = node.properties || {};
+			if (typeof node.properties.id !== 'string') {
+				let slug = slugger.slug(text);
+				if (slug.endsWith('-')) {
+					slug = slug.slice(0, -1);
+				}
+				node.properties.id = slug;
+			}
+			headings.push({ depth, slug: node.properties.id, text });
+		});
+		tree.children.unshift(
+			jsToTreeNode(`export function getHeadings() { return ${JSON.stringify(headings)} }`)
+		);
+	};
+}

packages/integrations/mdx/src/utils.ts

@@ -1,4 +1,8 @@
 import type { AstroConfig, SSRError } from 'astro';
+import type { Options as AcornOpts } from 'acorn';
+import type { MdxjsEsm } from 'mdast-util-mdx';
+import { parse } from 'acorn';
+
 import matter from 'gray-matter';
 
 function appendForwardSlash(path: string) {
@@ -58,3 +62,24 @@ export function getFrontmatter(code: string, id: string) {
 		}
 	}
 }
+
+export function jsToTreeNode(
+	jsString: string,
+	acornOpts: AcornOpts = {
+		ecmaVersion: 'latest',
+		sourceType: 'module',
+	}
+): MdxjsEsm {
+	return {
+		type: 'mdxjsEsm',
+		value: '',
+		data: {
+			estree: {
+				body: [],
+				...parse(jsString, acornOpts),
+				type: 'Program',
+				sourceType: 'module',
+			},
+		},
+	};
+}

packages/integrations/mdx/test/fixtures/mdx-get-headings/src/pages/pages.json.js

@@ -0,0 +1,11 @@
+export async function get() {
+	const mdxPages = await import.meta.glob('./*.mdx', { eager: true });
+
+	return {
+		body: JSON.stringify({
+			headingsByPage: Object.fromEntries(
+				Object.entries(mdxPages ?? {}).map(([k, v]) => [k, v?.getHeadings()])
+			),
+		}),
+	}
+}

packages/integrations/mdx/test/fixtures/mdx-get-headings/src/pages/test-with-jsx-expressions.mdx

@@ -0,0 +1,8 @@
+export const h2Title = "Section 1"
+export const h3Title = "Subsection 1"
+
+# Heading test with JSX expressions
+
+## {h2Title}
+
+### {h3Title}

packages/integrations/mdx/test/fixtures/mdx-get-headings/src/pages/test.mdx

@@ -0,0 +1,9 @@
+# Heading test
+
+## Section 1
+
+### Subsection 1
+
+### Subsection 2
+
+## Section 2

packages/integrations/mdx/test/fixtures/mdx-rehype-plugins/src/pages/reading-time.json.js

@@ -0,0 +1,7 @@
+import { readingTime } from './space-ipsum.mdx';
+
+export function get() {
+	return {
+		body: JSON.stringify(readingTime),
+	}
+}

packages/integrations/mdx/test/fixtures/mdx-rehype-plugins/src/pages/space-ipsum.mdx

@@ -0,0 +1,25 @@
+# Space ipsum
+
+For those who have seen the Earth from space, and for the hundreds and perhaps thousands more who will, the experience most certainly changes your perspective. The things that we share in our world are far more valuable than those which divide us.
+
+It suddenly struck me that that tiny pea, pretty and blue, was the Earth. I put up my thumb and shut one eye, and my thumb blotted out the planet Earth. I didn’t feel like a giant. I felt very, very small.
+
+Science has not yet mastered prophecy. We predict too much for the next year and yet far too little for the next 10.
+
+## Section 2
+
+We choose to go to the moon in this decade and do the other things, not because they are easy, but because they are hard, because that goal will serve to organize and measure the best of our energies and skills, because that challenge is one that we are willing to accept, one we are unwilling to postpone, and one which we intend to win.
+
+There can be no thought of finishing for ‘aiming for the stars.’ Both figuratively and literally, it is a task to occupy the generations. And no matter how much progress one makes, there is always the thrill of just beginning.
+
+As I stand out here in the wonders of the unknown at Hadley, I sort of realize there’s a fundamental truth to our nature, Man must explore . . . and this is exploration at its greatest.
+
+## Section 3
+
+Never in all their history have men been able truly to conceive of the world as one: a single sphere, a globe, having the qualities of a globe, a round earth in which all the directions eventually meet, in which there is no center because every point, or none, is center — an equal earth which all men occupy as equals. The airman’s earth, if free men make it, will be truly round: a globe in practice, not in theory.
+
+To be the first to enter the cosmos, to engage, single-handed, in an unprecedented duel with nature—could one dream of anything more?
+
+There can be no thought of finishing for ‘aiming for the stars.’ Both figuratively and literally, it is a task to occupy the generations. And no matter how much progress one makes, there is always the thrill of just beginning.
+
+We are all connected; To each other, biologically. To the earth, chemically. To the rest of the universe atomically.

packages/integrations/mdx/test/mdx-get-headings.test.js

@@ -0,0 +1,56 @@
+import mdx from '@astrojs/mdx';
+
+import { expect } from 'chai';
+import { parseHTML } from 'linkedom';
+import { loadFixture } from '../../../astro/test/test-utils.js';
+
+describe('MDX getHeadings', () => {
+	let fixture;
+
+	before(async () => {
+		fixture = await loadFixture({
+			root: new URL('./fixtures/mdx-get-headings/', import.meta.url),
+			integrations: [mdx()],
+		});
+
+		await fixture.build();
+	});
+
+	it('adds anchor IDs to headings', async () => {
+		const html = await fixture.readFile('/test/index.html');
+		const { document } = parseHTML(html);
+
+		const h2Ids = document.querySelectorAll('h2').map(el => el?.id);
+		const h3Ids = document.querySelectorAll('h3').map(el => el?.id);
+		expect(document.querySelector('h1').id).to.equal('heading-test');
+		expect(h2Ids).to.contain('section-1');
+		expect(h2Ids).to.contain('section-2');
+		expect(h3Ids).to.contain('subsection-1');
+		expect(h3Ids).to.contain('subsection-2');
+	});
+
+	it('generates correct getHeadings() export', async () => {
+		const { headingsByPage } = JSON.parse(await fixture.readFile('/pages.json'));
+		// TODO: make this a snapshot test :)
+		expect(JSON.stringify(headingsByPage['./test.mdx'])).to.equal(JSON.stringify([
+			{ depth: 1, slug: 'heading-test', text: 'Heading test' },
+			{ depth: 2, slug: 'section-1', text: 'Section 1' },
+			{ depth: 3, slug: 'subsection-1', text: 'Subsection 1' },
+			{ depth: 3, slug: 'subsection-2', text: 'Subsection 2' },
+			{ depth: 2, slug: 'section-2', text: 'Section 2' }
+		]));
+	});
+	
+	it('generates correct getHeadings() export for JSX expressions', async () => {
+		const { headingsByPage } = JSON.parse(await fixture.readFile('/pages.json'));
+		expect(JSON.stringify(headingsByPage['./test-with-jsx-expressions.mdx'])).to.equal(JSON.stringify([
+			{
+				depth: 1,
+				slug: 'heading-test-with-jsx-expressions',
+				text: 'Heading test with JSX expressions'
+			},
+			{ depth: 2, slug: 'h2title', text: 'h2Title' },
+			{ depth: 3, slug: 'h3title', text: 'h3Title' }
+		]));
+	});
+});

packages/integrations/mdx/test/mdx-rehype-plugins.test.js

@@ -0,0 +1,81 @@
+import mdx from '@astrojs/mdx';
+import { jsToTreeNode } from '../dist/utils.js';
+
+import { expect } from 'chai';
+import { parseHTML } from 'linkedom';
+import getReadingTime from 'reading-time';
+import { toString } from 'mdast-util-to-string';
+
+import { loadFixture } from '../../../astro/test/test-utils.js';
+
+export function rehypeReadingTime() {
+  return function (tree) {
+		const readingTime = getReadingTime(toString(tree))
+		tree.children.unshift(
+			jsToTreeNode(`export const readingTime = ${JSON.stringify(readingTime)}`)
+		)
+  };
+}
+
+const FIXTURE_ROOT = new URL('./fixtures/mdx-rehype-plugins/', import.meta.url);
+
+describe('MDX rehype plugins', () => {
+	describe('without "extends"', () => {
+		let fixture;
+		before(async () => {
+			fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				integrations: [
+					mdx({
+						rehypePlugins: [rehypeReadingTime],
+					}),
+				],
+			});
+			await fixture.build();
+		});
+
+		it('removes default getHeadings', async () => {
+			const html = await fixture.readFile('/space-ipsum/index.html');
+			const { document } = parseHTML(html);
+	
+			const headings = [...document.querySelectorAll('h1, h2')];
+			expect(headings.length).to.be.greaterThan(0);
+			for (const heading of headings) {
+				expect(heading.id).to.be.empty;
+			}
+		});
+
+		it('supports custom rehype plugins - reading time', async () => {
+			const readingTime = JSON.parse(await fixture.readFile('/reading-time.json'));
+	
+			expect(readingTime).to.not.be.null;
+			expect(readingTime.text).to.match(/^\d+ min read/);
+		});
+	});
+
+	describe('with "extends"', () => {
+		let fixture;
+		before(async () => {
+			fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				integrations: [
+					mdx({
+						rehypePlugins: { extends: [rehypeReadingTime] },
+					}),
+				],
+			});
+			await fixture.build();
+		});
+
+		it('preserves default getHeadings', async () => {
+			const html = await fixture.readFile('/space-ipsum/index.html');
+			const { document } = parseHTML(html);
+	
+			const headings = [...document.querySelectorAll('h1, h2')];
+			expect(headings.length).to.be.greaterThan(0);
+			for (const heading of headings) {
+				expect(heading.id).to.not.be.empty;
+			}
+		});
+	});
+});

pnpm-lock.yaml

@@ -2178,14 +2178,19 @@ importers:
       '@types/chai': ^4.3.1
       '@types/mocha': ^9.1.1
       '@types/yargs-parser': ^21.0.0
+      acorn: ^8.8.0
       astro: workspace:*
       astro-scripts: workspace:*
       chai: ^4.3.6
       es-module-lexer: ^0.10.5
+      github-slugger: ^1.4.0
       gray-matter: ^4.0.3
       linkedom: ^0.14.12
+      mdast-util-mdx: ^2.0.0
+      mdast-util-to-string: ^3.1.0
       mocha: ^9.2.2
       prismjs: ^1.28.0
+      reading-time: ^1.5.0
       rehype-raw: ^6.1.1
       remark-frontmatter: ^4.0.1
       remark-gfm: ^3.0.1
@@ -2199,8 +2204,11 @@ importers:
       '@astrojs/prism': link:../../astro-prism
       '@mdx-js/mdx': 2.1.2
       '@mdx-js/rollup': 2.1.2
+      acorn: 8.8.0
       es-module-lexer: 0.10.5
+      github-slugger: 1.4.0
       gray-matter: 4.0.3
+      mdast-util-mdx: 2.0.0
       prismjs: 1.28.0
       rehype-raw: 6.1.1
       remark-frontmatter: 4.0.1
@@ -2218,7 +2226,9 @@ importers:
       astro-scripts: link:../../../scripts
       chai: 4.3.6
       linkedom: 0.14.12
+      mdast-util-to-string: 3.1.0
       mocha: 9.2.2
+      reading-time: 1.5.0
       remark-toc: 8.0.1
 
   packages/integrations/netlify:
@@ -14728,6 +14738,10 @@ packages:
     dependencies:
       picomatch: 2.3.1
 
+  /reading-time/1.5.0:
+    resolution: {integrity: sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg==}
+    dev: true
+
   /recast/0.20.5:
     resolution: {integrity: sha512-E5qICoPoNL4yU0H0NoBDntNB0Q5oMSNh9usFctYniLBluTthi3RsQVBXIJNbApOlvSwW/RGxIuokPcAc59J5fQ==}
     engines: {node: '>= 4'}