[MDX] Add Prism and Shiki support (#4002)

* deps: add rehype-prism, shiki, rehype-pretty-code

* wip: apply rehype plugins depending on config

* wip: cherry-pick jsx-runtime fix?

* deps: rehype-pretty-code -> shiki-twoslash, add rehype-raw

* wip: add jsx-runtime fix

* feat: get shiki working!

* deps: add @astrojs/prism, prismjs, unist-util-visit

* feat: add prism support

* example: add small syntax highlight demo to with-mdx

* deps: remove rehype-prism

* chore: remove unused async

* chore: add .test.js to all mdx tests

* test: shiki, shikiConfig, prism

* fix: remove "is:raw" from prism output

* docs: add syntax highlighting section

* chore: add changeset

* nit: "Shiki config" -> Shiki config

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* Revert "wip: add jsx-runtime fix"

This reverts commit 07f4528f44.

* docs: link to integration README from example

Co-authored-by: Nate Moore <nate@astro.build>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/real-camels-roll.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': minor
+---
+
+Support Prism and Shiki syntax highlighting based on project config

examples/with-mdx/src/pages/index.mdx

@@ -15,3 +15,15 @@ Written by: {new Intl.ListFormat('en').format(authors.map(d => d.name))}.
 Published on: {new Intl.DateTimeFormat('en', {dateStyle: 'long'}).format(published)}.
 
 <Counter client:idle>This is a **counter**!</Counter>
+
+## Syntax highlighting
+
+We also support syntax highlighting in MDX out-of-the-box! This example uses our default [Shiki theme](https://github.com/shikijs/shiki). See the [MDX integration docs](https://docs.astro.build/en/guides/integrations-guide/mdx/#syntax-highlighting) for configuration options.
+
+```astro
+---
+const weSupportAstro = true
+---
+
+<h1>Hey, what theme is that? Looks nice!</h1>
+```

packages/integrations/mdx/README.md

@@ -134,6 +134,42 @@ const posts = await Astro.glob('./*.mdx');
 ))}
 ```
 
+### Syntax highlighting
+
+The MDX integration respects [your project's `markdown.syntaxHighlight` configuration](https://docs.astro.build/en/guides/markdown-content/#syntax-highlighting).
+
+We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default [using Shiki twoslash](https://shikijs.github.io/twoslash/). You can customize [this remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash) using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
+
+```js
+// astro.config.mjs
+export default {
+  markdown: {
+    shikiConfig: {
+      theme: 'dracula',
+    },
+  },
+  integrations: [mdx()],
+}
+```
+
+Visit [our Shiki configuration docs](https://docs.astro.build/en/guides/markdown-content/#shiki-configuration) for more on using Shiki with Astro.
+
+#### Switch to Prism
+
+You can also use the [Prism](https://prismjs.com/) syntax highlighter by setting `markdown.syntaxHighlight` to `'prism'` in your `astro.config` like so:
+
+```js
+// astro.config.mjs
+export default {
+  markdown: {
+    syntaxHighlight: 'prism',
+  },
+  integrations: [mdx()],
+}
+```
+
+This applies a minimal Prism renderer with added support for `astro` code blocks. Visit [our "Prism configuration" docs](https://docs.astro.build/en/guides/markdown-content/#prism-configuration) for more on using Prism with Astro.
+
 ## Configuration
 
 <details>

packages/integrations/mdx/package.json

@@ -30,12 +30,19 @@
     "test": "mocha --exit --timeout 20000"
   },
   "dependencies": {
+    "@astrojs/prism": "^0.6.1",
+    "@mdx-js/mdx": "^2.1.2",
     "@mdx-js/rollup": "^2.1.1",
     "es-module-lexer": "^0.10.5",
-    "remark-frontmatter": "^4.0.1",
+    "prismjs": "^1.28.0",
+    "rehype-raw": "^6.1.1",
     "remark-gfm": "^3.0.1",
-    "remark-mdx-frontmatter": "^2.0.2",
-    "remark-smartypants": "^2.0.0"
+    "remark-shiki-twoslash": "^3.1.0",
+    "remark-smartypants": "^2.0.0",
+    "shiki": "^0.10.1",
+    "unist-util-visit": "^4.1.0",
+    "remark-frontmatter": "^4.0.1",
+    "remark-mdx-frontmatter": "^2.0.2"
   },
   "devDependencies": {
     "@types/chai": "^4.3.1",

packages/integrations/mdx/src/index.ts

@@ -1,11 +1,15 @@
-import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
+import type { RemarkMdxFrontmatterOptions } from 'remark-mdx-frontmatter';
 import type { AstroIntegration } from 'astro';
+import remarkShikiTwoslash from 'remark-shiki-twoslash';
+import { nodeTypes } from '@mdx-js/mdx';
+import rehypeRaw from 'rehype-raw';
+import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
 import { parse as parseESM } from 'es-module-lexer';
 import remarkFrontmatter from 'remark-frontmatter';
 import remarkGfm from 'remark-gfm';
-import type { RemarkMdxFrontmatterOptions } from 'remark-mdx-frontmatter';
 import remarkMdxFrontmatter from 'remark-mdx-frontmatter';
 import remarkSmartypants from 'remark-smartypants';
+import remarkPrism from './remark-prism.js';
 import { getFileInfo } from './utils.js';
 
 type WithExtends<T> = T | { extends: T };
@@ -23,7 +27,10 @@ type MdxOptions = {
 
 const DEFAULT_REMARK_PLUGINS = [remarkGfm, remarkSmartypants];
 
-function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] = []): T[] {
+function handleExtends<T>(
+	config: WithExtends<T[] | undefined>,
+	defaults: T[] = [],
+): T[] {
 	if (Array.isArray(config)) return config;
 
 	return [...defaults, ...(config?.extends ?? [])];
@@ -35,27 +42,43 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 		hooks: {
 			'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
 				addPageExtension('.mdx');
+				let remarkPlugins = handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS);
+				let rehypePlugins = handleExtends(mdxOptions.rehypePlugins);
+
+				if (config.markdown.syntaxHighlight === 'shiki') {
+					remarkPlugins.push([
+						// Default export still requires ".default" chaining for some reason
+						// Workarounds tried:
+						// - "import * as remarkShikiTwoslash"
+						// - "import { default as remarkShikiTwoslash }"
+						(remarkShikiTwoslash as any).default,
+						config.markdown.shikiConfig,
+					]);
+					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
+				}
+				
+				if (config.markdown.syntaxHighlight === 'prism') {
+					remarkPlugins.push(remarkPrism);
+					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
+				}
+
+				remarkPlugins.push(remarkFrontmatter);
+				remarkPlugins.push([
+					remarkMdxFrontmatter,
+					{
+						name: 'frontmatter',
+						...mdxOptions.frontmatterOptions,
+					},
+				]);
+
 				updateConfig({
 					vite: {
 						plugins: [
 							{
 								enforce: 'pre',
 								...mdxPlugin({
-									remarkPlugins: [
-										...handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS),
-										// Frontmatter plugins should always be applied!
-										// We can revisit this if a strong use case to *remove*
-										// YAML frontmatter via config is reported.
-										remarkFrontmatter,
-										[
-											remarkMdxFrontmatter,
-											{
-												name: 'frontmatter',
-												...mdxOptions.frontmatterOptions,
-											},
-										],
-									],
-									rehypePlugins: handleExtends(mdxOptions.rehypePlugins),
+									remarkPlugins,
+									rehypePlugins,
 									jsx: true,
 									jsxImportSource: 'astro',
 									// Note: disable `.md` support

packages/integrations/mdx/src/remark-prism.ts

@@ -0,0 +1,59 @@
+// TODO: discuss extracting this file to @astrojs/prism
+import { addAstro } from '@astrojs/prism/internal';
+import Prism from 'prismjs';
+import loadLanguages from 'prismjs/components/index.js';
+import { visit } from 'unist-util-visit';
+
+const languageMap = new Map([['ts', 'typescript']]);
+
+function runHighlighter(lang: string, code: string) {
+	let classLanguage = `language-${lang}`;
+
+	if (lang == null) {
+		lang = 'plaintext';
+	}
+
+	const ensureLoaded = (language: string) => {
+		if (language && !Prism.languages[language]) {
+			loadLanguages([language]);
+		}
+	};
+
+	if (languageMap.has(lang)) {
+		ensureLoaded(languageMap.get(lang)!);
+	} else if (lang === 'astro') {
+		ensureLoaded('typescript');
+		addAstro(Prism);
+	} else {
+		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
+		ensureLoaded(lang);
+	}
+
+	if (lang && !Prism.languages[lang]) {
+		// eslint-disable-next-line no-console
+		console.warn(`Unable to load the language: ${lang}`);
+	}
+
+	const grammar = Prism.languages[lang];
+	let html = code;
+	if (grammar) {
+		html = Prism.highlight(code, grammar, lang);
+	}
+
+	return { classLanguage, html };
+}
+
+/**  */
+export default function remarkPrism() {
+	return (tree: any) => visit(tree, 'code', (node: any) => {
+		let { lang, value } = node;
+		node.type = 'html';
+
+		let { html, classLanguage } = runHighlighter(lang, value);
+		let classes = [classLanguage];
+		node.value = `<pre class="${classes.join(
+			' '
+		)}"><code class="${classLanguage}">${html}</code></pre>`;
+		return node;
+	});
+}

packages/integrations/mdx/test/fixtures/mdx-syntax-hightlighting/src/pages/index.mdx

@@ -0,0 +1,9 @@
+# Syntax highlighting
+
+```astro
+---
+const handlesAstroSyntax = true
+---
+
+<h1>{handlesAstroSyntax}</h1>
+```

packages/integrations/mdx/test/mdx-syntax-highlighting.test.js

@@ -0,0 +1,67 @@
+import mdx from '@astrojs/mdx';
+
+import { expect } from 'chai';
+import { parseHTML } from 'linkedom';
+import { loadFixture } from '../../../astro/test/test-utils.js';
+
+const FIXTURE_ROOT = new URL('./fixtures/mdx-syntax-hightlighting/', import.meta.url);
+
+describe('MDX syntax highlighting', () => {
+	describe('shiki', () => {
+		it('works', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'shiki',
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+	
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+	
+			const shikiCodeBlock = document.querySelector('pre.shiki');
+			expect(shikiCodeBlock).to.not.be.null;
+		});
+
+		it('respects markdown.shikiConfig.theme', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'shiki',
+					shikiConfig: {
+						theme: 'dracula',
+					},
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+	
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+	
+			const shikiCodeBlock = document.querySelector('pre.shiki.dracula');
+			expect(shikiCodeBlock).to.not.be.null;
+		});
+	});
+
+	describe('prism', () => {
+		it('works', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'prism',
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+	
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+	
+			const prismCodeBlock = document.querySelector('pre.language-astro');
+			expect(prismCodeBlock).to.not.be.null;
+		});
+	});
+});

pnpm-lock.yaml

@@ -2078,6 +2078,8 @@ importers:
 
   packages/integrations/mdx:
     specifiers:
+      '@astrojs/prism': ^0.6.1
+      '@mdx-js/mdx': ^2.1.2
       '@mdx-js/rollup': ^2.1.1
       '@types/chai': ^4.3.1
       '@types/mocha': ^9.1.1
@@ -2088,18 +2090,30 @@ importers:
       es-module-lexer: ^0.10.5
       linkedom: ^0.14.12
       mocha: ^9.2.2
+      prismjs: ^1.28.0
+      rehype-raw: ^6.1.1
       remark-frontmatter: ^4.0.1
       remark-gfm: ^3.0.1
       remark-mdx-frontmatter: ^2.0.2
+      remark-shiki-twoslash: ^3.1.0
       remark-smartypants: ^2.0.0
       remark-toc: ^8.0.1
+      shiki: ^0.10.1
+      unist-util-visit: ^4.1.0
     dependencies:
+      '@astrojs/prism': link:../../astro-prism
+      '@mdx-js/mdx': 2.1.2
       '@mdx-js/rollup': 2.1.2
       es-module-lexer: 0.10.5
+      prismjs: 1.28.0
+      rehype-raw: 6.1.1
       remark-frontmatter: 4.0.1
       remark-gfm: 3.0.1
       remark-mdx-frontmatter: 2.0.2
+      remark-shiki-twoslash: 3.1.0
       remark-smartypants: 2.0.0
+      shiki: 0.10.1
+      unist-util-visit: 4.1.0
     devDependencies:
       '@types/chai': 4.3.1
       '@types/mocha': 9.1.1
@@ -8467,6 +8481,32 @@ packages:
       eslint-visitor-keys: 3.3.0
     dev: true
 
+  /@typescript/twoslash/3.1.0:
+    resolution: {integrity: sha512-kTwMUQ8xtAZaC4wb2XuLkPqFVBj2dNBueMQ89NWEuw87k2nLBbuafeG5cob/QEr6YduxIdTVUjix0MtC7mPlmg==}
+    dependencies:
+      '@typescript/vfs': 1.3.5
+      debug: 4.3.4
+      lz-string: 1.4.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@typescript/vfs/1.3.4:
+    resolution: {integrity: sha512-RbyJiaAGQPIcAGWFa3jAXSuAexU4BFiDRF1g3hy7LmRqfNpYlTQWGXjcrOaVZjJ8YkkpuwG0FcsYvtWQpd9igQ==}
+    dependencies:
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@typescript/vfs/1.3.5:
+    resolution: {integrity: sha512-pI8Saqjupf9MfLw7w2+og+fmb0fZS0J6vsKXXrp4/PDXEFvntgzXmChCXC/KefZZS0YGS6AT8e0hGAJcTsdJlg==}
+    dependencies:
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /@ungap/promise-all-settled/1.1.2:
     resolution: {integrity: sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==}
     dev: true
@@ -10588,6 +10628,11 @@ packages:
       format: 0.2.2
     dev: false
 
+  /fenceparser/1.1.1:
+    resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
+    engines: {node: '>=12'}
+    dev: false
+
   /fetch-blob/3.2.0:
     resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
     engines: {node: ^12.20 || >= 14.13}
@@ -12002,6 +12047,11 @@ packages:
     dependencies:
       yallist: 4.0.0
 
+  /lz-string/1.4.4:
+    resolution: {integrity: sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==}
+    hasBin: true
+    dev: false
+
   /magic-string/0.25.9:
     resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
     dependencies:
@@ -14260,6 +14310,22 @@ packages:
       unified: 10.1.2
     dev: false
 
+  /remark-shiki-twoslash/3.1.0:
+    resolution: {integrity: sha512-6LqSqVtHQR4S0DKfdQ2/ePn9loTKUtpyopYvwk8johjDTeUW5MkaLQuZHlWNkkST/4aMbz6aTkstIcwfwcHpXg==}
+    dependencies:
+      '@typescript/twoslash': 3.1.0
+      '@typescript/vfs': 1.3.4
+      fenceparser: 1.1.1
+      regenerator-runtime: 0.13.9
+      shiki: 0.10.1
+      shiki-twoslash: 3.1.0
+      tslib: 2.1.0
+      typescript: 4.7.4
+      unist-util-visit: 2.0.3
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /remark-smartypants/2.0.0:
     resolution: {integrity: sha512-Rc0VDmr/yhnMQIz8n2ACYXlfw/P/XZev884QU1I5u+5DgJls32o97Vc1RbK3pfumLsJomS2yy8eT4Fxj/2MDVA==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -14539,6 +14605,17 @@ packages:
     resolution: {integrity: sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==}
     dev: true
 
+  /shiki-twoslash/3.1.0:
+    resolution: {integrity: sha512-uDqrTutOIZzyHbo103GsK7Vvc10saK1XCCivnOQ1NHJzgp3FBilEpftGeVzVSMOJs+JyhI7whkvhXV7kXQ5zCg==}
+    dependencies:
+      '@typescript/twoslash': 3.1.0
+      '@typescript/vfs': 1.3.4
+      shiki: 0.10.1
+      typescript: 4.7.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /shiki/0.10.1:
     resolution: {integrity: sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==}
     dependencies:
@@ -15239,6 +15316,10 @@ packages:
     resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}
     dev: true
 
+  /tslib/2.1.0:
+    resolution: {integrity: sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==}
+    dev: false
+
   /tslib/2.4.0:
     resolution: {integrity: sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==}
 
@@ -15456,7 +15537,6 @@ packages:
     resolution: {integrity: sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==}
     engines: {node: '>=4.2.0'}
     hasBin: true
-    dev: true
 
   /uhyphen/0.1.0:
     resolution: {integrity: sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw==}
@@ -15537,6 +15617,10 @@ packages:
     resolution: {integrity: sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==}
     dev: true
 
+  /unist-util-is/4.1.0:
+    resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==}
+    dev: false
+
   /unist-util-is/5.1.1:
     resolution: {integrity: sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==}
 
@@ -15586,6 +15670,13 @@ packages:
       unist-util-is: 3.0.0
     dev: true
 
+  /unist-util-visit-parents/3.1.1:
+    resolution: {integrity: sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==}
+    dependencies:
+      '@types/unist': 2.0.6
+      unist-util-is: 4.1.0
+    dev: false
+
   /unist-util-visit-parents/4.1.1:
     resolution: {integrity: sha512-1xAFJXAKpnnJl8G7K5KgU7FY55y3GcLIXqkzUj5QF/QVP7biUm0K0O2oqVkYsdjzJKifYeWn9+o6piAK2hGSHw==}
     dependencies:
@@ -15605,6 +15696,14 @@ packages:
       unist-util-visit-parents: 2.1.2
     dev: true
 
+  /unist-util-visit/2.0.3:
+    resolution: {integrity: sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==}
+    dependencies:
+      '@types/unist': 2.0.6
+      unist-util-is: 4.1.0
+      unist-util-visit-parents: 3.1.1
+    dev: false
+
   /unist-util-visit/3.1.0:
     resolution: {integrity: sha512-Szoh+R/Ll68QWAyQyZZpQzZQm2UPbxibDvaY8Xc9SUtYgPsDzx5AWSk++UUt2hJuow8mvwR+rG+LQLw+KsuAKA==}
     dependencies: