[MDX] Switch from Shiki Twoslash -> Astro Markdown highlighter (#4292)

* freat: twoslash -> Astro shiki parser

* test: update shiki style check

* feat: always apply rehypeRaw

* deps: move remark-shiki-twoslash to dev

* test: add shiki-twoslash test

* docs: update readme with twoslash example

* chore: changeset

* nit: remove "describe('disabled')"

.changeset/grumpy-wombats-laugh.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': minor
+---
+
+Switch from Shiki Twoslash to Astro's Shiki Markdown highlighter

packages/integrations/mdx/README.md

@@ -253,7 +253,7 @@ const { title, fancyJsHelper } = Astro.props;
 
 The MDX integration respects [your project's `markdown.syntaxHighlight` configuration](https://docs.astro.build/en/guides/markdown-content/#syntax-highlighting).
 
-We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default [using Shiki twoslash](https://shikijs.github.io/twoslash/). You can customize [this remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash) using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
+We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default. You can customize this highlighter using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
 
 ```js
 // astro.config.mjs
@@ -285,6 +285,23 @@ export default {
 
 This applies a minimal Prism renderer with added support for `astro` code blocks. Visit [our "Prism configuration" docs](https://docs.astro.build/en/guides/markdown-content/#prism-configuration) for more on using Prism with Astro.
 
+#### Switch to a custom syntax highlighter
+
+You may want to apply your own syntax highlighter too. If your highlighter offers a remark or rehype plugin, you can flip off our syntax highlighting by setting `markdown.syntaxHighlight: false` and wiring up your plugin. For example, say you want to apply [Shiki Twoslash's remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash):
+
+```js
+// astro.config.mjs
+import shikiTwoslash from 'remark-shiki-twoslash';
+
+export default {
+  markdown: {
+  syntaxHighlight: false,
+  },
+  integrations: [mdx({
+    remarkPlugins: [shikiTwoslash, { /* Shiki Twoslash config */ }],
+  })],
+```
+
 ## Configuration
 
 ### remarkPlugins

packages/integrations/mdx/package.json

@@ -39,7 +39,6 @@
     "rehype-raw": "^6.1.1",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
-    "remark-shiki-twoslash": "^3.1.0",
     "remark-smartypants": "^2.0.0",
     "shiki": "^0.10.1",
     "unist-util-visit": "^4.1.0",
@@ -56,6 +55,7 @@
     "mdast-util-to-string": "^3.1.0",
     "mocha": "^9.2.2",
     "reading-time": "^1.5.0",
+    "remark-shiki-twoslash": "^3.1.0",
     "remark-toc": "^8.0.1"
   },
   "engines": {

packages/integrations/mdx/src/index.ts

@@ -4,13 +4,13 @@ import type { AstroConfig, AstroIntegration } from 'astro';
 import { parse as parseESM } from 'es-module-lexer';
 import rehypeRaw from 'rehype-raw';
 import remarkGfm from 'remark-gfm';
-import remarkShikiTwoslash from 'remark-shiki-twoslash';
 import remarkSmartypants from 'remark-smartypants';
 import { VFile } from 'vfile';
 import type { Plugin as VitePlugin } from 'vite';
 import { rehypeApplyFrontmatterExport, remarkInitializeAstroData } from './astro-data-utils.js';
 import rehypeCollectHeadings from './rehype-collect-headings.js';
 import remarkPrism from './remark-prism.js';
+import remarkShiki from './remark-shiki.js';
 import { getFileInfo, parseFrontmatter } from './utils.js';
 
 type WithExtends<T> = T | { extends: T };
@@ -38,22 +38,17 @@ function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] =
 	return [...defaults, ...(config?.extends ?? [])];
 }
 
-function getRemarkPlugins(
+async function getRemarkPlugins(
 	mdxOptions: MdxOptions,
 	config: AstroConfig
-): MdxRollupPluginOptions['remarkPlugins'] {
+): Promise<MdxRollupPluginOptions['remarkPlugins']> {
 	let remarkPlugins = [
 		// Initialize vfile.data.astroExports before all plugins are run
 		remarkInitializeAstroData,
 		...handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS),
 	];
 	if (config.markdown.syntaxHighlight === 'shiki') {
-		// Default export still requires ".default" chaining for some reason
+		remarkPlugins.push([await remarkShiki(config.markdown.shikiConfig)]);
-		// Workarounds tried:
-		// - "import * as remarkShikiTwoslash"
-		// - "import { default as remarkShikiTwoslash }"
-		const shikiTwoslash = (remarkShikiTwoslash as any).default ?? remarkShikiTwoslash;
-		remarkPlugins.push([shikiTwoslash, config.markdown.shikiConfig]);
 	}
 	if (config.markdown.syntaxHighlight === 'prism') {
 		remarkPlugins.push(remarkPrism);
@@ -65,11 +60,11 @@ function getRehypePlugins(
 	mdxOptions: MdxOptions,
 	config: AstroConfig
 ): MdxRollupPluginOptions['rehypePlugins'] {
-	let rehypePlugins = handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS);
+	let rehypePlugins = [
+		[rehypeRaw, { passThrough: nodeTypes }] as any,
+		...handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS),
+	];
 
-	if (config.markdown.syntaxHighlight === 'shiki' || config.markdown.syntaxHighlight === 'prism') {
-		rehypePlugins.unshift([rehypeRaw, { passThrough: nodeTypes }]);
-	}
 	// getHeadings() is guaranteed by TS, so we can't allow user to override
 	rehypePlugins.unshift(rehypeCollectHeadings);
 
@@ -80,11 +75,11 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 	return {
 		name: '@astrojs/mdx',
 		hooks: {
-			'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
+			'astro:config:setup': async ({ updateConfig, config, addPageExtension, command }: any) => {
 				addPageExtension('.mdx');
 
 				const mdxPluginOpts: MdxRollupPluginOptions = {
-					remarkPlugins: getRemarkPlugins(mdxOptions, config),
+					remarkPlugins: await getRemarkPlugins(mdxOptions, config),
 					rehypePlugins: getRehypePlugins(mdxOptions, config),
 					jsx: true,
 					jsxImportSource: 'astro',

packages/integrations/mdx/src/remark-shiki.ts

@@ -0,0 +1,85 @@
+import type { ShikiConfig } from 'astro';
+import type * as shiki from 'shiki';
+import { getHighlighter } from 'shiki';
+import { visit } from 'unist-util-visit';
+
+/**
+ * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
+ * cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
+ * We make this async, so that multiple calls to parse markdown still share the same highlighter.
+ */
+const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
+
+const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
+	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
+	let highlighterAsync = highlighterCacheAsync.get(cacheID);
+	if (!highlighterAsync) {
+		highlighterAsync = getHighlighter({ theme });
+		highlighterCacheAsync.set(cacheID, highlighterAsync);
+	}
+	const highlighter = await highlighterAsync;
+
+	// NOTE: There may be a performance issue here for large sites that use `lang`.
+	// Since this will be called on every page load. Unclear how to fix this.
+	for (const lang of langs) {
+		await highlighter.loadLanguage(lang);
+	}
+
+	return () => (tree: any) => {
+		visit(tree, 'code', (node) => {
+			let lang: string;
+
+			if (typeof node.lang === 'string') {
+				const langExists = highlighter.getLoadedLanguages().includes(node.lang);
+				if (langExists) {
+					lang = node.lang;
+				} else {
+					// eslint-disable-next-line no-console
+					console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
+					lang = 'plaintext';
+				}
+			} else {
+				lang = 'plaintext';
+			}
+
+			let html = highlighter!.codeToHtml(node.value, { lang });
+
+			// Q: Couldn't these regexes match on a user's inputted code blocks?
+			// A: Nope! All rendered HTML is properly escaped.
+			// Ex. If a user typed `<span class="line"` into a code block,
+			// It would become this before hitting our regexes:
+			// &lt;span class=&quot;line&quot;
+
+			// Replace "shiki" class naming with "astro".
+			html = html.replace('<pre class="shiki"', `<pre class="astro-code"`);
+			// Replace "shiki" css variable naming with "astro".
+			html = html.replace(
+				/style="(background-)?color: var\(--shiki-/g,
+				'style="$1color: var(--astro-code-'
+			);
+			// Add "user-select: none;" for "+"/"-" diff symbols
+			if (node.lang === 'diff') {
+				html = html.replace(
+					/<span class="line"><span style="(.*?)">([\+|\-])/g,
+					'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
+				);
+			}
+			// Handle code wrapping
+			// if wrap=null, do nothing.
+			if (wrap === false) {
+				html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
+			} else if (wrap === true) {
+				html = html.replace(
+					/style="(.*?)"/,
+					'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
+				);
+			}
+
+			node.type = 'html';
+			node.value = html;
+			node.children = [];
+		});
+	};
+};
+
+export default remarkShiki;

packages/integrations/mdx/test/mdx-syntax-highlighting.test.js

@@ -3,6 +3,7 @@ import mdx from '@astrojs/mdx';
 import { expect } from 'chai';
 import { parseHTML } from 'linkedom';
 import { loadFixture } from '../../../astro/test/test-utils.js';
+import shikiTwoslash from 'remark-shiki-twoslash';
 
 const FIXTURE_ROOT = new URL('./fixtures/mdx-syntax-hightlighting/', import.meta.url);
 
@@ -21,8 +22,9 @@ describe('MDX syntax highlighting', () => {
 			const html = await fixture.readFile('/index.html');
 			const { document } = parseHTML(html);
 
-			const shikiCodeBlock = document.querySelector('pre.shiki');
+			const shikiCodeBlock = document.querySelector('pre.astro-code');
 			expect(shikiCodeBlock).to.not.be.null;
+			expect(shikiCodeBlock.getAttribute('style')).to.contain('background-color:#0d1117');
 		});
 
 		it('respects markdown.shikiConfig.theme', async () => {
@@ -41,8 +43,9 @@ describe('MDX syntax highlighting', () => {
 			const html = await fixture.readFile('/index.html');
 			const { document } = parseHTML(html);
 
-			const shikiCodeBlock = document.querySelector('pre.shiki.dracula');
+			const shikiCodeBlock = document.querySelector('pre.astro-code');
 			expect(shikiCodeBlock).to.not.be.null;
+			expect(shikiCodeBlock.getAttribute('style')).to.contain('background-color:#282A36');
 		});
 	});
 
@@ -64,4 +67,23 @@ describe('MDX syntax highlighting', () => {
 			expect(prismCodeBlock).to.not.be.null;
 		});
 	});
+
+	it('supports custom highlighter - shiki-twoslash', async () => {
+		const fixture = await loadFixture({
+			root: FIXTURE_ROOT,
+			markdown: {
+				syntaxHighlight: false,
+			},
+			integrations: [mdx({
+				remarkPlugins: [shikiTwoslash.default ?? shikiTwoslash],
+			})],
+		});
+		await fixture.build();
+
+		const html = await fixture.readFile('/index.html');
+		const { document } = parseHTML(html);
+
+		const twoslashCodeBlock = document.querySelector('pre.shiki');
+		expect(twoslashCodeBlock).to.not.be.null;
+	});
 });

pnpm-lock.yaml

@@ -2268,7 +2268,6 @@ importers:
       rehype-raw: 6.1.1
       remark-frontmatter: 4.0.1
       remark-gfm: 3.0.1
-      remark-shiki-twoslash: 3.1.0
       remark-smartypants: 2.0.0
       shiki: 0.10.1
       unist-util-visit: 4.1.0
@@ -2284,6 +2283,7 @@ importers:
       mdast-util-to-string: 3.1.0
       mocha: 9.2.2
       reading-time: 1.5.0
+      remark-shiki-twoslash: 3.1.0
       remark-toc: 8.0.1
 
   packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection:
@@ -8939,7 +8939,7 @@ packages:
       lz-string: 1.4.4
     transitivePeerDependencies:
       - supports-color
-    dev: false
+    dev: true
 
   /@typescript/vfs/1.3.4:
     resolution: {integrity: sha512-RbyJiaAGQPIcAGWFa3jAXSuAexU4BFiDRF1g3hy7LmRqfNpYlTQWGXjcrOaVZjJ8YkkpuwG0FcsYvtWQpd9igQ==}
@@ -8947,7 +8947,7 @@ packages:
       debug: 4.3.4
     transitivePeerDependencies:
       - supports-color
-    dev: false
+    dev: true
 
   /@typescript/vfs/1.3.5:
     resolution: {integrity: sha512-pI8Saqjupf9MfLw7w2+og+fmb0fZS0J6vsKXXrp4/PDXEFvntgzXmChCXC/KefZZS0YGS6AT8e0hGAJcTsdJlg==}
@@ -8955,7 +8955,7 @@ packages:
       debug: 4.3.4
     transitivePeerDependencies:
       - supports-color
-    dev: false
+    dev: true
 
   /@ungap/promise-all-settled/1.1.2:
     resolution: {integrity: sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==}
@@ -11297,7 +11297,7 @@ packages:
   /fenceparser/1.1.1:
     resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
     engines: {node: '>=12'}
-    dev: false
+    dev: true
 
   /fetch-blob/3.2.0:
     resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
@@ -12444,7 +12444,6 @@ packages:
 
   /jsonc-parser/3.1.0:
     resolution: {integrity: sha512-DRf0QjnNeCUds3xTjKlQQ3DpJD51GvDjJfnxUVWg6PZTo2otSm+slzNAxU/35hF8/oJIKoG9slq30JYOsF2azg==}
-    dev: false
 
   /jsonfile/4.0.0:
     resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
@@ -12660,7 +12659,7 @@ packages:
   /lz-string/1.4.4:
     resolution: {integrity: sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==}
     hasBin: true
-    dev: false
+    dev: true
 
   /magic-string/0.25.9:
     resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
@@ -15030,7 +15029,7 @@ packages:
       unist-util-visit: 2.0.3
     transitivePeerDependencies:
       - supports-color
-    dev: false
+    dev: true
 
   /remark-smartypants/2.0.0:
     resolution: {integrity: sha512-Rc0VDmr/yhnMQIz8n2ACYXlfw/P/XZev884QU1I5u+5DgJls32o97Vc1RbK3pfumLsJomS2yy8eT4Fxj/2MDVA==}
@@ -15361,7 +15360,7 @@ packages:
       typescript: 4.7.4
     transitivePeerDependencies:
       - supports-color
-    dev: false
+    dev: true
 
   /shiki/0.10.1:
     resolution: {integrity: sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==}
@@ -15369,7 +15368,6 @@ packages:
       jsonc-parser: 3.1.0
       vscode-oniguruma: 1.6.2
       vscode-textmate: 5.2.0
-    dev: false
 
   /side-channel/1.0.4:
     resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==}
@@ -16086,7 +16084,7 @@ packages:
 
   /tslib/2.1.0:
     resolution: {integrity: sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==}
-    dev: false
+    dev: true
 
   /tslib/2.4.0:
     resolution: {integrity: sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==}
@@ -16313,6 +16311,7 @@ packages:
     resolution: {integrity: sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==}
     engines: {node: '>=4.2.0'}
     hasBin: true
+    dev: true
 
   /uhyphen/0.1.0:
     resolution: {integrity: sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw==}
@@ -16401,7 +16400,7 @@ packages:
 
   /unist-util-is/4.1.0:
     resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==}
-    dev: false
+    dev: true
 
   /unist-util-is/5.1.1:
     resolution: {integrity: sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==}
@@ -16457,7 +16456,7 @@ packages:
     dependencies:
       '@types/unist': 2.0.6
       unist-util-is: 4.1.0
-    dev: false
+    dev: true
 
   /unist-util-visit-parents/4.1.1:
     resolution: {integrity: sha512-1xAFJXAKpnnJl8G7K5KgU7FY55y3GcLIXqkzUj5QF/QVP7biUm0K0O2oqVkYsdjzJKifYeWn9+o6piAK2hGSHw==}
@@ -16484,7 +16483,7 @@ packages:
       '@types/unist': 2.0.6
       unist-util-is: 4.1.0
       unist-util-visit-parents: 3.1.1
-    dev: false
+    dev: true
 
   /unist-util-visit/3.1.0:
     resolution: {integrity: sha512-Szoh+R/Ll68QWAyQyZZpQzZQm2UPbxibDvaY8Xc9SUtYgPsDzx5AWSk++UUt2hJuow8mvwR+rG+LQLw+KsuAKA==}
@@ -16793,11 +16792,9 @@ packages:
 
   /vscode-oniguruma/1.6.2:
     resolution: {integrity: sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA==}
-    dev: false
 
   /vscode-textmate/5.2.0:
     resolution: {integrity: sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==}
-    dev: false
 
   /vscode-uri/2.1.2:
     resolution: {integrity: sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==}