Refactor mdx remark plugins (#8430)

.changeset/cuddly-baboons-begin.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/markdown-remark': minor
+---
+
+Export remarkShiki and remarkPrism plugins

.changeset/stupid-olives-push.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': patch
+---
+
+Use exported remarkShiki and remarkPrism plugins from `@astrojs/markdown-remark`

packages/integrations/mdx/package.json

@@ -35,7 +35,6 @@
   },
   "dependencies": {
     "@astrojs/markdown-remark": "workspace:*",
-    "@astrojs/prism": "workspace:*",
     "@mdx-js/mdx": "^2.3.0",
     "acorn": "^8.10.0",
     "es-module-lexer": "^1.3.0",
@@ -45,10 +44,8 @@
     "hast-util-to-html": "^8.0.4",
     "kleur": "^4.1.4",
     "rehype-raw": "^6.1.1",
-    "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
     "remark-smartypants": "^2.0.0",
-    "shiki": "^0.14.3",
     "source-map": "^0.7.4",
     "unist-util-visit": "^4.1.2",
     "vfile": "^5.3.7"

packages/integrations/mdx/src/plugins.ts

@@ -1,4 +1,9 @@
-import { rehypeHeadingIds, remarkCollectImages } from '@astrojs/markdown-remark';
+import {
+	rehypeHeadingIds,
+	remarkCollectImages,
+	remarkPrism,
+	remarkShiki,
+} from '@astrojs/markdown-remark';
 import {
 	InvalidAstroDataError,
 	safelyGetAstroData,
@@ -16,8 +21,6 @@ import { rehypeInjectHeadingsExport } from './rehype-collect-headings.js';
 import rehypeMetaString from './rehype-meta-string.js';
 import { rehypeOptimizeStatic } from './rehype-optimize-static.js';
 import { remarkImageToComponent } from './remark-images-to-component.js';
-import remarkPrism from './remark-prism.js';
-import remarkShiki from './remark-shiki.js';
 import { jsToTreeNode } from './utils.js';
 
 // Skip nonessential plugins during performance benchmark runs
@@ -112,7 +115,7 @@ export async function getRemarkPlugins(mdxOptions: MdxOptions): Promise<Pluggabl
 	if (!isPerformanceBenchmark) {
 		// Apply syntax highlighters after user plugins to match `markdown/remark` behavior
 		if (mdxOptions.syntaxHighlight === 'shiki') {
-			remarkPlugins.push([await remarkShiki(mdxOptions.shikiConfig)]);
+			remarkPlugins.push([remarkShiki, mdxOptions.shikiConfig]);
 		}
 		if (mdxOptions.syntaxHighlight === 'prism') {
 			remarkPlugins.push(remarkPrism);

packages/integrations/mdx/src/remark-prism.ts

@@ -1,18 +0,0 @@
-import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
-import { visit } from 'unist-util-visit';
-
-/**  */
-export default function remarkPrism() {
-	return (tree: any) =>
-		visit(tree, 'code', (node: any) => {
-			let { lang, value } = node;
-			node.type = 'html';
-
-			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
-			let classes = [classLanguage];
-			node.value = `<pre class="${classes.join(
-				' '
-			)}"><code class="${classLanguage}">${html}</code></pre>`;
-			return node;
-		});
-}

packages/integrations/mdx/src/remark-shiki.ts

@@ -1,94 +0,0 @@
-import type { ShikiConfig } from 'astro';
-import type * as shiki from 'shiki';
-import { getHighlighter } from 'shiki';
-import { visit } from 'unist-util-visit';
-
-/**
- * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
- * cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
- * We make this async, so that multiple calls to parse markdown still share the same highlighter.
- */
-const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
-
-const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
-	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
-	let highlighterAsync = highlighterCacheAsync.get(cacheID);
-	if (!highlighterAsync) {
-		highlighterAsync = getHighlighter({ theme }).then((hl) => {
-			hl.setColorReplacements({
-				'#000001': 'var(--astro-code-color-text)',
-				'#000002': 'var(--astro-code-color-background)',
-				'#000004': 'var(--astro-code-token-constant)',
-				'#000005': 'var(--astro-code-token-string)',
-				'#000006': 'var(--astro-code-token-comment)',
-				'#000007': 'var(--astro-code-token-keyword)',
-				'#000008': 'var(--astro-code-token-parameter)',
-				'#000009': 'var(--astro-code-token-function)',
-				'#000010': 'var(--astro-code-token-string-expression)',
-				'#000011': 'var(--astro-code-token-punctuation)',
-				'#000012': 'var(--astro-code-token-link)',
-			});
-			return hl;
-		});
-		highlighterCacheAsync.set(cacheID, highlighterAsync);
-	}
-	const highlighter = await highlighterAsync;
-
-	// NOTE: There may be a performance issue here for large sites that use `lang`.
-	// Since this will be called on every page load. Unclear how to fix this.
-	for (const lang of langs) {
-		await highlighter.loadLanguage(lang);
-	}
-
-	return () => (tree: any) => {
-		visit(tree, 'code', (node) => {
-			let lang: string;
-
-			if (typeof node.lang === 'string') {
-				const langExists = highlighter.getLoadedLanguages().includes(node.lang);
-				if (langExists) {
-					lang = node.lang;
-				} else {
-					console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
-					lang = 'plaintext';
-				}
-			} else {
-				lang = 'plaintext';
-			}
-
-			let html = highlighter.codeToHtml(node.value, { lang });
-
-			// Q: Couldn't these regexes match on a user's inputted code blocks?
-			// A: Nope! All rendered HTML is properly escaped.
-			// Ex. If a user typed `<span class="line"` into a code block,
-			// It would become this before hitting our regexes:
-			// &lt;span class=&quot;line&quot;
-
-			// Replace "shiki" class naming with "astro".
-			html = html.replace(/<pre class="(.*?)shiki(.*?)"/, `<pre class="$1astro-code$2"`);
-			// Add "user-select: none;" for "+"/"-" diff symbols
-			if (node.lang === 'diff') {
-				html = html.replace(
-					/<span class="line"><span style="(.*?)">([\+|\-])/g,
-					'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
-				);
-			}
-			// Handle code wrapping
-			// if wrap=null, do nothing.
-			if (wrap === false) {
-				html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
-			} else if (wrap === true) {
-				html = html.replace(
-					/style="(.*?)"/,
-					'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
-				);
-			}
-
-			node.type = 'html';
-			node.value = html;
-			node.children = [];
-		});
-	};
-};
-
-export default remarkShiki;

packages/markdown/remark/src/index.ts

@@ -9,9 +9,8 @@ import { toRemarkInitializeAstroData } from './frontmatter-injection.js';
 import { loadPlugins } from './load-plugins.js';
 import { rehypeHeadingIds } from './rehype-collect-headings.js';
 import { remarkCollectImages } from './remark-collect-images.js';
-import remarkPrism from './remark-prism.js';
+import { remarkPrism } from './remark-prism.js';
-import scopedStyles from './remark-scoped-styles.js';
+import { remarkShiki } from './remark-shiki.js';
-import remarkShiki from './remark-shiki.js';
 
 import rehypeRaw from 'rehype-raw';
 import rehypeStringify from 'rehype-stringify';
@@ -25,6 +24,8 @@ import { rehypeImages } from './rehype-images.js';
 
 export { rehypeHeadingIds } from './rehype-collect-headings.js';
 export { remarkCollectImages } from './remark-collect-images.js';
+export { remarkPrism } from './remark-prism.js';
+export { remarkShiki } from './remark-shiki.js';
 export * from './types.js';
 
 export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {
@@ -61,7 +62,6 @@ export async function renderMarkdown(
 		frontmatter: userFrontmatter = {},
 	} = opts;
 	const input = new VFile({ value: content, path: fileURL });
-	const scopedClassName = opts.$?.scopedClassName;
 
 	let parser = unified()
 		.use(markdown)
@@ -85,18 +85,14 @@ export async function renderMarkdown(
 	});
 
 	if (!isPerformanceBenchmark) {
-		if (scopedClassName) {
-			parser.use([scopedStyles(scopedClassName)]);
-		}
-
 		if (syntaxHighlight === 'shiki') {
-			parser.use([await remarkShiki(shikiConfig, scopedClassName)]);
+			parser.use(remarkShiki, shikiConfig);
 		} else if (syntaxHighlight === 'prism') {
-			parser.use([remarkPrism(scopedClassName)]);
+			parser.use(remarkPrism);
 		}
 
 		// Apply later in case user plugins resolve relative image paths
-		parser.use([remarkCollectImages]);
+		parser.use(remarkCollectImages);
 	}
 
 	parser.use([

packages/markdown/remark/src/remark-prism.ts

@@ -1,31 +1,19 @@
 import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
 import { visit } from 'unist-util-visit';
+import type { RemarkPlugin } from './types.js';
 
-type MaybeString = string | null | undefined;
+export function remarkPrism(): ReturnType<RemarkPlugin> {
-
-/**  */
-function transformer(className: MaybeString) {
 	return function (tree: any) {
-		const visitor = (node: any) => {
+		visit(tree, 'code', (node) => {
 			let { lang, value } = node;
 			node.type = 'html';
 
 			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
 			let classes = [classLanguage];
-			if (className) {
-				classes.push(className);
-			}
 			node.value = `<pre class="${classes.join(
 				' '
 			)}"><code is:raw class="${classLanguage}">${html}</code></pre>`;
 			return node;
-		};
+		});
-		return visit(tree, 'code', visitor);
 	};
 }
-
-function plugin(className: MaybeString) {
-	return transformer.bind(null, className);
-}
-
-export default plugin;

packages/markdown/remark/src/remark-scoped-styles.ts

@@ -1,18 +0,0 @@
-import { visit } from 'unist-util-visit';
-const noVisit = new Set(['root', 'html', 'text']);
-
-/**  */
-export default function scopedStyles(className: string) {
-	const visitor = (node: any) => {
-		if (noVisit.has(node.type)) return;
-
-		const { data } = node;
-		let currentClassName = data?.hProperties?.class ?? '';
-		node.data = node.data || {};
-		node.data.hProperties = node.data.hProperties || {};
-		node.data.hProperties.class = `${className} ${currentClassName}`.trim();
-
-		return node;
-	};
-	return () => (tree: any) => visit(tree, visitor);
-}

packages/markdown/remark/src/remark-shiki.ts

@@ -1,7 +1,7 @@
 import type * as shiki from 'shiki';
 import { getHighlighter } from 'shiki';
 import { visit } from 'unist-util-visit';
-import type { ShikiConfig } from './types.js';
+import type { RemarkPlugin, ShikiConfig } from './types.js';
 
 /**
  * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
@@ -10,10 +10,11 @@ import type { ShikiConfig } from './types.js';
  */
 const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
 
-const remarkShiki = async (
+export function remarkShiki({
-	{ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig,
+	langs = [],
-	scopedClassName?: string | null
+	theme = 'github-dark',
-) => {
+	wrap = false,
+}: ShikiConfig = {}): ReturnType<RemarkPlugin> {
 	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
 	let highlighterAsync = highlighterCacheAsync.get(cacheID);
 	if (!highlighterAsync) {
@@ -35,15 +36,22 @@ const remarkShiki = async (
 		});
 		highlighterCacheAsync.set(cacheID, highlighterAsync);
 	}
-	const highlighter = await highlighterAsync;
 
-	// NOTE: There may be a performance issue here for large sites that use `lang`.
+	let highlighter: shiki.Highlighter;
-	// Since this will be called on every page load. Unclear how to fix this.
+
-	for (const lang of langs) {
+	return async (tree: any) => {
-		await highlighter.loadLanguage(lang);
+		// Lazily assign the highlighter as async can only happen within this function,
-	}
+		// and not on `remarkShiki` directly.
+		if (!highlighter) {
+			highlighter = await highlighterAsync!;
+
+			// NOTE: There may be a performance issue here for large sites that use `lang`.
+			// Since this will be called on every page load. Unclear how to fix this.
+			for (const lang of langs) {
+				await highlighter.loadLanguage(lang);
+			}
+		}
 
-	return () => (tree: any) => {
 		visit(tree, 'code', (node) => {
 			let lang: string;
 
@@ -69,10 +77,7 @@ const remarkShiki = async (
 			// &lt;span class=&quot;line&quot;
 
 			// Replace "shiki" class naming with "astro" and add "is:raw".
-			html = html.replace(
+			html = html.replace(/<pre class="(.*?)shiki(.*?)"/, `<pre is:raw class="$1astro-code$2"`);
-				/<pre class="(.*?)shiki(.*?)"/,
-				`<pre is:raw class="$1astro-code$2${scopedClassName ? ' ' + scopedClassName : ''}"`
-			);
 			// Add "user-select: none;" for "+"/"-" diff symbols
 			if (node.lang === 'diff') {
 				html = html.replace(
@@ -91,16 +96,9 @@ const remarkShiki = async (
 				);
 			}
 
-			// Apply scopedClassName to all nested lines
-			if (scopedClassName) {
-				html = html.replace(/\<span class="line"\>/g, `<span class="line ${scopedClassName}"`);
-			}
-
 			node.type = 'html';
 			node.value = html;
 			node.children = [];
 		});
 	};
-};
+}
-
-export default remarkShiki;

packages/markdown/remark/src/types.ts

@@ -61,10 +61,6 @@ export interface ImageMetadata {
 export interface MarkdownRenderingOptions extends AstroMarkdownOptions {
 	/** @internal */
 	fileURL?: URL;
-	/** @internal */
-	$?: {
-		scopedClassName: string | null;
-	};
 	/** Used for frontmatter injection plugins */
 	frontmatter?: Record<string, any>;
 }

pnpm-lock.yaml

@@ -3984,9 +3984,6 @@ importers:
       '@astrojs/markdown-remark':
         specifier: workspace:*
         version: link:../../markdown/remark
-      '@astrojs/prism':
-        specifier: workspace:*
-        version: link:../../astro-prism
       '@mdx-js/mdx':
         specifier: ^2.3.0
         version: 2.3.0
@@ -4014,18 +4011,12 @@ importers:
       rehype-raw:
         specifier: ^6.1.1
         version: 6.1.1
-      remark-frontmatter:
-        specifier: ^4.0.1
-        version: 4.0.1
       remark-gfm:
         specifier: ^3.0.1
         version: 3.0.1
       remark-smartypants:
         specifier: ^2.0.0
         version: 2.0.0
-      shiki:
-        specifier: ^0.14.3
-        version: 0.14.3
       source-map:
         specifier: ^0.7.4
         version: 0.7.4
@@ -4083,7 +4074,7 @@ importers:
         version: 4.0.3
       rehype-pretty-code:
         specifier: ^0.10.0
-        version: 0.10.0(shiki@0.14.3)
+        version: 0.10.0
       remark-math:
         specifier: ^5.1.1
         version: 5.1.1
@@ -11595,12 +11586,6 @@ packages:
     dependencies:
       reusify: 1.0.4
 
-  /fault@2.0.1:
-    resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==}
-    dependencies:
-      format: 0.2.2
-    dev: false
-
   /fenceparser@1.1.1:
     resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
     engines: {node: '>=12'}
@@ -11701,11 +11686,6 @@ packages:
       combined-stream: 1.0.8
       mime-types: 2.1.35
 
-  /format@0.2.2:
-    resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==}
-    engines: {node: '>=0.4.x'}
-    dev: false
-
   /formdata-polyfill@4.0.10:
     resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
     engines: {node: '>=12.20.0'}
@@ -13329,14 +13309,6 @@ packages:
     transitivePeerDependencies:
       - supports-color
 
-  /mdast-util-frontmatter@1.0.1:
-    resolution: {integrity: sha512-JjA2OjxRqAa8wEG8hloD0uTU0kdn8kbtOWpPP94NBkfAlbxn4S8gCGf/9DwFtEeGPXrDcNXdiDjVaRdUFqYokw==}
-    dependencies:
-      '@types/mdast': 3.0.12
-      mdast-util-to-markdown: 1.5.0
-      micromark-extension-frontmatter: 1.1.0
-    dev: false
-
   /mdast-util-gfm-autolink-literal@1.0.3:
     resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==}
     dependencies:
@@ -13599,15 +13571,6 @@ packages:
       micromark-util-types: 1.0.2
       uvu: 0.5.6
 
-  /micromark-extension-frontmatter@1.1.0:
-    resolution: {integrity: sha512-0nLelmvXR5aZ+F2IL6/Ed4cDnHLpL/VD/EELKuclsTWHrLI8UgxGHEmeoumeX2FXiM6z2WrBIOEcbKUZR8RYNg==}
-    dependencies:
-      fault: 2.0.1
-      micromark-util-character: 1.1.0
-      micromark-util-symbol: 1.0.1
-      micromark-util-types: 1.0.2
-    dev: false
-
   /micromark-extension-gfm-autolink-literal@1.0.4:
     resolution: {integrity: sha512-WCssN+M9rUyfHN5zPBn3/f0mIA7tqArHL/EKbv3CZK+LT2rG77FEikIQEqBkv46fOqXQK4NEW/Pc7Z27gshpeg==}
     dependencies:
@@ -15584,7 +15547,7 @@ packages:
       unified: 10.1.2
     dev: false
 
-  /rehype-pretty-code@0.10.0(shiki@0.14.3):
+  /rehype-pretty-code@0.10.0:
     resolution: {integrity: sha512-qCD071Y+vUxEy9yyrATPk2+W9q7qCbzZgtc9suZhu75bmRQvOlBhJt4d3WvqSMTamkKoFkvqtCjyAk+ggH+aXQ==}
     engines: {node: '>=16'}
     peerDependencies:
@@ -15593,7 +15556,6 @@ packages:
       '@types/hast': 2.3.5
       hash-obj: 4.0.0
       parse-numeric-range: 1.3.0
-      shiki: 0.14.3
     dev: true
 
   /rehype-raw@6.1.1:
@@ -15652,15 +15614,6 @@ packages:
     dependencies:
       unist-util-visit: 1.4.1
 
-  /remark-frontmatter@4.0.1:
-    resolution: {integrity: sha512-38fJrB0KnmD3E33a5jZC/5+gGAC2WKNiPw1/fdXJvijBlhA7RCsvJklrYJakS0HedninvaCYW8lQGf9C918GfA==}
-    dependencies:
-      '@types/mdast': 3.0.12
-      mdast-util-frontmatter: 1.0.1
-      micromark-extension-frontmatter: 1.1.0
-      unified: 10.1.2
-    dev: false
-
   /remark-gfm@3.0.1:
     resolution: {integrity: sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==}
     dependencies: