Markdoc - improve syntax highlighting support (#7209)

* feat: prism and shiki support, with better exports! * chore: update tests * chore: fix lock * chore: add prism test * chore: remove `async` from prism * docs: update syntax highlight readme * chore: changeset * edit: remove `await` from prism docs * chore: update old changest with new shiki instructions * fix: add trailing newline on ts-expect-error * refactor: resolve promises internally * docs: remove `await` from shiki examples
2023-05-25 11:35:07 -04:00 · 2023-05-25 11:35:07 -04:00 · 16b8364119
commit 16b8364119
parent 223e0131fc
11 changed files with 200 additions and 114 deletions
--- a/.changeset/eleven-tables-speak.md
+++ b/.changeset/eleven-tables-speak.md
@ -2,14 +2,16 @@
 '@astrojs/markdoc': patch
 ---

-Add support for syntax highlighting with Shiki. Install `shiki` in your project with `npm i shiki`, and apply to your Markdoc config using the `extends` option:
+Add support for syntax highlighting with Shiki. Apply to your Markdoc config using the `extends` property:

 ```js
 // markdoc.config.mjs
-import { defineMarkdocConfig, shiki } from '@astrojs/markdoc/config';
+import { defineMarkdocConfig } from '@astrojs/markdoc/config';
+import shiki from '@astrojs/markdoc/shiki';
+
 export default defineMarkdocConfig({
  extends: [
-    await shiki({ /** Shiki config options */ }),
+    shiki({ /** Shiki config options */ }),
  ],
 })
 ```
--- a/.changeset/popular-berries-travel.md
+++ b/.changeset/popular-berries-travel.md
@ -0,0 +1,17 @@
+---
+'@astrojs/markdoc': patch
+---
+
+Add a built-in extension for syntax highlighting with Prism. Apply to your Markdoc config using the `extends` property:
+
+```js
+// markdoc.config.mjs
+import { defineMarkdocConfig } from '@astrojs/markdoc/config';
+import prism from '@astrojs/markdoc/prism';
+
+export default defineMarkdocConfig({
+  extends: [prism()],
+})
+```
+
+Learn more in the [`@astrojs/markdoc` README.](https://docs.astro.build/en/guides/integrations-guide/markdoc/#syntax-highlighting)
--- a/packages/integrations/markdoc/README.md
+++ b/packages/integrations/markdoc/README.md
@ -205,23 +205,20 @@ export default defineMarkdocConfig({

 ### Syntax highlighting

-`@astrojs/markdoc` provides a [Shiki](https://github.com/shikijs/shiki) extension to highlight your code blocks.
+`@astrojs/markdoc` provides [Shiki](https://github.com/shikijs/shiki) and [Prism](https://github.com/PrismJS) extensions to highlight your code blocks.

-To use this extension, you must separately install `shiki` as a dependency:
+#### Shiki

-```bash
-npm i shiki
-```
-
-Then, apply the `shiki()` extension to your Markdoc config using the `extends` property. You can optionally pass a shiki configuration object:
+Apply the `shiki()` extension to your Markdoc config using the `extends` property. You can optionally pass a shiki configuration object:

 ```js
 // markdoc.config.mjs
-import { defineMarkdocConfig, shiki } from '@astrojs/markdoc/config';
+import { defineMarkdocConfig } from '@astrojs/markdoc/config';
+import shiki from '@astrojs/markdoc/shiki';

 export default defineMarkdocConfig({
  extends: [
-    await shiki({
+    shiki({
      // Choose from Shiki's built-in themes (or add your own)
      // Default: 'github-dark'
      // https://github.com/shikijs/shiki/blob/main/docs/themes.md
@ -238,6 +235,22 @@ export default defineMarkdocConfig({
 })
 ```

+#### Prism
+
+Apply the `prism()` extension to your Markdoc config using the `extends` property.
+
+```js
+// markdoc.config.mjs
+import { defineMarkdocConfig } from '@astrojs/markdoc/config';
+import prism from '@astrojs/markdoc/prism';
+
+export default defineMarkdocConfig({
+  extends: [prism()],
+})
+```
+
+📚 To learn about configuring Prism stylesheets, [see our syntax highlighting guide.](https://docs.astro.build/en/guides/markdown-content/#prism-configuration)
+
 ### Access frontmatter and content collection information from your templates

 You can access content collection information from your Markdoc templates using the `$entry` variable. This includes the entry `slug`, `collection` name, and frontmatter `data` parsed by your content collection schema (if any). This example renders the `title` frontmatter property as a heading:
--- a/packages/integrations/markdoc/package.json
+++ b/packages/integrations/markdoc/package.json
@ -19,6 +19,8 @@
  "bugs": "https://github.com/withastro/astro/issues",
  "homepage": "https://docs.astro.build/en/guides/integrations-guide/markdoc/",
  "exports": {
+    "./prism": "./dist/extensions/prism.js",
+    "./shiki": "./dist/extensions/shiki.js",
    ".": "./dist/index.js",
    "./components": "./components/index.ts",
    "./runtime": "./dist/runtime.js",
@ -39,7 +41,9 @@
    "test:match": "mocha --timeout 20000 -g"
  },
  "dependencies": {
-    "@markdoc/markdoc": "^0.2.2",
+    "shiki": "^0.14.1",
+    "@astrojs/prism": "^2.1.2",
+    "@markdoc/markdoc": "^0.3.0",
    "esbuild": "^0.17.12",
    "github-slugger": "^2.0.0",
    "gray-matter": "^4.0.3",
@ -47,13 +51,7 @@
    "zod": "^3.17.3"
  },
  "peerDependencies": {
-    "astro": "workspace:^2.5.5",
-    "shiki": "^0.14.1"
-  },
-  "peerDependenciesMeta": {
-    "shiki": {
-      "optional": true
-    }
+    "astro": "workspace:^2.5.5"
  },
  "devDependencies": {
    "@astrojs/markdown-remark": "^2.2.1",
@ -67,7 +65,6 @@
    "linkedom": "^0.14.12",
    "mocha": "^9.2.2",
    "rollup": "^3.20.1",
-    "shiki": "^0.14.1",
    "vite": "^4.3.1"
  },
  "engines": {
--- a/packages/integrations/markdoc/src/config.ts
+++ b/packages/integrations/markdoc/src/config.ts
@ -12,7 +12,6 @@ export type ResolvedAstroMarkdocConfig = Omit<AstroMarkdocConfig, 'extends'>;

 export const Markdoc = _Markdoc;
 export const nodes = { ...Markdoc.nodes, heading };
-export { shiki } from './extensions/shiki.js';

 export function defineMarkdocConfig(config: AstroMarkdocConfig): AstroMarkdocConfig {
 	return config;
--- a/packages/integrations/markdoc/src/extensions/prism.ts
+++ b/packages/integrations/markdoc/src/extensions/prism.ts
@ -0,0 +1,24 @@
+// leave space, so organize imports doesn't mess up comments
+// @ts-expect-error Cannot find module 'astro/runtime/server/index.js' or its corresponding type declarations.
+import { unescapeHTML } from 'astro/runtime/server/index.js';
+
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
+import { Markdoc, type AstroMarkdocConfig } from '../config.js';
+
+export default function prism(): AstroMarkdocConfig {
+	return {
+		nodes: {
+			fence: {
+				attributes: Markdoc.nodes.fence.attributes!,
+				transform({ attributes: { language, content } }) {
+					const { html, classLanguage } = runHighlighterWithAstro(language, content);
+
+					// Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML
+					return unescapeHTML(
+						`<pre class="${classLanguage}"><code class="${classLanguage}">${html}</code></pre>`
+					);
+				},
+			},
+		},
+	};
+}
--- a/packages/integrations/markdoc/src/extensions/shiki.ts
+++ b/packages/integrations/markdoc/src/extensions/shiki.ts
@ -2,11 +2,11 @@
 // @ts-expect-error Cannot find module 'astro/runtime/server/index.js' or its corresponding type declarations.
 import { unescapeHTML } from 'astro/runtime/server/index.js';

-import Markdoc from '@markdoc/markdoc';
 import type { ShikiConfig } from 'astro';
 import type * as shikiTypes from 'shiki';
 import type { AstroMarkdocConfig } from '../config.js';
-import { MarkdocError } from '../utils.js';
+import Markdoc from '@markdoc/markdoc';
+import { getHighlighter } from 'shiki';

 // Map of old theme names to new names to preserve compatibility when we upgrade shiki
 const compatThemes: Record<string, string> = {
@ -51,19 +51,11 @@ const INLINE_STYLE_SELECTOR = /style="(.*?)"/;
 */
 const highlighterCache = new Map<string, shikiTypes.Highlighter>();

-export async function shiki({
+export default async function shiki({
 	langs = [],
 	theme = 'github-dark',
 	wrap = false,
 }: ShikiConfig = {}): Promise<AstroMarkdocConfig> {
-	let getHighlighter: (options: shikiTypes.HighlighterOptions) => Promise<shikiTypes.Highlighter>;
-	try {
-		getHighlighter = (await import('shiki')).getHighlighter;
-	} catch {
-		throw new MarkdocError({
-			message: 'Shiki is not installed. Run `npm install shiki` to use the `shiki` extension.',
-		});
-	}
 	theme = normalizeTheme(theme);

 	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
--- a/packages/integrations/markdoc/src/index.ts
+++ b/packages/integrations/markdoc/src/index.ts
@ -32,7 +32,19 @@ export default function markdocIntegration(legacyConfig?: any): AstroIntegration
 		name: '@astrojs/markdoc',
 		hooks: {
 			'astro:config:setup': async (params) => {
-				const { config: astroConfig, addContentEntryType } = params as SetupHookParams;
+				const {
+					config: astroConfig,
+					updateConfig,
+					addContentEntryType,
+				} = params as SetupHookParams;
+
+				updateConfig({
+					vite: {
+						ssr: {
+							external: ['@astrojs/markdoc/prism', '@astrojs/markdoc/shiki'],
+						},
+					},
+				});

 				markdocConfigResult = await loadMarkdocConfig(astroConfig);
 				const userMarkdocConfig = markdocConfigResult?.config ?? {};
@ -52,11 +64,7 @@ export default function markdocIntegration(legacyConfig?: any): AstroIntegration
 					async getRenderModule({ entry, viteId }) {
 						const ast = Markdoc.parse(entry.body);
 						const pluginContext = this;
-						const markdocConfig = setupConfig(
-							userMarkdocConfig,
-							entry,
-							markdocConfigResult?.fileUrl.pathname
-						);
+						const markdocConfig = await setupConfig(userMarkdocConfig, entry);

 						const validationErrors = Markdoc.validate(ast, markdocConfig).filter((e) => {
 							return (
@ -94,7 +102,7 @@ export default function markdocIntegration(legacyConfig?: any): AstroIntegration

 						const res = `import { jsx as h } from 'astro/jsx-runtime';
 						import { Renderer } from '@astrojs/markdoc/components';
-						import { collectHeadings, setupConfig, Markdoc } from '@astrojs/markdoc/runtime';
+						import { collectHeadings, setupConfig, setupConfigSync, Markdoc } from '@astrojs/markdoc/runtime';
 import * as entry from ${JSON.stringify(viteId + '?astroContentCollectionEntry')};
 ${
 	markdocConfigResult
@ -118,13 +126,13 @@ export function getHeadings() {
 		''
 	}
 	const headingConfig = userConfig.nodes?.heading;
-	const config = setupConfig(headingConfig ? { nodes: { heading: headingConfig } } : {}, entry);
+	const config = setupConfigSync(headingConfig ? { nodes: { heading: headingConfig } } : {}, entry);
 	const ast = Markdoc.Ast.fromJSON(stringifiedAst);
 	const content = Markdoc.transform(ast, config);
 	return collectHeadings(Array.isArray(content) ? content : content.children);
 }
 export async function Content (props) {
-	const config = setupConfig({
+	const config = await setupConfig({
 		...userConfig,
 		variables: { ...userConfig.variables, ...props },
 	}, entry);
--- a/packages/integrations/markdoc/src/runtime.ts
+++ b/packages/integrations/markdoc/src/runtime.ts
@ -13,26 +13,19 @@ export { default as Markdoc } from '@markdoc/markdoc';
 * Called on each file's individual transform.
 * TODO: virtual module to merge configs per-build instead of per-file?
 */
-export function setupConfig(
+export async function setupConfig(
 	userConfig: AstroMarkdocConfig,
-	entry: ContentEntryModule,
-	markdocConfigPath?: string
-): Omit<AstroMarkdocConfig, 'extends'> {
+	entry: ContentEntryModule
+): Promise<Omit<AstroMarkdocConfig, 'extends'>> {
 	let defaultConfig: AstroMarkdocConfig = {
 		...setupHeadingConfig(),
 		variables: { entry },
 	};

 	if (userConfig.extends) {
-		for (const extension of userConfig.extends) {
+		for (let extension of userConfig.extends) {
 			if (extension instanceof Promise) {
-				throw new MarkdocError({
-					message: 'An extension passed to `extends` in your markdoc config returns a Promise.',
-					hint: 'Call `await` for async extensions. Example: `extends: [await myExtension()]`',
-					location: {
-						file: markdocConfigPath,
-					},
-				});
+				extension = await extension;
 			}

 			defaultConfig = mergeConfig(defaultConfig, extension);
@ -42,6 +35,19 @@ export function setupConfig(
 	return mergeConfig(defaultConfig, userConfig);
 }

+/** Used for synchronous `getHeadings()` function */
+export function setupConfigSync(
+	userConfig: AstroMarkdocConfig,
+	entry: ContentEntryModule
+): Omit<AstroMarkdocConfig, 'extends'> {
+	let defaultConfig: AstroMarkdocConfig = {
+		...setupHeadingConfig(),
+		variables: { entry },
+	};
+
+	return mergeConfig(defaultConfig, userConfig);
+}
+
 /** Merge function from `@markdoc/markdoc` internals */
 function mergeConfig(configA: AstroMarkdocConfig, configB: AstroMarkdocConfig): AstroMarkdocConfig {
 	return {
--- a/packages/integrations/markdoc/test/syntax-highlighting.test.js
+++ b/packages/integrations/markdoc/test/syntax-highlighting.test.js
@ -1,7 +1,8 @@
 import { parseHTML } from 'linkedom';
 import { expect } from 'chai';
 import Markdoc from '@markdoc/markdoc';
-import { shiki } from '../dist/config.js';
+import shiki from '../dist/extensions/shiki.js';
+import prism from '../dist/extensions/prism.js';
 import { setupConfig } from '../dist/runtime.js';
 import { isHTMLString } from 'astro/runtime/server/index.js';

@ -18,52 +19,76 @@ const highlighting = true;
 `;

 describe('Markdoc - syntax highlighting', () => {
-	it('transforms with defaults', async () => {
-		const ast = Markdoc.parse(entry);
-		const content = Markdoc.transform(ast, await getConfigExtendingShiki());
+	describe('shiki', () => {
+		it('transforms with defaults', async () => {
+			const ast = Markdoc.parse(entry);
+			const content = Markdoc.transform(ast, await getConfigExtendingShiki());

-		expect(content.children).to.have.lengthOf(2);
-		for (const codeBlock of content.children) {
-			expect(isHTMLString(codeBlock)).to.be.true;
+			expect(content.children).to.have.lengthOf(2);
+			for (const codeBlock of content.children) {
+				expect(isHTMLString(codeBlock)).to.be.true;

-			const pre = parsePreTag(codeBlock);
-			expect(pre.classList).to.include('astro-code');
-			expect(pre.classList).to.include('github-dark');
-		}
+				const pre = parsePreTag(codeBlock);
+				expect(pre.classList).to.include('astro-code');
+				expect(pre.classList).to.include('github-dark');
+			}
+		});
+		it('transforms with `theme` property', async () => {
+			const ast = Markdoc.parse(entry);
+			const content = Markdoc.transform(
+				ast,
+				await getConfigExtendingShiki({
+					theme: 'dracula',
+				})
+			);
+			expect(content.children).to.have.lengthOf(2);
+			for (const codeBlock of content.children) {
+				expect(isHTMLString(codeBlock)).to.be.true;
+
+				const pre = parsePreTag(codeBlock);
+				expect(pre.classList).to.include('astro-code');
+				expect(pre.classList).to.include('dracula');
+			}
+		});
+		it('transforms with `wrap` property', async () => {
+			const ast = Markdoc.parse(entry);
+			const content = Markdoc.transform(
+				ast,
+				await getConfigExtendingShiki({
+					wrap: true,
+				})
+			);
+			expect(content.children).to.have.lengthOf(2);
+			for (const codeBlock of content.children) {
+				expect(isHTMLString(codeBlock)).to.be.true;
+
+				const pre = parsePreTag(codeBlock);
+				expect(pre.getAttribute('style')).to.include('white-space: pre-wrap');
+				expect(pre.getAttribute('style')).to.include('word-wrap: break-word');
+			}
+		});
 	});
-	it('transforms with `theme` property', async () => {
-		const ast = Markdoc.parse(entry);
-		const content = Markdoc.transform(
-			ast,
-			await getConfigExtendingShiki({
-				theme: 'dracula',
-			})
-		);
-		expect(content.children).to.have.lengthOf(2);
-		for (const codeBlock of content.children) {
-			expect(isHTMLString(codeBlock)).to.be.true;

-			const pre = parsePreTag(codeBlock);
-			expect(pre.classList).to.include('astro-code');
-			expect(pre.classList).to.include('dracula');
-		}
-	});
-	it('transforms with `wrap` property', async () => {
-		const ast = Markdoc.parse(entry);
-		const content = Markdoc.transform(
-			ast,
-			await getConfigExtendingShiki({
-				wrap: true,
-			})
-		);
-		expect(content.children).to.have.lengthOf(2);
-		for (const codeBlock of content.children) {
-			expect(isHTMLString(codeBlock)).to.be.true;
+	describe('prism', () => {
+		it('transforms', async () => {
+			const ast = Markdoc.parse(entry);
+			const config = await setupConfig({
+				extends: [prism()],
+			});
+			const content = Markdoc.transform(ast, config);

-			const pre = parsePreTag(codeBlock);
-			expect(pre.getAttribute('style')).to.include('white-space: pre-wrap');
-			expect(pre.getAttribute('style')).to.include('word-wrap: break-word');
-		}
+			expect(content.children).to.have.lengthOf(2);
+			const [tsBlock, cssBlock] = content.children;
+
+			expect(isHTMLString(tsBlock)).to.be.true;
+			expect(isHTMLString(cssBlock)).to.be.true;
+
+			const preTs = parsePreTag(tsBlock);
+			expect(preTs.classList).to.include('language-ts');
+
+			const preCss = parsePreTag(cssBlock);
+			expect(preCss.classList).to.include('language-css');
+		});
 	});
 });

@ -72,8 +97,8 @@ describe('Markdoc - syntax highlighting', () => {
 * @returns {import('../src/config.js').AstroMarkdocConfig}
 */
 async function getConfigExtendingShiki(config) {
-	return setupConfig({
-		extends: [await shiki(config)],
+	return await setupConfig({
+		extends: [shiki(config)],
 	});
 }

--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -3973,9 +3973,12 @@ importers:

  packages/integrations/markdoc:
    dependencies:
+      '@astrojs/prism':
+        specifier: ^2.1.2
+        version: link:../../astro-prism
      '@markdoc/markdoc':
-        specifier: ^0.2.2
-        version: 0.2.2
+        specifier: ^0.3.0
+        version: 0.3.0
      esbuild:
        specifier: ^0.17.12
        version: 0.17.12
@ -3988,6 +3991,9 @@ importers:
      kleur:
        specifier: ^4.1.5
        version: 4.1.5
+      shiki:
+        specifier: ^0.14.1
+        version: 0.14.1
      zod:
        specifier: ^3.17.3
        version: 3.20.6
@ -4025,9 +4031,6 @@ importers:
      rollup:
        specifier: ^3.20.1
        version: 3.20.1
-      shiki:
-        specifier: ^0.14.1
-        version: 0.14.1
      vite:
        specifier: ^4.3.1
        version: 4.3.1(@types/node@18.16.3)(sass@1.52.2)
@ -8055,15 +8058,15 @@ packages:
      nopt: 5.0.0
      npmlog: 5.0.1
      rimraf: 3.0.2
-      semver: 7.3.8
-      tar: 6.1.11
+      semver: 7.5.1
+      tar: 6.1.14
    transitivePeerDependencies:
      - encoding
      - supports-color
    dev: false

-  /@markdoc/markdoc@0.2.2:
-    resolution: {integrity: sha512-0TiD9jmA5h5znN4lxo7HECAu3WieU5g5vUsfByeucrdR/x88hEilpt16EydFyJwJddQ/3w5HQgW7Ovy62r4cyw==}
+  /@markdoc/markdoc@0.3.0:
+    resolution: {integrity: sha512-QWCF8krIIw52ulflfnoff0yG1eKl9CCGA3KAiOjHyYtHNzSEouFh8lO52nAaO3qV2Ctj1GTB8TTb2rTfvISQfA==}
    engines: {node: '>=14.7.0'}
    peerDependencies:
      '@types/react': '*'
@ -8734,8 +8737,8 @@ packages:
  /@types/babel__core@7.1.19:
    resolution: {integrity: sha512-WEOTgRsbYkvA/KCsDwVEGkd7WAr1e3g31VHQ8zy5gul/V1qKullU/BU5I68X5v7V3GnB9eotmom4v5a5gjxorw==}
    dependencies:
-      '@babel/parser': 7.18.4
-      '@babel/types': 7.18.4
+      '@babel/parser': 7.21.8
+      '@babel/types': 7.21.5
      '@types/babel__generator': 7.6.4
      '@types/babel__template': 7.4.1
      '@types/babel__traverse': 7.17.1
@ -8744,19 +8747,19 @@ packages:
  /@types/babel__generator@7.6.4:
    resolution: {integrity: sha512-tFkciB9j2K755yrTALxD44McOrk+gfpIpvC3sxHjRawj6PfnQxrse4Clq5y/Rq+G3mrBurMax/lG8Qn2t9mSsg==}
    dependencies:
-      '@babel/types': 7.18.4
+      '@babel/types': 7.21.5

  /@types/babel__template@7.4.1:
    resolution: {integrity: sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g==}
    dependencies:
-      '@babel/parser': 7.18.4
-      '@babel/types': 7.18.4
+      '@babel/parser': 7.21.8
+      '@babel/types': 7.21.5
    dev: false

  /@types/babel__traverse@7.17.1:
    resolution: {integrity: sha512-kVzjari1s2YVi77D3w1yuvohV2idweYXMCDzqBiVNN63TcDWrIlTVOYpqVrvbbyOE/IyzBoTKF0fdnLPEORFxA==}
    dependencies:
-      '@babel/types': 7.18.4
+      '@babel/types': 7.21.5

  /@types/better-sqlite3@7.6.4:
    resolution: {integrity: sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg==}
@ -10496,7 +10499,7 @@ packages:
    dev: false

  /concat-map@0.0.1:
-    resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
+    resolution: {integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=}

  /concordance@5.0.4:
    resolution: {integrity: sha512-OAcsnTEYu1ARJqWVGwf4zh4JDfHZEaSNlNccFmt8YjB2l/n19/PF2viLINHc57vO4FKIAFl2FWASIGZZWZ2Kxw==}