[MDX] Switch from Shiki Twoslash -> Astro Markdown highlighter (#4292)

* freat: twoslash -> Astro shiki parser

* test: update shiki style check

* feat: always apply rehypeRaw

* deps: move remark-shiki-twoslash to dev

* test: add shiki-twoslash test

* docs: update readme with twoslash example

* chore: changeset

* nit: remove "describe('disabled')"
This commit is contained in:
Ben Holmes 2022-08-15 10:43:12 -04:00 committed by GitHub
parent 3889a7fa75
commit f1a52c18af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 156 additions and 35 deletions

View file

@ -0,0 +1,5 @@
---
'@astrojs/mdx': minor
---
Switch from Shiki Twoslash to Astro's Shiki Markdown highlighter

View file

@ -253,7 +253,7 @@ const { title, fancyJsHelper } = Astro.props;
The MDX integration respects [your project's `markdown.syntaxHighlight` configuration](https://docs.astro.build/en/guides/markdown-content/#syntax-highlighting).
We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default [using Shiki twoslash](https://shikijs.github.io/twoslash/). You can customize [this remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash) using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default. You can customize this highlighter using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
```js
// astro.config.mjs
@ -285,6 +285,23 @@ export default {
This applies a minimal Prism renderer with added support for `astro` code blocks. Visit [our "Prism configuration" docs](https://docs.astro.build/en/guides/markdown-content/#prism-configuration) for more on using Prism with Astro.
#### Switch to a custom syntax highlighter
You may want to apply your own syntax highlighter too. If your highlighter offers a remark or rehype plugin, you can flip off our syntax highlighting by setting `markdown.syntaxHighlight: false` and wiring up your plugin. For example, say you want to apply [Shiki Twoslash's remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash):
```js
// astro.config.mjs
import shikiTwoslash from 'remark-shiki-twoslash';
export default {
markdown: {
syntaxHighlight: false,
},
integrations: [mdx({
remarkPlugins: [shikiTwoslash, { /* Shiki Twoslash config */ }],
})],
```
## Configuration
### remarkPlugins

View file

@ -39,7 +39,6 @@
"rehype-raw": "^6.1.1",
"remark-frontmatter": "^4.0.1",
"remark-gfm": "^3.0.1",
"remark-shiki-twoslash": "^3.1.0",
"remark-smartypants": "^2.0.0",
"shiki": "^0.10.1",
"unist-util-visit": "^4.1.0",
@ -56,6 +55,7 @@
"mdast-util-to-string": "^3.1.0",
"mocha": "^9.2.2",
"reading-time": "^1.5.0",
"remark-shiki-twoslash": "^3.1.0",
"remark-toc": "^8.0.1"
},
"engines": {

View file

@ -4,13 +4,13 @@ import type { AstroConfig, AstroIntegration } from 'astro';
import { parse as parseESM } from 'es-module-lexer';
import rehypeRaw from 'rehype-raw';
import remarkGfm from 'remark-gfm';
import remarkShikiTwoslash from 'remark-shiki-twoslash';
import remarkSmartypants from 'remark-smartypants';
import { VFile } from 'vfile';
import type { Plugin as VitePlugin } from 'vite';
import { rehypeApplyFrontmatterExport, remarkInitializeAstroData } from './astro-data-utils.js';
import rehypeCollectHeadings from './rehype-collect-headings.js';
import remarkPrism from './remark-prism.js';
import remarkShiki from './remark-shiki.js';
import { getFileInfo, parseFrontmatter } from './utils.js';
type WithExtends<T> = T | { extends: T };
@ -38,22 +38,17 @@ function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] =
return [...defaults, ...(config?.extends ?? [])];
}
function getRemarkPlugins(
async function getRemarkPlugins(
mdxOptions: MdxOptions,
config: AstroConfig
): MdxRollupPluginOptions['remarkPlugins'] {
): Promise<MdxRollupPluginOptions['remarkPlugins']> {
let remarkPlugins = [
// Initialize vfile.data.astroExports before all plugins are run
remarkInitializeAstroData,
...handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS),
];
if (config.markdown.syntaxHighlight === 'shiki') {
// Default export still requires ".default" chaining for some reason
// Workarounds tried:
// - "import * as remarkShikiTwoslash"
// - "import { default as remarkShikiTwoslash }"
const shikiTwoslash = (remarkShikiTwoslash as any).default ?? remarkShikiTwoslash;
remarkPlugins.push([shikiTwoslash, config.markdown.shikiConfig]);
remarkPlugins.push([await remarkShiki(config.markdown.shikiConfig)]);
}
if (config.markdown.syntaxHighlight === 'prism') {
remarkPlugins.push(remarkPrism);
@ -65,11 +60,11 @@ function getRehypePlugins(
mdxOptions: MdxOptions,
config: AstroConfig
): MdxRollupPluginOptions['rehypePlugins'] {
let rehypePlugins = handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS);
let rehypePlugins = [
[rehypeRaw, { passThrough: nodeTypes }] as any,
...handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS),
];
if (config.markdown.syntaxHighlight === 'shiki' || config.markdown.syntaxHighlight === 'prism') {
rehypePlugins.unshift([rehypeRaw, { passThrough: nodeTypes }]);
}
// getHeadings() is guaranteed by TS, so we can't allow user to override
rehypePlugins.unshift(rehypeCollectHeadings);
@ -80,11 +75,11 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
return {
name: '@astrojs/mdx',
hooks: {
'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
'astro:config:setup': async ({ updateConfig, config, addPageExtension, command }: any) => {
addPageExtension('.mdx');
const mdxPluginOpts: MdxRollupPluginOptions = {
remarkPlugins: getRemarkPlugins(mdxOptions, config),
remarkPlugins: await getRemarkPlugins(mdxOptions, config),
rehypePlugins: getRehypePlugins(mdxOptions, config),
jsx: true,
jsxImportSource: 'astro',

View file

@ -0,0 +1,85 @@
import type { ShikiConfig } from 'astro';
import type * as shiki from 'shiki';
import { getHighlighter } from 'shiki';
import { visit } from 'unist-util-visit';
/**
* getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
* cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
* We make this async, so that multiple calls to parse markdown still share the same highlighter.
*/
const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
const cacheID: string = typeof theme === 'string' ? theme : theme.name;
let highlighterAsync = highlighterCacheAsync.get(cacheID);
if (!highlighterAsync) {
highlighterAsync = getHighlighter({ theme });
highlighterCacheAsync.set(cacheID, highlighterAsync);
}
const highlighter = await highlighterAsync;
// NOTE: There may be a performance issue here for large sites that use `lang`.
// Since this will be called on every page load. Unclear how to fix this.
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
return () => (tree: any) => {
visit(tree, 'code', (node) => {
let lang: string;
if (typeof node.lang === 'string') {
const langExists = highlighter.getLoadedLanguages().includes(node.lang);
if (langExists) {
lang = node.lang;
} else {
// eslint-disable-next-line no-console
console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
lang = 'plaintext';
}
} else {
lang = 'plaintext';
}
let html = highlighter!.codeToHtml(node.value, { lang });
// Q: Couldn't these regexes match on a user's inputted code blocks?
// A: Nope! All rendered HTML is properly escaped.
// Ex. If a user typed `<span class="line"` into a code block,
// It would become this before hitting our regexes:
// &lt;span class=&quot;line&quot;
// Replace "shiki" class naming with "astro".
html = html.replace('<pre class="shiki"', `<pre class="astro-code"`);
// Replace "shiki" css variable naming with "astro".
html = html.replace(
/style="(background-)?color: var\(--shiki-/g,
'style="$1color: var(--astro-code-'
);
// Add "user-select: none;" for "+"/"-" diff symbols
if (node.lang === 'diff') {
html = html.replace(
/<span class="line"><span style="(.*?)">([\+|\-])/g,
'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
);
}
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
} else if (wrap === true) {
html = html.replace(
/style="(.*?)"/,
'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
);
}
node.type = 'html';
node.value = html;
node.children = [];
});
};
};
export default remarkShiki;

View file

@ -3,6 +3,7 @@ import mdx from '@astrojs/mdx';
import { expect } from 'chai';
import { parseHTML } from 'linkedom';
import { loadFixture } from '../../../astro/test/test-utils.js';
import shikiTwoslash from 'remark-shiki-twoslash';
const FIXTURE_ROOT = new URL('./fixtures/mdx-syntax-hightlighting/', import.meta.url);
@ -21,8 +22,9 @@ describe('MDX syntax highlighting', () => {
const html = await fixture.readFile('/index.html');
const { document } = parseHTML(html);
const shikiCodeBlock = document.querySelector('pre.shiki');
const shikiCodeBlock = document.querySelector('pre.astro-code');
expect(shikiCodeBlock).to.not.be.null;
expect(shikiCodeBlock.getAttribute('style')).to.contain('background-color:#0d1117');
});
it('respects markdown.shikiConfig.theme', async () => {
@ -41,8 +43,9 @@ describe('MDX syntax highlighting', () => {
const html = await fixture.readFile('/index.html');
const { document } = parseHTML(html);
const shikiCodeBlock = document.querySelector('pre.shiki.dracula');
const shikiCodeBlock = document.querySelector('pre.astro-code');
expect(shikiCodeBlock).to.not.be.null;
expect(shikiCodeBlock.getAttribute('style')).to.contain('background-color:#282A36');
});
});
@ -64,4 +67,23 @@ describe('MDX syntax highlighting', () => {
expect(prismCodeBlock).to.not.be.null;
});
});
it('supports custom highlighter - shiki-twoslash', async () => {
const fixture = await loadFixture({
root: FIXTURE_ROOT,
markdown: {
syntaxHighlight: false,
},
integrations: [mdx({
remarkPlugins: [shikiTwoslash.default ?? shikiTwoslash],
})],
});
await fixture.build();
const html = await fixture.readFile('/index.html');
const { document } = parseHTML(html);
const twoslashCodeBlock = document.querySelector('pre.shiki');
expect(twoslashCodeBlock).to.not.be.null;
});
});

View file

@ -2268,7 +2268,6 @@ importers:
rehype-raw: 6.1.1
remark-frontmatter: 4.0.1
remark-gfm: 3.0.1
remark-shiki-twoslash: 3.1.0
remark-smartypants: 2.0.0
shiki: 0.10.1
unist-util-visit: 4.1.0
@ -2284,6 +2283,7 @@ importers:
mdast-util-to-string: 3.1.0
mocha: 9.2.2
reading-time: 1.5.0
remark-shiki-twoslash: 3.1.0
remark-toc: 8.0.1
packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection:
@ -8939,7 +8939,7 @@ packages:
lz-string: 1.4.4
transitivePeerDependencies:
- supports-color
dev: false
dev: true
/@typescript/vfs/1.3.4:
resolution: {integrity: sha512-RbyJiaAGQPIcAGWFa3jAXSuAexU4BFiDRF1g3hy7LmRqfNpYlTQWGXjcrOaVZjJ8YkkpuwG0FcsYvtWQpd9igQ==}
@ -8947,7 +8947,7 @@ packages:
debug: 4.3.4
transitivePeerDependencies:
- supports-color
dev: false
dev: true
/@typescript/vfs/1.3.5:
resolution: {integrity: sha512-pI8Saqjupf9MfLw7w2+og+fmb0fZS0J6vsKXXrp4/PDXEFvntgzXmChCXC/KefZZS0YGS6AT8e0hGAJcTsdJlg==}
@ -8955,7 +8955,7 @@ packages:
debug: 4.3.4
transitivePeerDependencies:
- supports-color
dev: false
dev: true
/@ungap/promise-all-settled/1.1.2:
resolution: {integrity: sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==}
@ -11297,7 +11297,7 @@ packages:
/fenceparser/1.1.1:
resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
engines: {node: '>=12'}
dev: false
dev: true
/fetch-blob/3.2.0:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
@ -12444,7 +12444,6 @@ packages:
/jsonc-parser/3.1.0:
resolution: {integrity: sha512-DRf0QjnNeCUds3xTjKlQQ3DpJD51GvDjJfnxUVWg6PZTo2otSm+slzNAxU/35hF8/oJIKoG9slq30JYOsF2azg==}
dev: false
/jsonfile/4.0.0:
resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
@ -12660,7 +12659,7 @@ packages:
/lz-string/1.4.4:
resolution: {integrity: sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==}
hasBin: true
dev: false
dev: true
/magic-string/0.25.9:
resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
@ -15030,7 +15029,7 @@ packages:
unist-util-visit: 2.0.3
transitivePeerDependencies:
- supports-color
dev: false
dev: true
/remark-smartypants/2.0.0:
resolution: {integrity: sha512-Rc0VDmr/yhnMQIz8n2ACYXlfw/P/XZev884QU1I5u+5DgJls32o97Vc1RbK3pfumLsJomS2yy8eT4Fxj/2MDVA==}
@ -15361,7 +15360,7 @@ packages:
typescript: 4.7.4
transitivePeerDependencies:
- supports-color
dev: false
dev: true
/shiki/0.10.1:
resolution: {integrity: sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==}
@ -15369,7 +15368,6 @@ packages:
jsonc-parser: 3.1.0
vscode-oniguruma: 1.6.2
vscode-textmate: 5.2.0
dev: false
/side-channel/1.0.4:
resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==}
@ -16086,7 +16084,7 @@ packages:
/tslib/2.1.0:
resolution: {integrity: sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==}
dev: false
dev: true
/tslib/2.4.0:
resolution: {integrity: sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==}
@ -16313,6 +16311,7 @@ packages:
resolution: {integrity: sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==}
engines: {node: '>=4.2.0'}
hasBin: true
dev: true
/uhyphen/0.1.0:
resolution: {integrity: sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw==}
@ -16401,7 +16400,7 @@ packages:
/unist-util-is/4.1.0:
resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==}
dev: false
dev: true
/unist-util-is/5.1.1:
resolution: {integrity: sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==}
@ -16457,7 +16456,7 @@ packages:
dependencies:
'@types/unist': 2.0.6
unist-util-is: 4.1.0
dev: false
dev: true
/unist-util-visit-parents/4.1.1:
resolution: {integrity: sha512-1xAFJXAKpnnJl8G7K5KgU7FY55y3GcLIXqkzUj5QF/QVP7biUm0K0O2oqVkYsdjzJKifYeWn9+o6piAK2hGSHw==}
@ -16484,7 +16483,7 @@ packages:
'@types/unist': 2.0.6
unist-util-is: 4.1.0
unist-util-visit-parents: 3.1.1
dev: false
dev: true
/unist-util-visit/3.1.0:
resolution: {integrity: sha512-Szoh+R/Ll68QWAyQyZZpQzZQm2UPbxibDvaY8Xc9SUtYgPsDzx5AWSk++UUt2hJuow8mvwR+rG+LQLw+KsuAKA==}
@ -16793,11 +16792,9 @@ packages:
/vscode-oniguruma/1.6.2:
resolution: {integrity: sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA==}
dev: false
/vscode-textmate/5.2.0:
resolution: {integrity: sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==}
dev: false
/vscode-uri/2.1.2:
resolution: {integrity: sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==}