michael/astro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor mdx remark plugins (#8430)

Browse source
This commit is contained in:
Bjorn Lu 2023-09-07 22:28:02 +08:00 committed by GitHub
parent 0fa483283e
commit f3f62a5a20
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 52 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/cuddly-baboons-begin.md Normal file
View file

@ -0,0 +1,5 @@
---
'@astrojs/markdown-remark': minor
---
Export remarkShiki and remarkPrism plugins

5
.changeset/stupid-olives-push.md Normal file
View file

@ -0,0 +1,5 @@
---
'@astrojs/mdx': patch
---
Use exported remarkShiki and remarkPrism plugins from `@astrojs/markdown-remark`

3
packages/integrations/mdx/package.json
View file

@ -35,7 +35,6 @@
},
"dependencies": {
"@astrojs/markdown-remark": "workspace:*",
"@astrojs/prism": "workspace:*",
"@mdx-js/mdx": "^2.3.0",
"acorn": "^8.10.0",
"es-module-lexer": "^1.3.0",
@ -45,10 +44,8 @@
"hast-util-to-html": "^8.0.4",
"kleur": "^4.1.4",
"rehype-raw": "^6.1.1",
"remark-frontmatter": "^4.0.1",
"remark-gfm": "^3.0.1",
"remark-smartypants": "^2.0.0",
"shiki": "^0.14.3",
"source-map": "^0.7.4",
"unist-util-visit": "^4.1.2",
"vfile": "^5.3.7"

11
packages/integrations/mdx/src/plugins.ts
View file

@ -1,4 +1,9 @@
import { rehypeHeadingIds, remarkCollectImages } from '@astrojs/markdown-remark';
import {
rehypeHeadingIds,
remarkCollectImages,
remarkPrism,
remarkShiki,
} from '@astrojs/markdown-remark';
import {
InvalidAstroDataError,
safelyGetAstroData,
@ -16,8 +21,6 @@ import { rehypeInjectHeadingsExport } from './rehype-collect-headings.js';
import rehypeMetaString from './rehype-meta-string.js';
import { rehypeOptimizeStatic } from './rehype-optimize-static.js';
import { remarkImageToComponent } from './remark-images-to-component.js';
import remarkPrism from './remark-prism.js';
import remarkShiki from './remark-shiki.js';
import { jsToTreeNode } from './utils.js';
// Skip nonessential plugins during performance benchmark runs
@ -112,7 +115,7 @@ export async function getRemarkPlugins(mdxOptions: MdxOptions): Promise<Pluggabl
if (!isPerformanceBenchmark) {
// Apply syntax highlighters after user plugins to match `markdown/remark` behavior
if (mdxOptions.syntaxHighlight === 'shiki') {
remarkPlugins.push([await remarkShiki(mdxOptions.shikiConfig)]);
remarkPlugins.push([remarkShiki, mdxOptions.shikiConfig]);
}
if (mdxOptions.syntaxHighlight === 'prism') {
remarkPlugins.push(remarkPrism);

18
packages/integrations/mdx/src/remark-prism.ts
View file

@ -1,18 +0,0 @@
import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
import { visit } from 'unist-util-visit';
/** */
export default function remarkPrism() {
return (tree: any) =>
visit(tree, 'code', (node: any) => {
let { lang, value } = node;
node.type = 'html';
let { html, classLanguage } = runHighlighterWithAstro(lang, value);
let classes = [classLanguage];
node.value = `<pre class="${classes.join(
' '
)}"><code class="${classLanguage}">${html}</code></pre>`;
return node;
});
}

94
packages/integrations/mdx/src/remark-shiki.ts
View file

@ -1,94 +0,0 @@
import type { ShikiConfig } from 'astro';
import type * as shiki from 'shiki';
import { getHighlighter } from 'shiki';
import { visit } from 'unist-util-visit';
/**
* getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
* cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
* We make this async, so that multiple calls to parse markdown still share the same highlighter.
*/
const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
const cacheID: string = typeof theme === 'string' ? theme : theme.name;
let highlighterAsync = highlighterCacheAsync.get(cacheID);
if (!highlighterAsync) {
highlighterAsync = getHighlighter({ theme }).then((hl) => {
hl.setColorReplacements({
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
});
return hl;
});
highlighterCacheAsync.set(cacheID, highlighterAsync);
}
const highlighter = await highlighterAsync;
// NOTE: There may be a performance issue here for large sites that use `lang`.
// Since this will be called on every page load. Unclear how to fix this.
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
return () => (tree: any) => {
visit(tree, 'code', (node) => {
let lang: string;
if (typeof node.lang === 'string') {
const langExists = highlighter.getLoadedLanguages().includes(node.lang);
if (langExists) {
lang = node.lang;
} else {
console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
lang = 'plaintext';
}
} else {
lang = 'plaintext';
}
let html = highlighter.codeToHtml(node.value, { lang });
// Q: Couldn't these regexes match on a user's inputted code blocks?
// A: Nope! All rendered HTML is properly escaped.
// Ex. If a user typed `<span class="line"` into a code block,
// It would become this before hitting our regexes:
// &lt;span class=&quot;line&quot;
// Replace "shiki" class naming with "astro".
html = html.replace(/<pre class="(.*?)shiki(.*?)"/, `<pre class="$1astro-code$2"`);
// Add "user-select: none;" for "+"/"-" diff symbols
if (node.lang === 'diff') {
html = html.replace(
/<span class="line"><span style="(.*?)">([\+|\-])/g,
'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
);
}
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
} else if (wrap === true) {
html = html.replace(
/style="(.*?)"/,
'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
);
}
node.type = 'html';
node.value = html;
node.children = [];
});
};
};
export default remarkShiki;

18
packages/markdown/remark/src/index.ts
View file

@ -9,9 +9,8 @@ import { toRemarkInitializeAstroData } from './frontmatter-injection.js';
import { loadPlugins } from './load-plugins.js';
import { rehypeHeadingIds } from './rehype-collect-headings.js';
import { remarkCollectImages } from './remark-collect-images.js';
import remarkPrism from './remark-prism.js';
import scopedStyles from './remark-scoped-styles.js';
import remarkShiki from './remark-shiki.js';
import { remarkPrism } from './remark-prism.js';
import { remarkShiki } from './remark-shiki.js';
import rehypeRaw from 'rehype-raw';
import rehypeStringify from 'rehype-stringify';
@ -25,6 +24,8 @@ import { rehypeImages } from './rehype-images.js';
export { rehypeHeadingIds } from './rehype-collect-headings.js';
export { remarkCollectImages } from './remark-collect-images.js';
export { remarkPrism } from './remark-prism.js';
export { remarkShiki } from './remark-shiki.js';
export * from './types.js';
export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {
@ -61,7 +62,6 @@ export async function renderMarkdown(
frontmatter: userFrontmatter = {},
} = opts;
const input = new VFile({ value: content, path: fileURL });
const scopedClassName = opts.$?.scopedClassName;
let parser = unified()
.use(markdown)
@ -85,18 +85,14 @@ export async function renderMarkdown(
});
if (!isPerformanceBenchmark) {
if (scopedClassName) {
parser.use([scopedStyles(scopedClassName)]);
}
if (syntaxHighlight === 'shiki') {
parser.use([await remarkShiki(shikiConfig, scopedClassName)]);
parser.use(remarkShiki, shikiConfig);
} else if (syntaxHighlight === 'prism') {
parser.use([remarkPrism(scopedClassName)]);
parser.use(remarkPrism);
}
// Apply later in case user plugins resolve relative image paths
parser.use([remarkCollectImages]);
parser.use(remarkCollectImages);
}
parser.use([

20
packages/markdown/remark/src/remark-prism.ts
View file

@ -1,31 +1,19 @@
import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
import { visit } from 'unist-util-visit';
import type { RemarkPlugin } from './types.js';
type MaybeString = string | null | undefined;
/** */
function transformer(className: MaybeString) {
export function remarkPrism(): ReturnType<RemarkPlugin> {
return function (tree: any) {
const visitor = (node: any) => {
visit(tree, 'code', (node) => {
let { lang, value } = node;
node.type = 'html';
let { html, classLanguage } = runHighlighterWithAstro(lang, value);
let classes = [classLanguage];
if (className) {
classes.push(className);
}
node.value = `<pre class="${classes.join(
' '
)}"><code is:raw class="${classLanguage}">${html}</code></pre>`;
return node;
};
return visit(tree, 'code', visitor);
});
};
}
function plugin(className: MaybeString) {
return transformer.bind(null, className);
}
export default plugin;

18
packages/markdown/remark/src/remark-scoped-styles.ts
View file

@ -1,18 +0,0 @@
import { visit } from 'unist-util-visit';
const noVisit = new Set(['root', 'html', 'text']);
/** */
export default function scopedStyles(className: string) {
const visitor = (node: any) => {
if (noVisit.has(node.type)) return;
const { data } = node;
let currentClassName = data?.hProperties?.class ?? '';
node.data = node.data || {};
node.data.hProperties = node.data.hProperties || {};
node.data.hProperties.class = `${className} ${currentClassName}`.trim();
return node;
};
return () => (tree: any) => visit(tree, visitor);
}

46
packages/markdown/remark/src/remark-shiki.ts
View file

@ -1,7 +1,7 @@
import type * as shiki from 'shiki';
import { getHighlighter } from 'shiki';
import { visit } from 'unist-util-visit';
import type { ShikiConfig } from './types.js';
import type { RemarkPlugin, ShikiConfig } from './types.js';
/**
* getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
@ -10,10 +10,11 @@ import type { ShikiConfig } from './types.js';
*/
const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
const remarkShiki = async (
{ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig,
scopedClassName?: string | null
) => {
export function remarkShiki({
langs = [],
theme = 'github-dark',
wrap = false,
}: ShikiConfig = {}): ReturnType<RemarkPlugin> {
const cacheID: string = typeof theme === 'string' ? theme : theme.name;
let highlighterAsync = highlighterCacheAsync.get(cacheID);
if (!highlighterAsync) {
@ -35,15 +36,22 @@ const remarkShiki = async (
});
highlighterCacheAsync.set(cacheID, highlighterAsync);
}
const highlighter = await highlighterAsync;
// NOTE: There may be a performance issue here for large sites that use `lang`.
// Since this will be called on every page load. Unclear how to fix this.
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
let highlighter: shiki.Highlighter;
return async (tree: any) => {
// Lazily assign the highlighter as async can only happen within this function,
// and not on `remarkShiki` directly.
if (!highlighter) {
highlighter = await highlighterAsync!;
// NOTE: There may be a performance issue here for large sites that use `lang`.
// Since this will be called on every page load. Unclear how to fix this.
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
}
return () => (tree: any) => {
visit(tree, 'code', (node) => {
let lang: string;
@ -69,10 +77,7 @@ const remarkShiki = async (
// &lt;span class=&quot;line&quot;
// Replace "shiki" class naming with "astro" and add "is:raw".
html = html.replace(
/<pre class="(.*?)shiki(.*?)"/,
`<pre is:raw class="$1astro-code$2${scopedClassName ? ' ' + scopedClassName : ''}"`
);
html = html.replace(/<pre class="(.*?)shiki(.*?)"/, `<pre is:raw class="$1astro-code$2"`);
// Add "user-select: none;" for "+"/"-" diff symbols
if (node.lang === 'diff') {
html = html.replace(
@ -91,16 +96,9 @@ const remarkShiki = async (
);
}
// Apply scopedClassName to all nested lines
if (scopedClassName) {
html = html.replace(/\<span class="line"\>/g, `<span class="line ${scopedClassName}"`);
}
node.type = 'html';
node.value = html;
node.children = [];
});
};
};
export default remarkShiki;
}

4
packages/markdown/remark/src/types.ts
View file

@ -61,10 +61,6 @@ export interface ImageMetadata {
export interface MarkdownRenderingOptions extends AstroMarkdownOptions {
/** @internal */
fileURL?: URL;
/** @internal */
$?: {
scopedClassName: string | null;
};
/** Used for frontmatter injection plugins */
frontmatter?: Record<string, any>;
}

51
pnpm-lock.yaml
View file

@ -3984,9 +3984,6 @@ importers:
'@astrojs/markdown-remark':
specifier: workspace:*
version: link:../../markdown/remark
'@astrojs/prism':
specifier: workspace:*
version: link:../../astro-prism
'@mdx-js/mdx':
specifier: ^2.3.0
version: 2.3.0
@ -4014,18 +4011,12 @@ importers:
rehype-raw:
specifier: ^6.1.1
version: 6.1.1
remark-frontmatter:
specifier: ^4.0.1
version: 4.0.1
remark-gfm:
specifier: ^3.0.1
version: 3.0.1
remark-smartypants:
specifier: ^2.0.0
version: 2.0.0
shiki:
specifier: ^0.14.3
version: 0.14.3
source-map:
specifier: ^0.7.4
version: 0.7.4
@ -4083,7 +4074,7 @@ importers:
version: 4.0.3
rehype-pretty-code:
specifier: ^0.10.0
version: 0.10.0(shiki@0.14.3)
version: 0.10.0
remark-math:
specifier: ^5.1.1
version: 5.1.1
@ -11595,12 +11586,6 @@ packages:
dependencies:
reusify: 1.0.4
/fault@2.0.1:
resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==}
dependencies:
format: 0.2.2
dev: false
/fenceparser@1.1.1:
resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
engines: {node: '>=12'}
@ -11701,11 +11686,6 @@ packages:
combined-stream: 1.0.8
mime-types: 2.1.35
/format@0.2.2:
resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==}
engines: {node: '>=0.4.x'}
dev: false
/formdata-polyfill@4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
@ -13329,14 +13309,6 @@ packages:
transitivePeerDependencies:
- supports-color
/mdast-util-frontmatter@1.0.1:
resolution: {integrity: sha512-JjA2OjxRqAa8wEG8hloD0uTU0kdn8kbtOWpPP94NBkfAlbxn4S8gCGf/9DwFtEeGPXrDcNXdiDjVaRdUFqYokw==}
dependencies:
'@types/mdast': 3.0.12
mdast-util-to-markdown: 1.5.0
micromark-extension-frontmatter: 1.1.0
dev: false
/mdast-util-gfm-autolink-literal@1.0.3:
resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==}
dependencies:
@ -13599,15 +13571,6 @@ packages:
micromark-util-types: 1.0.2
uvu: 0.5.6
/micromark-extension-frontmatter@1.1.0:
resolution: {integrity: sha512-0nLelmvXR5aZ+F2IL6/Ed4cDnHLpL/VD/EELKuclsTWHrLI8UgxGHEmeoumeX2FXiM6z2WrBIOEcbKUZR8RYNg==}
dependencies:
fault: 2.0.1
micromark-util-character: 1.1.0
micromark-util-symbol: 1.0.1
micromark-util-types: 1.0.2
dev: false
/micromark-extension-gfm-autolink-literal@1.0.4:
resolution: {integrity: sha512-WCssN+M9rUyfHN5zPBn3/f0mIA7tqArHL/EKbv3CZK+LT2rG77FEikIQEqBkv46fOqXQK4NEW/Pc7Z27gshpeg==}
dependencies:
@ -15584,7 +15547,7 @@ packages:
unified: 10.1.2
dev: false
/rehype-pretty-code@0.10.0(shiki@0.14.3):
/rehype-pretty-code@0.10.0:
resolution: {integrity: sha512-qCD071Y+vUxEy9yyrATPk2+W9q7qCbzZgtc9suZhu75bmRQvOlBhJt4d3WvqSMTamkKoFkvqtCjyAk+ggH+aXQ==}
engines: {node: '>=16'}
peerDependencies:
@ -15593,7 +15556,6 @@ packages:
'@types/hast': 2.3.5
hash-obj: 4.0.0
parse-numeric-range: 1.3.0
shiki: 0.14.3
dev: true
/rehype-raw@6.1.1:
@ -15652,15 +15614,6 @@ packages:
dependencies:
unist-util-visit: 1.4.1
/remark-frontmatter@4.0.1:
resolution: {integrity: sha512-38fJrB0KnmD3E33a5jZC/5+gGAC2WKNiPw1/fdXJvijBlhA7RCsvJklrYJakS0HedninvaCYW8lQGf9C918GfA==}
dependencies:
'@types/mdast': 3.0.12
mdast-util-frontmatter: 1.0.1
micromark-extension-frontmatter: 1.1.0
unified: 10.1.2
dev: false
/remark-gfm@3.0.1:
resolution: {integrity: sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==}
dependencies: