From 85686373795440b380925649797e77656cb58b81 Mon Sep 17 00:00:00 2001 From: bluwy Date: Fri, 19 May 2023 21:19:05 +0800 Subject: [PATCH] MDX to Astro JSX somehow --- packages/astro/src/vite-plugin-jsx/index.ts | 39 +-- packages/integrations/mdx/package.json | 6 +- .../mdx/src/hast-util-to-astro-html.ts | 262 ++++++++++++++++++ packages/integrations/mdx/src/index.ts | 91 +++--- packages/integrations/mdx/src/plugins.ts | 3 + packages/integrations/mdx/src/rehype-astro.ts | 48 ++++ packages/integrations/mdx/src/utils.ts | 28 ++ pnpm-lock.yaml | 12 + 8 files changed, 419 insertions(+), 70 deletions(-) create mode 100644 packages/integrations/mdx/src/hast-util-to-astro-html.ts create mode 100644 packages/integrations/mdx/src/rehype-astro.ts diff --git a/packages/astro/src/vite-plugin-jsx/index.ts b/packages/astro/src/vite-plugin-jsx/index.ts index 91aa63909..af63564ce 100644 --- a/packages/astro/src/vite-plugin-jsx/index.ts +++ b/packages/astro/src/vite-plugin-jsx/index.ts @@ -153,25 +153,26 @@ export default function jsx({ settings, logging }: AstroPluginJSXOptions): Plugi const { mode } = viteConfig; // Shortcut: only use Astro renderer for MD and MDX files if (id.endsWith('.mdx')) { - const { code: jsxCode } = await transformWithEsbuild(code, id, { - loader: getEsbuildLoader(id), - jsx: 'preserve', - sourcemap: 'inline', - tsconfigRaw: { - compilerOptions: { - // Ensure client:only imports are treeshaken - importsNotUsedAsValues: 'remove', - }, - }, - }); - return transformJSX({ - code: jsxCode, - id, - renderer: astroJSXRenderer, - mode, - ssr, - root: settings.config.root, - }); + return + // const { code: jsxCode } = await transformWithEsbuild(code, id, { + // loader: getEsbuildLoader(id), + // jsx: 'preserve', + // sourcemap: 'inline', + // tsconfigRaw: { + // compilerOptions: { + // // Ensure client:only imports are treeshaken + // importsNotUsedAsValues: 'remove', + // }, + // }, + // }); + // return transformJSX({ + // code: jsxCode, + // id, + // renderer: astroJSXRenderer, + // mode, + // ssr, + // root: settings.config.root, + // }); } if (defaultJSXRendererEntry && jsxRenderersIntegrationOnly.size === 1) { // downlevel any non-standard syntax, but preserve JSX diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index 52627a5ca..baad7afad 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -43,7 +43,10 @@ "estree-util-visit": "^1.2.0", "github-slugger": "^1.4.0", "gray-matter": "^4.0.3", + "hast-util-to-html": "^8.0.4", + "html-void-elements": "^2.0.1", "kleur": "^4.1.4", + "property-information": "^6.2.0", "rehype-raw": "^6.1.1", "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.1", @@ -51,7 +54,8 @@ "shiki": "^0.14.1", "source-map": "^0.7.4", "unist-util-visit": "^4.1.0", - "vfile": "^5.3.2" + "vfile": "^5.3.2", + "zwitch": "^2.0.4" }, "devDependencies": { "@types/chai": "^4.3.1", diff --git a/packages/integrations/mdx/src/hast-util-to-astro-html.ts b/packages/integrations/mdx/src/hast-util-to-astro-html.ts new file mode 100644 index 000000000..cdde6ddca --- /dev/null +++ b/packages/integrations/mdx/src/hast-util-to-astro-html.ts @@ -0,0 +1,262 @@ +import { visit } from 'estree-util-visit'; +import { html, svg } from 'property-information'; +import { htmlVoidElements } from 'html-void-elements'; +import type { Options } from 'hast-util-to-html'; +import type { State as HtmlState } from 'hast-util-to-html/lib/types'; +import { zwitch } from 'zwitch'; +import { comment } from 'hast-util-to-html/lib/handle/comment.js'; +import { doctype } from 'hast-util-to-html/lib/handle/doctype.js'; +import { element } from 'hast-util-to-html/lib/handle/element.js'; +import { raw } from 'hast-util-to-html/lib/handle/raw.js'; +import { root } from 'hast-util-to-html/lib/handle/root.js'; +import { text } from 'hast-util-to-html/lib/handle/text.js'; +import { resolvePath } from './utils.js'; + +interface State extends HtmlState { + tree: any; + importer: string; + metadata: any; +} + +type Handler = (node: any, index: number | undefined, parent: any, state: State) => string; + +const mdxJsxFlowElement: Handler = (node, index, parent, state) => { + if (node.name.includes('.') || node.name.match(/^[A-Z]/)) { + const clientAttribute = node.attributes.find((attr: any) => attr.name.startsWith('client:')); + const clientValue = clientAttribute ? clientAttribute.name.slice(7) : undefined; + + let extraAttrs: string[] = []; + if (clientValue) { + extraAttrs.push(`"client:component-hydration":"${clientValue}"`); + + for (const rootChild of state.tree.children) { + if (rootChild.type === 'mdxjsEsm') { + const ast = rootChild.data.estree; + visit(ast, (n: any) => { + if (n.type === 'ImportDeclaration') { + const specifier = n.specifiers.find((s: any) => s.local.name === node.name); + if (!specifier) return; + + const component = { + exportName: specifier.imported ? specifier.imported.name : 'default', + specifier: n.source.value, + resolvedPath: resolvePath(n.source.value, state.importer), + }; + + // $$metadata.resolvePath will be postprocessed in Vite plugin later + extraAttrs.push(`"client:component-path":${JSON.stringify(component.resolvedPath)}`); + extraAttrs.push(`"client:component-export":${JSON.stringify(component.exportName)}`); + + if (clientValue === 'only') { + state.metadata.clientOnlyComponents.push(component); + } else { + state.metadata.hydratedComponents.push(component); + } + } + }); + } + } + } + + const slots: string[] = []; + const rootNode = + state.schema.space === 'html' && node.tagName === 'template' ? node.content : node; + { + const children = rootNode.children || []; + let defaultSlot = []; + let index = -1; + + while (++index < children.length) { + const child = children[index]; + const slotName = child.attributes?.find((a) => a.name === 'slot')?.value?.trim(); + if (slotName && slotName !== 'default') { + slots.push( + `"${slotName}": () => $$render\`${state.one.call(state, child, index, rootNode)}\`` + ); + } else { + defaultSlot.push(state.one.call(state, child, index, rootNode)); + } + } + + // consolidate default slot + if (defaultSlot.length) { + slots.push(`"default": () => $$render\`${defaultSlot.join('')}\``); + } + } + + const attrStr = serializeAttributesAsObjectString(node.attributes); + + return `\${$$renderComponent($$result, ${JSON.stringify(node.name)}, ${node.name}, {${ + attrStr ? attrStr + ',' : '' + } ${extraAttrs.join(',')}}${slots.length ? `, {${slots.join(',')}}` : ''})}`; + } else { + return `<${node.name}${serializeAttributesAsString(node.attributes)}>${state.all(node)}`; + } +}; + +function serializeAttributesAsString(attributes: any[]) { + return Object.values(attributes) + .map((attr) => { + // spread + if (attr.type === 'mdxJsxExpressionAttribute') { + const value = attr.value.trim(); + const varName = value.slice(3); + return `\${$$spreadAttribute(varName, ${JSON.stringify(varName)})}`; + } + // normal attribute + else if (attr.type === 'mdxJsxAttribute') { + if (attr.value == null) { + return ` ${attr.name}`; + } else if (typeof attr.value === 'string') { + return ` ${attr.name}="${attr.value}"`; + } else { + return `\${$$addAttribute(${attr.value.value}, ${JSON.stringify(attr.name)})}`; + } + } + }) + .join(''); +} + +function serializeAttributesAsObjectString(attributes: any[]) { + return Object.values(attributes) + .map((attr) => { + // spread + if (attr.type === 'mdxJsxExpressionAttribute') { + return attr.value; + } + // normal attribute + else if (attr.type === 'mdxJsxAttribute') { + if (attr.value == null) { + return `${JSON.stringify(attr.name)}: true`; + } else if (typeof attr.value === 'string') { + return `${JSON.stringify(attr.name)}: ${JSON.stringify(attr.value)}`; + } else { + return `${JSON.stringify(attr.name)}: ${attr.value.value}`; + } + } + }) + .join(','); +} + +const mdxFlowExpression: Handler = (node, index, parent, state) => { + const value = node.value.trim(); + if (!value || (value.startsWith('/*') && value.endsWith('*/'))) { + return ''; + } else { + return `\${${value}}`; + } +}; + +const handle = zwitch('type', { + invalid, + unknown, + handlers: { + comment, + doctype, + element, + raw, + root, + text, + mdxJsxFlowElement, + mdxJsxTextElement: mdxJsxFlowElement, + mdxFlowExpression, + mdxTextExpression: mdxFlowExpression, + }, +}); + +export function toAstroHtml(tree: any, options: Options, fullTree: any, importer: string) { + const options_ = options || {}; + const quote = options_.quote || '"'; + const alternative = quote === '"' ? "'" : '"'; + + if (quote !== '"' && quote !== "'") { + throw new Error('Invalid quote `' + quote + '`, expected `\'` or `"`'); + } + + const state: State = { + one, + all, + settings: { + omitOptionalTags: options_.omitOptionalTags || false, + allowParseErrors: options_.allowParseErrors || false, + allowDangerousCharacters: options_.allowDangerousCharacters || false, + quoteSmart: options_.quoteSmart || false, + preferUnquoted: options_.preferUnquoted || false, + tightAttributes: options_.tightAttributes || false, + upperDoctype: options_.upperDoctype || false, + tightDoctype: options_.tightDoctype || false, + bogusComments: options_.bogusComments || false, + tightCommaSeparatedLists: options_.tightCommaSeparatedLists || false, + tightSelfClosing: options_.tightSelfClosing || false, + collapseEmptyAttributes: options_.collapseEmptyAttributes || false, + allowDangerousHtml: options_.allowDangerousHtml || false, + voids: options_.voids || htmlVoidElements, + characterReferences: options_.characterReferences || options_.entities || {}, + closeSelfClosing: options_.closeSelfClosing || false, + closeEmptyElements: options_.closeEmptyElements || false, + }, + schema: options_.space === 'svg' ? svg : html, + quote, + alternative, + // Add original tree to find client: import path + tree: fullTree, + importer, + metadata: { + hydratedComponents: [], + clientOnlyComponents: [], + scripts: [], + propagation: 'none', + containsHead: false, + pageOptions: {}, + }, + }; + + // escape backticks. this would be more performant if it's done in hast-util-to-html raw + // directly. but i don't want to reimplement it. + visit(tree, (node) => { + if (typeof node.value === 'string') { + node.value = escapeTemplateLiterals(node.value); + } + }); + + const renderCode = state.one( + Array.isArray(tree) ? { type: 'root', children: tree } : tree, + undefined, + undefined + ); + + return { + renderCode, + metadata: state.metadata, + }; +} + +function one(this: State, node: any, index: number | undefined, parent: any) { + return handle(node, index, parent, this); +} + +function all(this: State, parent: any) { + const results: string[] = []; + const children = (parent && parent.children) || []; + let index = -1; + + while (++index < children.length) { + results[index] = this.one(children[index], index, parent); + } + + return results.join(''); +} + +function invalid(node: any) { + throw new Error('Expected node, not `' + node + '`'); +} + +function unknown(node: any) { + throw new Error('Cannot compile unknown node `' + node.type + '`'); +} + +function escapeTemplateLiterals(str: string) { + return str.replace(/\\/g, '\\\\').replace(/\`/g, '\\`').replace(/\$\{/g, '\\${'); +} diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index 2ccf66266..20240fc5e 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -1,10 +1,9 @@ import { markdownConfigDefaults } from '@astrojs/markdown-remark'; import { toRemarkInitializeAstroData } from '@astrojs/markdown-remark/dist/internal.js'; -import { compile as mdxCompile } from '@mdx-js/mdx'; +import { createProcessor } from '@mdx-js/mdx'; import type { PluggableList } from '@mdx-js/mdx/lib/core.js'; import mdxPlugin, { type Options as MdxRollupPluginOptions } from '@mdx-js/rollup'; import type { AstroIntegration, ContentEntryType, HookParameters } from 'astro'; -import { parse as parseESM } from 'es-module-lexer'; import fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; @@ -98,12 +97,14 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI if (!id.endsWith('mdx')) return; // Read code from file manually to prevent Vite from parsing `import.meta.env` expressions - const { fileId } = getFileInfo(id, config); + const { fileUrl, fileId } = getFileInfo(id, config); const code = await fs.readFile(fileId, 'utf-8'); const { data: frontmatter, content: pageContent } = parseFrontmatter(code, id); - const compiled = await mdxCompile(new VFile({ value: pageContent, path: id }), { + const vfile = new VFile({ value: pageContent, path: id }); + const processor = createProcessor({ ...mdxPluginOpts, + format: 'mdx', elementAttributeNameCase: 'html', remarkPlugins: [ // Ensure `data.astro` is available to all remark plugins @@ -119,60 +120,50 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI : undefined, }); - return { - code: escapeViteEnvReferences(String(compiled.value)), - map: compiled.map, - }; - }, - }, - { - name: '@astrojs/mdx-postprocess', - // These transforms must happen *after* JSX runtime transformations - transform(code, id) { - if (!id.endsWith('.mdx')) return; - - const [moduleImports, moduleExports] = parseESM(code); - - // Fragment import should already be injected, but check just to be safe. - const importsFromJSXRuntime = moduleImports - .filter(({ n }) => n === 'astro/jsx-runtime') - .map(({ ss, se }) => code.substring(ss, se)); - const hasFragmentImport = importsFromJSXRuntime.some((statement) => - /[\s,{](Fragment,|Fragment\s*})/.test(statement) - ); - if (!hasFragmentImport) { - code = 'import { Fragment } from "astro/jsx-runtime"\n' + code; - } - - const { fileUrl, fileId } = getFileInfo(id, config); - if (!moduleExports.find(({ n }) => n === 'url')) { - code += `\nexport const url = ${JSON.stringify(fileUrl)};`; - } - if (!moduleExports.find(({ n }) => n === 'file')) { - code += `\nexport const file = ${JSON.stringify(fileId)};`; - } - if (!moduleExports.find(({ n }) => n === 'Content')) { - // Make `Content` the default export so we can wrap `MDXContent` and pass in `Fragment` - code = code.replace('export default MDXContent;', ''); - code += `\nexport const Content = (props = {}) => MDXContent({ - ...props, - components: { Fragment, ...props.components }, - }); - export default Content;`; + // strip out recma plugins + const unwantedRecmaPluginNames = [ + 'recmaDocument', + 'recmaJsxRewrite', + 'recmaJsxBuild', + ]; + for (let i = 0; i < processor.attachers.length; i++) { + const attacher = processor.attachers[i]; + if (unwantedRecmaPluginNames.includes(attacher[0].name)) { + processor.attachers.splice(i, 1); + } } + const compiled = await processor.process(vfile); + let compiledCode = compiled.toString(); + // Remove `<>` from the end of the file + compiledCode = compiledCode.replace('<>;', ''); + // Add metadata + compiledCode += `\nexport const url = ${JSON.stringify(fileUrl)};`; + compiledCode += `\nexport const file = ${JSON.stringify(fileId)};`; // Ensures styles and scripts are injected into a `` // When a layout is not applied - code += `\nContent[Symbol.for('astro.needsHeadRendering')] = !Boolean(frontmatter.layout);`; - code += `\nContent.moduleId = ${JSON.stringify(id)};`; + compiledCode += `\nContent[Symbol.for('astro.needsHeadRendering')] = !Boolean(frontmatter.layout);`; + compiledCode += `\nContent.moduleId = ${JSON.stringify(id)};`; if (command === 'dev') { // TODO: decline HMR updates until we have a stable approach - code += `\nif (import.meta.hot) { - import.meta.hot.decline(); - }`; + compiledCode += `\nif (import.meta.hot) { + import.meta.hot.decline(); +}`; } - return { code: escapeViteEnvReferences(code), map: null }; + + // console.log(compiledCode) + + return { + code: escapeViteEnvReferences(compiledCode), + map: compiled.map, + meta: { + astro: vfile.data.rehypeAstro, + vite: { + lang: 'ts', + }, + }, + }; }, }, ] as VitePlugin[], diff --git a/packages/integrations/mdx/src/plugins.ts b/packages/integrations/mdx/src/plugins.ts index 859cf3bbe..d557d3e44 100644 --- a/packages/integrations/mdx/src/plugins.ts +++ b/packages/integrations/mdx/src/plugins.ts @@ -20,6 +20,7 @@ import { remarkImageToComponent } from './remark-images-to-component.js'; import remarkPrism from './remark-prism.js'; import remarkShiki from './remark-shiki.js'; import { jsToTreeNode } from './utils.js'; +import { rehypeAstro } from './rehype-astro.js'; // Skip nonessential plugins during performance benchmark runs const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK); @@ -144,6 +145,8 @@ export function getRehypePlugins(mdxOptions: MdxOptions): MdxRollupPluginOptions ...(isPerformanceBenchmark ? [] : [rehypeHeadingIds, rehypeInjectHeadingsExport]), // computed from `astro.data.frontmatter` in VFile data rehypeApplyFrontmatterExport, + // render hast to js using Astro's runtime + rehypeAstro, ]; return rehypePlugins; } diff --git a/packages/integrations/mdx/src/rehype-astro.ts b/packages/integrations/mdx/src/rehype-astro.ts new file mode 100644 index 000000000..e320a92fe --- /dev/null +++ b/packages/integrations/mdx/src/rehype-astro.ts @@ -0,0 +1,48 @@ +import { toAstroHtml } from './hast-util-to-astro-html.js'; +import { jsToTreeNode } from './utils.js'; + +export function rehypeAstro() { + return function (tree: any, vfile: any) { + const newChildren = []; + const contentNodes = []; + + // hoist all esm code to top + for (const child of tree.children) { + if (child.type === 'mdxjsEsm') { + newChildren.push(child); + } else { + contentNodes.push(child); + } + } + + const { renderCode, metadata } = toAstroHtml(contentNodes, {}, tree, vfile.path); + + const js = ` +import { + Fragment, + render as $$render, + createComponent as $$createComponent, + renderComponent as $$renderComponent, + addAttribute as $$addAttribute, + spreadAttributes as $$spreadAttributes +} from "astro/server/index.js"; + +export const Content = $$createComponent(async ($$result, $$props, $$slots) => { + return $$render\`${renderCode}\`; +}); + +export default Content;`; + + try { + newChildren.push(jsToTreeNode(js)); + } catch (e) { + console.log('failed to parse', js); + throw e; + } + + // mutate tree as js entirely + tree.children = newChildren; + + vfile.data.rehypeAstro = metadata; + }; +} diff --git a/packages/integrations/mdx/src/utils.ts b/packages/integrations/mdx/src/utils.ts index 80f8c3e20..6357106d6 100644 --- a/packages/integrations/mdx/src/utils.ts +++ b/packages/integrations/mdx/src/utils.ts @@ -1,3 +1,5 @@ +import npath from 'path'; +import fs from 'fs'; import type { PluggableList } from '@mdx-js/mdx/lib/core.js'; import type { Options as AcornOpts } from 'acorn'; import { parse } from 'acorn'; @@ -106,3 +108,29 @@ export function ignoreStringPlugins(plugins: any[]): PluggableList { } return validPlugins; } + +export function resolveJsToTs(filePath: string) { + if (filePath.endsWith('.jsx') && !fs.existsSync(filePath)) { + const tryPath = filePath.slice(0, -4) + '.tsx'; + if (fs.existsSync(tryPath)) { + return tryPath; + } + } + return filePath; +} + +/** + * Resolve the hydration paths so that it can be imported in the client + */ +export function resolvePath(specifier: string, importer: string) { + if (specifier.startsWith('.')) { + const absoluteSpecifier = npath.resolve(npath.dirname(importer), specifier); + return resolveJsToTs(normalizePath(absoluteSpecifier)); + } else { + return specifier; + } +} + +export function normalizePath(id: string): string { + return npath.posix.normalize(id.replace(/\\/g, '/')); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d75e43cba..41d80153a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4140,9 +4140,18 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 + hast-util-to-html: + specifier: ^8.0.4 + version: 8.0.4 + html-void-elements: + specifier: ^2.0.1 + version: 2.0.1 kleur: specifier: ^4.1.4 version: 4.1.5 + property-information: + specifier: ^6.2.0 + version: 6.2.0 rehype-raw: specifier: ^6.1.1 version: 6.1.1 @@ -4167,6 +4176,9 @@ importers: vfile: specifier: ^5.3.2 version: 5.3.2 + zwitch: + specifier: ^2.0.4 + version: 2.0.4 devDependencies: '@types/chai': specifier: ^4.3.1