MDX to Astro JSX somehow

packages/astro/src/vite-plugin-jsx/index.ts

@@ -153,25 +153,26 @@ export default function jsx({ settings, logging }: AstroPluginJSXOptions): Plugi
 			const { mode } = viteConfig;
 			// Shortcut: only use Astro renderer for MD and MDX files
 			if (id.endsWith('.mdx')) {
-				const { code: jsxCode } = await transformWithEsbuild(code, id, {
-					loader: getEsbuildLoader(id),
-					jsx: 'preserve',
-					sourcemap: 'inline',
-					tsconfigRaw: {
-						compilerOptions: {
-							// Ensure client:only imports are treeshaken
-							importsNotUsedAsValues: 'remove',
-						},
-					},
-				});
-				return transformJSX({
-					code: jsxCode,
-					id,
-					renderer: astroJSXRenderer,
-					mode,
-					ssr,
-					root: settings.config.root,
-				});
+				return 
+				// const { code: jsxCode } = await transformWithEsbuild(code, id, {
+				// 	loader: getEsbuildLoader(id),
+				// 	jsx: 'preserve',
+				// 	sourcemap: 'inline',
+				// 	tsconfigRaw: {
+				// 		compilerOptions: {
+				// 			// Ensure client:only imports are treeshaken
+				// 			importsNotUsedAsValues: 'remove',
+				// 		},
+				// 	},
+				// });
+				// return transformJSX({
+				// 	code: jsxCode,
+				// 	id,
+				// 	renderer: astroJSXRenderer,
+				// 	mode,
+				// 	ssr,
+				// 	root: settings.config.root,
+				// });
 			}
 			if (defaultJSXRendererEntry && jsxRenderersIntegrationOnly.size === 1) {
 				// downlevel any non-standard syntax, but preserve JSX

packages/integrations/mdx/package.json

@@ -43,7 +43,10 @@
     "estree-util-visit": "^1.2.0",
     "github-slugger": "^1.4.0",
     "gray-matter": "^4.0.3",
+    "hast-util-to-html": "^8.0.4",
+    "html-void-elements": "^2.0.1",
     "kleur": "^4.1.4",
+    "property-information": "^6.2.0",
     "rehype-raw": "^6.1.1",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
@@ -51,7 +54,8 @@
     "shiki": "^0.14.1",
     "source-map": "^0.7.4",
     "unist-util-visit": "^4.1.0",
-    "vfile": "^5.3.2"
+    "vfile": "^5.3.2",
+    "zwitch": "^2.0.4"
   },
   "devDependencies": {
     "@types/chai": "^4.3.1",

packages/integrations/mdx/src/hast-util-to-astro-html.ts

@@ -0,0 +1,262 @@
+import { visit } from 'estree-util-visit';
+import { html, svg } from 'property-information';
+import { htmlVoidElements } from 'html-void-elements';
+import type { Options } from 'hast-util-to-html';
+import type { State as HtmlState } from 'hast-util-to-html/lib/types';
+import { zwitch } from 'zwitch';
+import { comment } from 'hast-util-to-html/lib/handle/comment.js';
+import { doctype } from 'hast-util-to-html/lib/handle/doctype.js';
+import { element } from 'hast-util-to-html/lib/handle/element.js';
+import { raw } from 'hast-util-to-html/lib/handle/raw.js';
+import { root } from 'hast-util-to-html/lib/handle/root.js';
+import { text } from 'hast-util-to-html/lib/handle/text.js';
+import { resolvePath } from './utils.js';
+
+interface State extends HtmlState {
+	tree: any;
+	importer: string;
+	metadata: any;
+}
+
+type Handler = (node: any, index: number | undefined, parent: any, state: State) => string;
+
+const mdxJsxFlowElement: Handler = (node, index, parent, state) => {
+	if (node.name.includes('.') || node.name.match(/^[A-Z]/)) {
+		const clientAttribute = node.attributes.find((attr: any) => attr.name.startsWith('client:'));
+		const clientValue = clientAttribute ? clientAttribute.name.slice(7) : undefined;
+
+		let extraAttrs: string[] = [];
+		if (clientValue) {
+			extraAttrs.push(`"client:component-hydration":"${clientValue}"`);
+
+			for (const rootChild of state.tree.children) {
+				if (rootChild.type === 'mdxjsEsm') {
+					const ast = rootChild.data.estree;
+					visit(ast, (n: any) => {
+						if (n.type === 'ImportDeclaration') {
+							const specifier = n.specifiers.find((s: any) => s.local.name === node.name);
+							if (!specifier) return;
+
+							const component = {
+								exportName: specifier.imported ? specifier.imported.name : 'default',
+								specifier: n.source.value,
+								resolvedPath: resolvePath(n.source.value, state.importer),
+							};
+
+							// $$metadata.resolvePath will be postprocessed in Vite plugin later
+							extraAttrs.push(`"client:component-path":${JSON.stringify(component.resolvedPath)}`);
+							extraAttrs.push(`"client:component-export":${JSON.stringify(component.exportName)}`);
+
+							if (clientValue === 'only') {
+								state.metadata.clientOnlyComponents.push(component);
+							} else {
+								state.metadata.hydratedComponents.push(component);
+							}
+						}
+					});
+				}
+			}
+		}
+
+		const slots: string[] = [];
+		const rootNode =
+			state.schema.space === 'html' && node.tagName === 'template' ? node.content : node;
+		{
+			const children = rootNode.children || [];
+			let defaultSlot = [];
+			let index = -1;
+
+			while (++index < children.length) {
+				const child = children[index];
+				const slotName = child.attributes?.find((a) => a.name === 'slot')?.value?.trim();
+				if (slotName && slotName !== 'default') {
+					slots.push(
+						`"${slotName}": () => $$render\`${state.one.call(state, child, index, rootNode)}\``
+					);
+				} else {
+					defaultSlot.push(state.one.call(state, child, index, rootNode));
+				}
+			}
+
+			// consolidate default slot
+			if (defaultSlot.length) {
+				slots.push(`"default": () => $$render\`${defaultSlot.join('')}\``);
+			}
+		}
+
+		const attrStr = serializeAttributesAsObjectString(node.attributes);
+
+		return `\${$$renderComponent($$result, ${JSON.stringify(node.name)}, ${node.name}, {${
+			attrStr ? attrStr + ',' : ''
+		} ${extraAttrs.join(',')}}${slots.length ? `, {${slots.join(',')}}` : ''})}`;
+	} else {
+		return `<${node.name}${serializeAttributesAsString(node.attributes)}>${state.all(node)}</${
+			node.name
+		}>`;
+	}
+};
+
+function serializeAttributesAsString(attributes: any[]) {
+	return Object.values(attributes)
+		.map((attr) => {
+			// spread
+			if (attr.type === 'mdxJsxExpressionAttribute') {
+				const value = attr.value.trim();
+				const varName = value.slice(3);
+				return `\${$$spreadAttribute(varName, ${JSON.stringify(varName)})}`;
+			}
+			// normal attribute
+			else if (attr.type === 'mdxJsxAttribute') {
+				if (attr.value == null) {
+					return ` ${attr.name}`;
+				} else if (typeof attr.value === 'string') {
+					return ` ${attr.name}="${attr.value}"`;
+				} else {
+					return `\${$$addAttribute(${attr.value.value}, ${JSON.stringify(attr.name)})}`;
+				}
+			}
+		})
+		.join('');
+}
+
+function serializeAttributesAsObjectString(attributes: any[]) {
+	return Object.values(attributes)
+		.map((attr) => {
+			// spread
+			if (attr.type === 'mdxJsxExpressionAttribute') {
+				return attr.value;
+			}
+			// normal attribute
+			else if (attr.type === 'mdxJsxAttribute') {
+				if (attr.value == null) {
+					return `${JSON.stringify(attr.name)}: true`;
+				} else if (typeof attr.value === 'string') {
+					return `${JSON.stringify(attr.name)}: ${JSON.stringify(attr.value)}`;
+				} else {
+					return `${JSON.stringify(attr.name)}: ${attr.value.value}`;
+				}
+			}
+		})
+		.join(',');
+}
+
+const mdxFlowExpression: Handler = (node, index, parent, state) => {
+	const value = node.value.trim();
+	if (!value || (value.startsWith('/*') && value.endsWith('*/'))) {
+		return '';
+	} else {
+		return `\${${value}}`;
+	}
+};
+
+const handle = zwitch('type', {
+	invalid,
+	unknown,
+	handlers: {
+		comment,
+		doctype,
+		element,
+		raw,
+		root,
+		text,
+		mdxJsxFlowElement,
+		mdxJsxTextElement: mdxJsxFlowElement,
+		mdxFlowExpression,
+		mdxTextExpression: mdxFlowExpression,
+	},
+});
+
+export function toAstroHtml(tree: any, options: Options, fullTree: any, importer: string) {
+	const options_ = options || {};
+	const quote = options_.quote || '"';
+	const alternative = quote === '"' ? "'" : '"';
+
+	if (quote !== '"' && quote !== "'") {
+		throw new Error('Invalid quote `' + quote + '`, expected `\'` or `"`');
+	}
+
+	const state: State = {
+		one,
+		all,
+		settings: {
+			omitOptionalTags: options_.omitOptionalTags || false,
+			allowParseErrors: options_.allowParseErrors || false,
+			allowDangerousCharacters: options_.allowDangerousCharacters || false,
+			quoteSmart: options_.quoteSmart || false,
+			preferUnquoted: options_.preferUnquoted || false,
+			tightAttributes: options_.tightAttributes || false,
+			upperDoctype: options_.upperDoctype || false,
+			tightDoctype: options_.tightDoctype || false,
+			bogusComments: options_.bogusComments || false,
+			tightCommaSeparatedLists: options_.tightCommaSeparatedLists || false,
+			tightSelfClosing: options_.tightSelfClosing || false,
+			collapseEmptyAttributes: options_.collapseEmptyAttributes || false,
+			allowDangerousHtml: options_.allowDangerousHtml || false,
+			voids: options_.voids || htmlVoidElements,
+			characterReferences: options_.characterReferences || options_.entities || {},
+			closeSelfClosing: options_.closeSelfClosing || false,
+			closeEmptyElements: options_.closeEmptyElements || false,
+		},
+		schema: options_.space === 'svg' ? svg : html,
+		quote,
+		alternative,
+		// Add original tree to find client: import path
+		tree: fullTree,
+		importer,
+		metadata: {
+			hydratedComponents: [],
+			clientOnlyComponents: [],
+			scripts: [],
+			propagation: 'none',
+			containsHead: false,
+			pageOptions: {},
+		},
+	};
+
+	// escape backticks. this would be more performant if it's done in hast-util-to-html raw
+	// directly. but i don't want to reimplement it.
+	visit(tree, (node) => {
+		if (typeof node.value === 'string') {
+			node.value = escapeTemplateLiterals(node.value);
+		}
+	});
+
+	const renderCode = state.one(
+		Array.isArray(tree) ? { type: 'root', children: tree } : tree,
+		undefined,
+		undefined
+	);
+
+	return {
+		renderCode,
+		metadata: state.metadata,
+	};
+}
+
+function one(this: State, node: any, index: number | undefined, parent: any) {
+	return handle(node, index, parent, this);
+}
+
+function all(this: State, parent: any) {
+	const results: string[] = [];
+	const children = (parent && parent.children) || [];
+	let index = -1;
+
+	while (++index < children.length) {
+		results[index] = this.one(children[index], index, parent);
+	}
+
+	return results.join('');
+}
+
+function invalid(node: any) {
+	throw new Error('Expected node, not `' + node + '`');
+}
+
+function unknown(node: any) {
+	throw new Error('Cannot compile unknown node `' + node.type + '`');
+}
+
+function escapeTemplateLiterals(str: string) {
+	return str.replace(/\\/g, '\\\\').replace(/\`/g, '\\`').replace(/\$\{/g, '\\${');
+}

packages/integrations/mdx/src/index.ts

@@ -1,10 +1,9 @@
 import { markdownConfigDefaults } from '@astrojs/markdown-remark';
 import { toRemarkInitializeAstroData } from '@astrojs/markdown-remark/dist/internal.js';
-import { compile as mdxCompile } from '@mdx-js/mdx';
+import { createProcessor } from '@mdx-js/mdx';
 import type { PluggableList } from '@mdx-js/mdx/lib/core.js';
 import mdxPlugin, { type Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
 import type { AstroIntegration, ContentEntryType, HookParameters } from 'astro';
-import { parse as parseESM } from 'es-module-lexer';
 import fs from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 import type { Options as RemarkRehypeOptions } from 'remark-rehype';
@@ -98,12 +97,14 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 									if (!id.endsWith('mdx')) return;
 
 									// Read code from file manually to prevent Vite from parsing `import.meta.env` expressions
-									const { fileId } = getFileInfo(id, config);
+									const { fileUrl, fileId } = getFileInfo(id, config);
 									const code = await fs.readFile(fileId, 'utf-8');
 
 									const { data: frontmatter, content: pageContent } = parseFrontmatter(code, id);
-									const compiled = await mdxCompile(new VFile({ value: pageContent, path: id }), {
+									const vfile = new VFile({ value: pageContent, path: id });
+									const processor = createProcessor({
 										...mdxPluginOpts,
+										format: 'mdx',
 										elementAttributeNameCase: 'html',
 										remarkPlugins: [
 											// Ensure `data.astro` is available to all remark plugins
@@ -119,60 +120,50 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 											: undefined,
 									});
 
-									return {
-										code: escapeViteEnvReferences(String(compiled.value)),
-										map: compiled.map,
-									};
-								},
-							},
-							{
-								name: '@astrojs/mdx-postprocess',
-								// These transforms must happen *after* JSX runtime transformations
-								transform(code, id) {
-									if (!id.endsWith('.mdx')) return;
-
-									const [moduleImports, moduleExports] = parseESM(code);
-
-									// Fragment import should already be injected, but check just to be safe.
-									const importsFromJSXRuntime = moduleImports
-										.filter(({ n }) => n === 'astro/jsx-runtime')
-										.map(({ ss, se }) => code.substring(ss, se));
-									const hasFragmentImport = importsFromJSXRuntime.some((statement) =>
-										/[\s,{](Fragment,|Fragment\s*})/.test(statement)
-									);
-									if (!hasFragmentImport) {
-										code = 'import { Fragment } from "astro/jsx-runtime"\n' + code;
-									}
-
-									const { fileUrl, fileId } = getFileInfo(id, config);
-									if (!moduleExports.find(({ n }) => n === 'url')) {
-										code += `\nexport const url = ${JSON.stringify(fileUrl)};`;
-									}
-									if (!moduleExports.find(({ n }) => n === 'file')) {
-										code += `\nexport const file = ${JSON.stringify(fileId)};`;
-									}
-									if (!moduleExports.find(({ n }) => n === 'Content')) {
-										// Make `Content` the default export so we can wrap `MDXContent` and pass in `Fragment`
-										code = code.replace('export default MDXContent;', '');
-										code += `\nexport const Content = (props = {}) => MDXContent({
-											...props,
-											components: { Fragment, ...props.components },
-										});
-										export default Content;`;
+									// strip out recma plugins
+									const unwantedRecmaPluginNames = [
+										'recmaDocument',
+										'recmaJsxRewrite',
+										'recmaJsxBuild',
+									];
+									for (let i = 0; i < processor.attachers.length; i++) {
+										const attacher = processor.attachers[i];
+										if (unwantedRecmaPluginNames.includes(attacher[0].name)) {
+											processor.attachers.splice(i, 1);
+										}
 									}
 
+									const compiled = await processor.process(vfile);
+									let compiledCode = compiled.toString();
+									// Remove `<></>` from the end of the file
+									compiledCode = compiledCode.replace('<></>;', '');
+									// Add metadata
+									compiledCode += `\nexport const url = ${JSON.stringify(fileUrl)};`;
+									compiledCode += `\nexport const file = ${JSON.stringify(fileId)};`;
 									// Ensures styles and scripts are injected into a `<head>`
 									// When a layout is not applied
-									code += `\nContent[Symbol.for('astro.needsHeadRendering')] = !Boolean(frontmatter.layout);`;
-									code += `\nContent.moduleId = ${JSON.stringify(id)};`;
+									compiledCode += `\nContent[Symbol.for('astro.needsHeadRendering')] = !Boolean(frontmatter.layout);`;
+									compiledCode += `\nContent.moduleId = ${JSON.stringify(id)};`;
 
 									if (command === 'dev') {
 										// TODO: decline HMR updates until we have a stable approach
-										code += `\nif (import.meta.hot) {
-											import.meta.hot.decline();
-										}`;
+										compiledCode += `\nif (import.meta.hot) {
+	import.meta.hot.decline();
+}`;
 									}
-									return { code: escapeViteEnvReferences(code), map: null };
+
+									// console.log(compiledCode)
+
+									return {
+										code: escapeViteEnvReferences(compiledCode),
+										map: compiled.map,
+										meta: {
+											astro: vfile.data.rehypeAstro,
+											vite: {
+												lang: 'ts',
+											},
+										},
+									};
 								},
 							},
 						] as VitePlugin[],

packages/integrations/mdx/src/plugins.ts

@@ -20,6 +20,7 @@ import { remarkImageToComponent } from './remark-images-to-component.js';
 import remarkPrism from './remark-prism.js';
 import remarkShiki from './remark-shiki.js';
 import { jsToTreeNode } from './utils.js';
+import { rehypeAstro } from './rehype-astro.js';
 
 // Skip nonessential plugins during performance benchmark runs
 const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK);
@@ -144,6 +145,8 @@ export function getRehypePlugins(mdxOptions: MdxOptions): MdxRollupPluginOptions
 		...(isPerformanceBenchmark ? [] : [rehypeHeadingIds, rehypeInjectHeadingsExport]),
 		// computed from `astro.data.frontmatter` in VFile data
 		rehypeApplyFrontmatterExport,
+		// render hast to js using Astro's runtime
+		rehypeAstro,
 	];
 	return rehypePlugins;
 }

packages/integrations/mdx/src/rehype-astro.ts

@@ -0,0 +1,48 @@
+import { toAstroHtml } from './hast-util-to-astro-html.js';
+import { jsToTreeNode } from './utils.js';
+
+export function rehypeAstro() {
+	return function (tree: any, vfile: any) {
+		const newChildren = [];
+		const contentNodes = [];
+
+		// hoist all esm code to top
+		for (const child of tree.children) {
+			if (child.type === 'mdxjsEsm') {
+				newChildren.push(child);
+			} else {
+				contentNodes.push(child);
+			}
+		}
+
+		const { renderCode, metadata } = toAstroHtml(contentNodes, {}, tree, vfile.path);
+
+		const js = `
+import {
+	Fragment,
+	render as $$render,
+	createComponent as $$createComponent,
+	renderComponent as $$renderComponent,
+	addAttribute as $$addAttribute,
+	spreadAttributes as $$spreadAttributes
+} from "astro/server/index.js";
+
+export const Content = $$createComponent(async ($$result, $$props, $$slots) => {
+  return $$render\`${renderCode}\`;
+});
+
+export default Content;`;
+
+		try {
+			newChildren.push(jsToTreeNode(js));
+		} catch (e) {
+			console.log('failed to parse', js);
+			throw e;
+		}
+
+		// mutate tree as js entirely
+		tree.children = newChildren;
+
+		vfile.data.rehypeAstro = metadata;
+	};
+}

packages/integrations/mdx/src/utils.ts

@@ -1,3 +1,5 @@
+import npath from 'path';
+import fs from 'fs';
 import type { PluggableList } from '@mdx-js/mdx/lib/core.js';
 import type { Options as AcornOpts } from 'acorn';
 import { parse } from 'acorn';
@@ -106,3 +108,29 @@ export function ignoreStringPlugins(plugins: any[]): PluggableList {
 	}
 	return validPlugins;
 }
+
+export function resolveJsToTs(filePath: string) {
+	if (filePath.endsWith('.jsx') && !fs.existsSync(filePath)) {
+		const tryPath = filePath.slice(0, -4) + '.tsx';
+		if (fs.existsSync(tryPath)) {
+			return tryPath;
+		}
+	}
+	return filePath;
+}
+
+/**
+ * Resolve the hydration paths so that it can be imported in the client
+ */
+export function resolvePath(specifier: string, importer: string) {
+	if (specifier.startsWith('.')) {
+		const absoluteSpecifier = npath.resolve(npath.dirname(importer), specifier);
+		return resolveJsToTs(normalizePath(absoluteSpecifier));
+	} else {
+		return specifier;
+	}
+}
+
+export function normalizePath(id: string): string {
+	return npath.posix.normalize(id.replace(/\\/g, '/'));
+}

pnpm-lock.yaml

@@ -4140,9 +4140,18 @@ importers:
       gray-matter:
         specifier: ^4.0.3
         version: 4.0.3
+      hast-util-to-html:
+        specifier: ^8.0.4
+        version: 8.0.4
+      html-void-elements:
+        specifier: ^2.0.1
+        version: 2.0.1
       kleur:
         specifier: ^4.1.4
         version: 4.1.5
+      property-information:
+        specifier: ^6.2.0
+        version: 6.2.0
       rehype-raw:
         specifier: ^6.1.1
         version: 6.1.1
@@ -4167,6 +4176,9 @@ importers:
       vfile:
         specifier: ^5.3.2
         version: 5.3.2
+      zwitch:
+        specifier: ^2.0.4
+        version: 2.0.4
     devDependencies:
       '@types/chai':
         specifier: ^4.3.1