Refactor content collection transforms (#6817)

* feat: json collection POC

* wip: add test json file

* refactor: rework content flag transforms

* refactor: simplify propagatedAsset check

* chore: remove JSON playground code

* chore: respect build sourcemap option

* deps: magic-string, source-map

* chore: formatting

* fix: add sourcemaps to MDX plugin

* chore: changeset

* deps: remove magic-string from mdx

* chore: remove unecessary MagicString

.changeset/dirty-panthers-approve.md

@@ -0,0 +1,6 @@
+---
+'astro': patch
+'@astrojs/mdx': patch
+---
+
+Fix sourcemap warnings when using Content Collections and MDX with the `vite.build.sourcemap` option

packages/astro/src/content/index.ts

@@ -5,3 +5,4 @@ export { contentObservable, getContentPaths, getDotAstroTypeReference } from './
 export { astroContentAssetPropagationPlugin } from './vite-plugin-content-assets.js';
 export { astroContentImportPlugin } from './vite-plugin-content-imports.js';
 export { astroContentVirtualModPlugin } from './vite-plugin-content-virtual-mod.js';
+export { CONTENT_FLAG, PROPAGATED_ASSET_FLAG } from './consts.js';

packages/astro/src/content/vite-plugin-content-assets.ts

@@ -18,12 +18,9 @@ import {
 } from './consts.js';
 import { getContentEntryExts } from './utils.js';
 
-function isPropagatedAsset(viteId: string, contentEntryExts: string[]): boolean {
+function isPropagatedAsset(viteId: string) {
-	const url = new URL(viteId, 'file://');
+	const flags = new URLSearchParams(viteId.split('?')[1]);
-	return (
+	return flags.has(PROPAGATED_ASSET_FLAG);
-		url.searchParams.has(PROPAGATED_ASSET_FLAG) &&
-		contentEntryExts.some((ext) => url.pathname.endsWith(ext))
-	);
 }
 
 export function astroContentAssetPropagationPlugin({
@@ -37,51 +34,56 @@ export function astroContentAssetPropagationPlugin({
 	const contentEntryExts = getContentEntryExts(settings);
 	return {
 		name: 'astro:content-asset-propagation',
-		enforce: 'pre',
 		configureServer(server) {
 			if (mode === 'dev') {
 				devModuleLoader = createViteLoader(server);
 			}
 		},
-		load(id) {
+		async transform(_, id, options) {
-			if (isPropagatedAsset(id, contentEntryExts)) {
+			if (isPropagatedAsset(id)) {
 				const basePath = id.split('?')[0];
+				let stringifiedLinks: string, stringifiedStyles: string, stringifiedScripts: string;
+
+				// We can access the server in dev,
+				// so resolve collected styles and scripts here.
+				if (options?.ssr && devModuleLoader) {
+					if (!devModuleLoader.getModuleById(basePath)?.ssrModule) {
+						await devModuleLoader.import(basePath);
+					}
+					const { stylesMap, urls } = await getStylesForURL(
+						pathToFileURL(basePath),
+						devModuleLoader,
+						'development'
+					);
+
+					const hoistedScripts = await getScriptsForURL(
+						pathToFileURL(basePath),
+						settings.config.root,
+						devModuleLoader
+					);
+
+					stringifiedLinks = JSON.stringify([...urls]);
+					stringifiedStyles = JSON.stringify([...stylesMap.values()]);
+					stringifiedScripts = JSON.stringify([...hoistedScripts]);
+				} else {
+					// Otherwise, use placeholders to inject styles and scripts
+					// during the production bundle step.
+					// @see the `astro:content-build-plugin` below.
+					stringifiedLinks = JSON.stringify(LINKS_PLACEHOLDER);
+					stringifiedStyles = JSON.stringify(STYLES_PLACEHOLDER);
+					stringifiedScripts = JSON.stringify(SCRIPTS_PLACEHOLDER);
+				}
+
 				const code = `
 					export async function getMod() {
 						return import(${JSON.stringify(basePath)});
 					}
-					export const collectedLinks = ${JSON.stringify(LINKS_PLACEHOLDER)};
+					export const collectedLinks = ${stringifiedLinks};
-					export const collectedStyles = ${JSON.stringify(STYLES_PLACEHOLDER)};
+					export const collectedStyles = ${stringifiedStyles};
-					export const collectedScripts = ${JSON.stringify(SCRIPTS_PLACEHOLDER)};
+					export const collectedScripts = ${stringifiedScripts};
 				`;
-				return { code };
-			}
-		},
-		async transform(code, id, options) {
-			if (!options?.ssr) return;
-			if (devModuleLoader && isPropagatedAsset(id, contentEntryExts)) {
-				const basePath = id.split('?')[0];
-				if (!devModuleLoader.getModuleById(basePath)?.ssrModule) {
-					await devModuleLoader.import(basePath);
-				}
-				const { stylesMap, urls } = await getStylesForURL(
-					pathToFileURL(basePath),
-					devModuleLoader,
-					'development'
-				);
 
-				const hoistedScripts = await getScriptsForURL(
+				return { code, map: { mappings: '' } };
-					pathToFileURL(basePath),
-					settings.config.root,
-					devModuleLoader
-				);
-
-				return {
-					code: code
-						.replace(JSON.stringify(LINKS_PLACEHOLDER), JSON.stringify([...urls]))
-						.replace(JSON.stringify(STYLES_PLACEHOLDER), JSON.stringify([...stylesMap.values()]))
-						.replace(JSON.stringify(SCRIPTS_PLACEHOLDER), JSON.stringify([...hoistedScripts])),
-				};
 			}
 		},
 	};

packages/astro/src/content/vite-plugin-content-imports.ts

@@ -21,9 +21,9 @@ import {
 	type ContentConfig,
 } from './utils.js';
 
-function isContentFlagImport(viteId: string, contentEntryExts: string[]) {
+function isContentFlagImport(viteId: string) {
-	const { searchParams, pathname } = new URL(viteId, 'file://');
+	const flags = new URLSearchParams(viteId.split('?')[1]);
-	return searchParams.has(CONTENT_FLAG) && contentEntryExts.some((ext) => pathname.endsWith(ext));
+	return flags.has(CONTENT_FLAG);
 }
 
 function getContentRendererByViteId(
@@ -65,26 +65,26 @@ export function astroContentImportPlugin({
 	const plugins: Plugin[] = [
 		{
 			name: 'astro:content-imports',
-			async load(viteId) {
+			async transform(_, viteId) {
-				if (isContentFlagImport(viteId, contentEntryExts)) {
+				if (isContentFlagImport(viteId)) {
-					const { fileId } = getFileInfo(viteId, settings.config);
+					const fileId = viteId.split('?')[0];
 					const { id, slug, collection, body, data, _internal } = await setContentEntryModuleCache({
 						fileId,
 						pluginContext: this,
 					});
 
 					const code = escapeViteEnvReferences(`
-export const id = ${JSON.stringify(id)};
+						export const id = ${JSON.stringify(id)};
-export const collection = ${JSON.stringify(collection)};
+						export const collection = ${JSON.stringify(collection)};
-export const slug = ${JSON.stringify(slug)};
+						export const slug = ${JSON.stringify(slug)};
-export const body = ${JSON.stringify(body)};
+						export const body = ${JSON.stringify(body)};
-export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
+						export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
-export const _internal = {
+						export const _internal = {
-	filePath: ${JSON.stringify(_internal.filePath)},
+							filePath: ${JSON.stringify(_internal.filePath)},
-	rawData: ${JSON.stringify(_internal.rawData)},
+							rawData: ${JSON.stringify(_internal.rawData)},
-};
+						};`);
-`);
+
-					return { code };
+					return { code, map: { mappings: '' } };
 				}
 			},
 			configureServer(viteServer) {
@@ -96,7 +96,7 @@ export const _internal = {
 						// Content modules depend on config, so we need to invalidate them.
 						for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {
 							if (
-								isContentFlagImport(modUrl, contentEntryExts) ||
+								isContentFlagImport(modUrl) ||
 								Boolean(getContentRendererByViteId(modUrl, settings))
 							) {
 								const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl);
@@ -108,13 +108,6 @@ export const _internal = {
 					}
 				});
 			},
-			async transform(code, id) {
-				if (isContentFlagImport(id, contentEntryExts)) {
-					// Escape before Rollup internal transform.
-					// Base on MUCH trial-and-error, inspired by MDX integration 2-step transform.
-					return { code: escapeViteEnvReferences(code) };
-				}
-			},
 		},
 	];
 

packages/astro/src/vite-plugin-jsx/index.ts

@@ -16,6 +16,7 @@ import { error } from '../core/logger/core.js';
 import { removeQueryString } from '../core/path.js';
 import { detectImportSource } from './import-source.js';
 import tagExportsPlugin from './tag.js';
+import { CONTENT_FLAG, PROPAGATED_ASSET_FLAG } from '../content/index.js';
 
 const JSX_EXTENSIONS = new Set(['.jsx', '.tsx', '.mdx']);
 const IMPORT_STATEMENTS: Record<string, string> = {
@@ -104,6 +105,11 @@ interface AstroPluginJSXOptions {
 	logging: LogOptions;
 }
 
+// Format inspired by https://github.com/vitejs/vite/blob/main/packages/vite/src/node/constants.ts#L54
+const SPECIAL_QUERY_REGEX = new RegExp(
+	`[?&](?:worker|sharedworker|raw|url|${CONTENT_FLAG}|${PROPAGATED_ASSET_FLAG})\\b`
+);
+
 /** Use Astro config to allow for alternate or multiple JSX renderers (by default Vite will assume React) */
 export default function jsx({ settings, logging }: AstroPluginJSXOptions): Plugin {
 	let viteConfig: ResolvedConfig;
@@ -133,6 +139,9 @@ export default function jsx({ settings, logging }: AstroPluginJSXOptions): Plugi
 		},
 		async transform(code, id, opts) {
 			const ssr = Boolean(opts?.ssr);
+			if (SPECIAL_QUERY_REGEX.test(id)) {
+				return null;
+			}
 			id = removeQueryString(id);
 			if (!JSX_EXTENSIONS.has(path.extname(id))) {
 				return null;

packages/integrations/mdx/package.json

@@ -45,6 +45,7 @@
     "remark-gfm": "^3.0.1",
     "remark-smartypants": "^2.0.0",
     "shiki": "^0.11.1",
+    "source-map": "^0.7.4",
     "unist-util-visit": "^4.1.0",
     "vfile": "^5.3.2"
   },

packages/integrations/mdx/src/index.ts

@@ -12,6 +12,7 @@ import { VFile } from 'vfile';
 import type { Plugin as VitePlugin } from 'vite';
 import { getRehypePlugins, getRemarkPlugins, recmaInjectImportMetaEnvPlugin } from './plugins.js';
 import { getFileInfo, ignoreStringPlugins, parseFrontmatter } from './utils.js';
+import { SourceMapGenerator } from 'source-map';
 
 export type MdxOptions = Omit<typeof markdownConfigDefaults, 'remarkPlugins' | 'rehypePlugins'> & {
 	extendMarkdownConfig: boolean;
@@ -113,6 +114,9 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 											...(mdxPluginOpts.recmaPlugins ?? []),
 											() => recmaInjectImportMetaEnvPlugin({ importMetaEnv }),
 										],
+										SourceMapGenerator: config.vite.build?.sourcemap
+											? SourceMapGenerator
+											: undefined,
 									});
 
 									return {
@@ -168,7 +172,7 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 											import.meta.hot.decline();
 										}`;
 									}
-									return escapeViteEnvReferences(code);
+									return { code: escapeViteEnvReferences(code), map: null };
 								},
 							},
 						] as VitePlugin[],

pnpm-lock.yaml

@@ -3254,12 +3254,13 @@ importers:
       remark-smartypants: ^2.0.0
       remark-toc: ^8.0.1
       shiki: ^0.11.1
+      source-map: ^0.7.4
       unist-util-visit: ^4.1.0
       vfile: ^5.3.2
       vite: ^4.2.1
     dependencies:
-      '@astrojs/markdown-remark': link:../../markdown/remark
+      '@astrojs/markdown-remark': 2.1.3_astro@packages+astro
-      '@astrojs/prism': link:../../astro-prism
+      '@astrojs/prism': 2.1.1
       '@mdx-js/mdx': 2.3.0
       '@mdx-js/rollup': 2.3.0
       acorn: 8.8.2
@@ -3273,6 +3274,7 @@ importers:
       remark-gfm: 3.0.1
       remark-smartypants: 2.0.0
       shiki: 0.11.1
+      source-map: 0.7.4
       unist-util-visit: 4.1.2
       vfile: 5.3.7
     devDependencies:
@@ -4304,6 +4306,36 @@ packages:
       vscode-uri: 3.0.7
     dev: false
 
+  /@astrojs/markdown-remark/2.1.3_astro@packages+astro:
+    resolution: {integrity: sha512-Di8Qbit9p7L7eqKklAJmiW9nVD+XMsNHpaNzCLduWjOonDu9fVgEzdjeDrTVCDtgrvkfhpAekuNXrp5+w4F91g==}
+    peerDependencies:
+      astro: '*'
+    dependencies:
+      '@astrojs/prism': 2.1.1
+      astro: link:packages/astro
+      github-slugger: 1.5.0
+      import-meta-resolve: 2.2.1
+      rehype-raw: 6.1.1
+      rehype-stringify: 9.0.3
+      remark-gfm: 3.0.1
+      remark-parse: 10.0.1
+      remark-rehype: 10.1.0
+      remark-smartypants: 2.0.0
+      shiki: 0.11.1
+      unified: 10.1.2
+      unist-util-visit: 4.1.2
+      vfile: 5.3.7
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@astrojs/prism/2.1.1:
+    resolution: {integrity: sha512-Gnwnlb1lGJzCQEg89r4/WqgfCGPNFC7Kuh2D/k289Cbdi/2PD7Lrdstz86y1itDvcb2ijiRqjqWnJ5rsfu/QOA==}
+    engines: {node: '>=16.12.0'}
+    dependencies:
+      prismjs: 1.29.0
+    dev: false
+
   /@babel/code-frame/7.18.6:
     resolution: {integrity: sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==}
     engines: {node: '>=6.9.0'}