[next] Fix Astro.fetchContent (#1480)

* fix Astro.fetchContent

* fix(fetchContent): cast type

Co-authored-by: Nate Moore <nate@skypack.dev>

packages/astro/package.json

@@ -51,6 +51,8 @@
     "@astrojs/renderer-vue": "0.1.9",
     "@babel/code-frame": "^7.12.13",
     "@babel/core": "^7.15.5",
+    "@babel/traverse": "^7.15.4",
+    "@babel/types": "^7.15.6",
     "@web/rollup-plugin-html": "^1.10.1",
     "astring": "^1.7.5",
     "cheerio": "^1.0.0-rc.10",
@@ -73,6 +75,8 @@
     "shiki": "^0.9.10",
     "shorthash": "^0.0.2",
     "slash": "^4.0.0",
+    "sourcemap-codec": "^1.4.8",
+    "srcset-parse": "^1.1.0",
     "string-width": "^5.0.0",
     "strip-ansi": "^7.0.1",
     "supports-esm": "^1.0.0",

packages/astro/src/runtime/fetch-content.ts

@@ -1,23 +0,0 @@
-/**
- * Convert the result of an `import.meta.globEager()` call to an array of processed
- * Markdown content objects. Filter out any non-Markdown files matched in the glob
- * result, by default.
- */
-export function fetchContent(importMetaGlobResult: Record<string, any>, url: string) {
-  return [...Object.entries(importMetaGlobResult)]
-    .map(([spec, mod]) => {
-      // Only return Markdown files, which export the __content object.
-      if (!mod.__content) {
-        return;
-      }
-      const urlSpec = new URL(spec, url.replace(/^(file:\/\/)?/, 'file://')).href; // note: "href" will always be forward-slashed ("pathname" may not be)
-      if (!urlSpec.includes('/pages/')) {
-        return mod.__content;
-      }
-      return {
-        ...mod.__content,
-        url: urlSpec.replace(/^.*\/pages\//, '/').replace(/\.md$/, ''),
-      };
-    })
-    .filter(Boolean);
-}

packages/astro/src/runtime/ssr.ts

@@ -2,7 +2,7 @@ import type { BuildResult } from 'esbuild';
 import type { ViteDevServer } from 'vite';
 import type { AstroConfig, ComponentInstance, GetStaticPathsResult, Params, Props, Renderer, RouteCache, RouteData, RuntimeMode, SSRError } from '../@types/astro';
 import type { SSRResult } from '../@types/ssr';
-import type { FetchContentResultBase, FetchContentResult } from '../@types/astro-file';
+import type { Astro } from '../@types/astro-file';
 import type { LogOptions } from '../logger';
 
 import cheerio from 'cheerio';
@@ -123,6 +123,30 @@ async function resolveImportedModules(viteServer: ViteDevServer, file: URL) {
   return importedModules;
 }
 
+/** Create the Astro.fetchContent() runtime function. */
+function createFetchContentFn(url: URL) {
+  const fetchContent = (importMetaGlobResult: Record<string, any>) => {
+    let allEntries = [...Object.entries(importMetaGlobResult)];
+    if (allEntries.length === 0) {
+      throw new Error(`[${url.pathname}] Astro.fetchContent() no matches found.`);
+    }
+    return allEntries
+      .map(([spec, mod]) => {
+        // Only return Markdown files for now.
+        if (!mod.frontmatter) {
+          return;
+        }
+        return {
+          content: mod.metadata,
+          metadata: mod.frontmatter,
+          file: new URL(spec, url),
+        };
+      })
+      .filter(Boolean);
+  };
+  return fetchContent;
+};
+
 /** use Vite to SSR */
 export async function ssr({ astroConfig, filePath, logging, mode, origin, pathname, route, routeCache, viteServer }: SSROptions): Promise<string> {
   try {
@@ -185,7 +209,8 @@ export async function ssr({ astroConfig, filePath, logging, mode, origin, pathna
         const site = new URL(origin);
         const url = new URL('.' + pathname, site);
         const canonicalURL = getCanonicalURL(pathname, astroConfig.buildOptions.site || origin);
-        const fetchContent = createFetchContent(fileURLToPath(filePath));
+        // Cast this type because the actual fetchContent implementation relies on import.meta.globEager
+        const fetchContent = createFetchContentFn(filePath) as unknown as Astro['fetchContent'];
         return {
           isPage: true,
           site,
@@ -208,34 +233,6 @@ export async function ssr({ astroConfig, filePath, logging, mode, origin, pathna
       _metadata: { importedModules, renderers },
     };
 
-    function createFetchContent(currentFilePath: string) {
-      const fetchContentCache = new Map<string, any>();
-      return async function fetchContent<T>(pattern: string): Promise<FetchContentResult<T>[]> {
-        const cwd = path.dirname(currentFilePath);
-        const cacheKey = `${cwd}:${pattern}`;
-        if (fetchContentCache.has(cacheKey)) {
-          return fetchContentCache.get(cacheKey);
-        }
-        const files = await glob(pattern, { cwd, absolute: true });
-        const contents: FetchContentResult<T>[] = await Promise.all(
-          files.map(async (file) => {
-            const loadedModule = await viteServer.ssrLoadModule(file);
-            const astro = (loadedModule.metadata || {}) as FetchContentResultBase['astro'];
-            const frontmatter = loadedModule.frontmatter || {};
-            //eslint-disable-next-line no-shadow
-            const result: FetchContentResult<T> = {
-              ...frontmatter,
-              astro,
-              url: new URL('http://example.com') // TODO fix
-            };
-            return result;
-          })
-        );
-        fetchContentCache.set(cacheKey, contents);
-        return contents;
-      };
-    };
-
     let html = await renderPage(result, Component, {}, null);
 
     // 4. modify response
@@ -251,6 +248,7 @@ export async function ssr({ astroConfig, filePath, logging, mode, origin, pathna
     // 5. finish
     return html;
   } catch (e: any) {
+    viteServer.ssrFixStacktrace(e);
     // Astro error (thrown by esbuild so it needs to be formatted for Vite)
     if (e.errors) {
       const { location, pluginName, text } = (e as BuildResult).errors[0];

packages/astro/src/runtime/vite/config.ts

@@ -7,9 +7,10 @@ import { fileURLToPath } from 'url';
 import { createRequire } from 'module';
 import vite from 'vite';
 import { getPackageJSON, parseNpmName } from '../util.js';
-import astro from './plugin-astro.js';
+import astroVitePlugin from './plugin-astro.js';
-import markdown from './plugin-markdown.js';
+import astroPostprocessVitePlugin from './plugin-astro-postprocess.js';
-import jsx from './plugin-jsx.js';
+import markdownVitePlugin from './plugin-markdown.js';
+import jsxVitePlugin from './plugin-jsx.js';
 import { AstroDevServer } from '../../dev';
 
 const require = createRequire(import.meta.url);
@@ -80,7 +81,13 @@ export async function loadViteConfig(
         /** Always include these dependencies for optimization */
         include: [...optimizedDeps],
       },
-      plugins: [astro({ config: astroConfig, devServer }), markdown({ config: astroConfig, devServer }), jsx({ config: astroConfig, logging }), ...plugins],
+      plugins: [
+        astroVitePlugin({ config: astroConfig, devServer }), 
+        markdownVitePlugin({ config: astroConfig, devServer }), 
+        jsxVitePlugin({ config: astroConfig, logging }), 
+        astroPostprocessVitePlugin({ config: astroConfig, devServer }),
+        ...plugins
+      ],
       publicDir: fileURLToPath(astroConfig.public),
       resolve: {
         dedupe: [...dedupe],

packages/astro/src/runtime/vite/plugin-astro-postprocess.ts

@@ -0,0 +1,70 @@
+import * as babel from '@babel/core';
+import * as babelTraverse from '@babel/traverse';
+import type * as t from '@babel/types';
+import type { Plugin } from 'vite';
+import type { AstroConfig } from '../../@types/astro.js';
+import { AstroDevServer } from '../../dev/index.js';
+
+interface AstroPluginOptions {
+  config: AstroConfig;
+  devServer?: AstroDevServer;
+}
+
+export default function astro({ config, devServer }: AstroPluginOptions): Plugin {
+  return {
+    name: '@astrojs/vite-plugin-astro-postprocess',
+    async transform(code, id) {
+      // Currently only supported in ".astro" & ".md" files
+      if (!id.endsWith('.astro') && !id.endsWith('.md')) {
+        return null;
+      }
+      // Optimization: only run on a probably match
+      // Open this up if need for post-pass extends past fetchContent
+      if (!code.includes('fetchContent')) {
+        return null;
+      }
+      // Handle the second-pass JS AST Traversal
+      const result = await babel.transformAsync(code, {
+        sourceType: 'module',
+        sourceMaps: true,
+        plugins: [
+          () => {
+            return {
+              visitor: {
+                StringLiteral(path: babelTraverse.NodePath<t.StringLiteral>) {
+                  if (
+                    path.parent.type !== 'CallExpression' ||
+                    path.parent.callee.type !== 'MemberExpression' ||
+                    (path.parent.callee.object as any).name !== 'Astro' ||
+                    (path.parent.callee.property as any).name !== 'fetchContent'
+                  ) {
+                    return;
+                  }
+                  const { value } = path.node;
+                  if (/[a-z]\:\/\//.test(value)) {
+                    return;
+                  }
+                    path.replaceWith({
+                      type: 'CallExpression',
+                      callee: {
+                        type: 'MemberExpression',
+                        object: { type: 'MetaProperty', meta: { type: 'Identifier', name: 'import' }, property: { type: 'Identifier', name: 'meta' } },
+                        property: { type: 'Identifier', name: 'globEager' },
+                        computed: false,
+                      },
+                      arguments: [path.node],
+                    } as any);
+                },
+              },
+            };
+          },
+        ],
+      });
+      // Undocumented baby behavior, but possible according to Babel types.
+      if (!result || !result.code) {
+        return null;
+      }
+      return { code: result.code, map: result.map };
+    },
+  };
+}

packages/astro/src/runtime/vite/plugin-astro.ts

@@ -28,9 +28,9 @@ export default function astro({ config, devServer }: AstroPluginOptions): Plugin
         try {
           // 1. Transform from `.astro` to valid `.ts`
           // use `sourcemap: "inline"` so that the sourcemap is included in the "code" result that we pass to esbuild.
-          tsResult = await transform(source, { sourcefile: id, sourcemap: 'inline', internalURL: 'astro/internal' });
+          tsResult = await transform(source, { sourcefile: id, sourcemap: 'both', internalURL: 'astro/internal' });
           // 2. Compile `.ts` to `.js`
-          const { code, map } = await esbuild.transform(tsResult.code, { loader: 'ts', sourcemap: 'inline', sourcefile: id });
+          const { code, map } = await esbuild.transform(tsResult.code, { loader: 'ts', sourcemap: 'external', sourcefile: id });
           
           return {
             code,
@@ -38,10 +38,8 @@ export default function astro({ config, devServer }: AstroPluginOptions): Plugin
           };
         } catch (err: any) {
           // if esbuild threw the error, find original code source to display
-          if (err.errors) {
+          if (err.errors && tsResult?.map) {
-            const sourcemapb64 = (tsResult?.code.match(/^\/\/# sourceMappingURL=data:application\/json;charset=utf-8;base64,(.*)/m) || [])[1];
+            const json = JSON.parse(tsResult.map);
-            if (!sourcemapb64) throw err;
-            const json = JSON.parse(new Buffer(sourcemapb64, 'base64').toString());
             const mappings = decode(json.mappings);
             const focusMapping = mappings[err.errors[0].location.line + 1];
             err.sourceLoc = { file: id, line: (focusMapping[0][2] || 0) + 1, column: (focusMapping[0][3] || 0) + 1 };

packages/astro/src/runtime/vite/plugin-markdown.ts

@@ -24,8 +24,8 @@ export default function markdown({ config }: AstroPluginOptions): Plugin {
         let render = config.markdownOptions.render;
         let renderOpts = {};
         if (Array.isArray(render)) {
-          render = render[0];
           renderOpts = render[1];
+          render = render[0];
         }
         if (typeof render === 'string') {
           ({ default: render } = await import(render));

yarn.lock

@@ -9538,7 +9538,7 @@ source-map@^0.7.3, source-map@~0.7.2:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383"
   integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==
 
-sourcemap-codec@^1.4.4:
+sourcemap-codec@^1.4.4, sourcemap-codec@^1.4.8:
   version "1.4.8"
   resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4"
   integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==
@@ -9623,6 +9623,11 @@ sprintf-js@~1.0.2:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
   integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
 
+srcset-parse@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/srcset-parse/-/srcset-parse-1.1.0.tgz#73f787f38b73ede2c5af775e0a3465579488122b"
+  integrity sha512-JWp4cG2eybkvKA1QUHGoNK6JDEYcOnSuhzNGjZuYUPqXreDl/VkkvP2sZW7Rmh+icuCttrR9ccb2WPIazyM/Cw==
+
 sshpk@^1.7.0:
   version "1.16.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"