Add collections to build (#94)

src/build.ts

@@ -1,6 +1,6 @@
 import type { AstroConfig, RuntimeMode } from './@types/astro';
 import type { LogOptions } from './logger';
-import type { LoadResult } from './runtime';
+import type { AstroRuntime, LoadResult } from './runtime';
 
 import { existsSync, promises as fsPromises } from 'fs';
 import { relative as pathRelative } from 'path';
@@ -13,6 +13,18 @@ import { collectStatics } from './build/static.js';
 
 const { mkdir, readdir, readFile, stat, writeFile } = fsPromises;
 
+interface PageBuildOptions {
+  astroRoot: URL;
+  dist: URL;
+  filepath: URL;
+  runtime: AstroRuntime;
+  statics: Set<string>;
+}
+
+interface PageResult {
+  statusCode: number;
+}
+
 const logging: LogOptions = {
   level: 'debug',
   dest: defaultLogDestination,
@@ -55,6 +67,78 @@ async function writeResult(result: LoadResult, outPath: URL, encoding: null | 'u
   }
 }
 
+/** Collection utility */
+function getPageType(filepath: URL): 'collection' | 'static' {
+  if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection';
+  return 'static';
+}
+
+/** Build collection */
+async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise<PageResult> {
+  const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro
+  const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`;
+  const builtURLs = new Set<string>(); // !important: internal cache that prevents building the same URLs
+
+  /** Recursively build collection URLs */
+  async function loadCollection(url: string): Promise<LoadResult | undefined> {
+    if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over
+    const result = await runtime.load(url);
+    builtURLs.add(url);
+    if (result.statusCode === 200) {
+      const outPath = new URL('./' + url + '/index.html', dist);
+      await writeResult(result, outPath, 'utf-8');
+      mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
+    }
+    return result;
+  }
+
+  const result = (await loadCollection(pagePath)) as LoadResult;
+  if (result.statusCode === 200 && !result.collectionInfo) {
+    throw new Error(`[${rel}]: Collection page must export createCollection() function`);
+  }
+
+  // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be
+  if (result.collectionInfo) {
+    await Promise.all(
+      [...result.collectionInfo.additionalURLs].map(async (url) => {
+        // for the top set of additional URLs, we render every new URL generated
+        const addlResult = await loadCollection(url);
+        if (addlResult && addlResult.collectionInfo) {
+          // believe it or not, we may still have a few unbuilt pages left. this is our last crawl:
+          await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2)));
+        }
+      })
+    );
+  }
+
+  return {
+    statusCode: result.statusCode,
+  };
+}
+
+/** Build static page */
+async function buildStaticPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise<PageResult> {
+  const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro
+  const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`;
+
+  let relPath = './' + rel.replace(/\.(astro|md)$/, '.html');
+  if (!relPath.endsWith('index.html')) {
+    relPath = relPath.replace(/\.html$/, '/index.html');
+  }
+
+  const outPath = new URL(relPath, dist);
+  const result = await runtime.load(pagePath);
+
+  await writeResult(result, outPath, 'utf-8');
+  if (result.statusCode === 200) {
+    mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
+  }
+
+  return {
+    statusCode: result.statusCode,
+  };
+}
+
 /** The primary build action */
 export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
   const { projectRoot, astroRoot } = astroConfig;
@@ -77,30 +161,27 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
   const statics = new Set<string>();
   const collectImportsOptions = { astroConfig, logging, resolve, mode };
 
-  for (const pathname of await allPages(pageRoot)) {
+  const pages = await allPages(pageRoot);
-    const filepath = new URL(`file://${pathname}`);
-    const rel = pathRelative(astroRoot.pathname + '/pages', filepath.pathname); // pages/index.astro
-    const pagePath = `/${rel.replace(/\.(astro|md)/, '')}`;
 
-    try {
+  try {
-      let relPath = './' + rel.replace(/\.(astro|md)$/, '.html');
+    await Promise.all(
-      if (!relPath.endsWith('index.html')) {
+      pages.map(async (pathname) => {
-        relPath = relPath.replace(/\.html$/, '/index.html');
+        const filepath = new URL(`file://${pathname}`);
-      }
 
-      const outPath = new URL(relPath, dist);
+        const pageType = getPageType(filepath);
-      const result = await runtime.load(pagePath);
+        const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, statics };
+        if (pageType === 'collection') {
+          await buildCollectionPage(pageOptions);
+        } else {
+          await buildStaticPage(pageOptions);
+        }
 
-      await writeResult(result, outPath, 'utf-8');
+        mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions));
-      if (result.statusCode === 200) {
+      })
-        mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
+    );
-      }
+  } catch (err) {
-    } catch (err) {
+    error(logging, 'generate', err);
-      error(logging, 'generate', err);
+    return 1;
-      return 1;
-    }
-
-    mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions));
   }
 
   for (const pathname of await allPages(componentRoot)) {

src/runtime.ts

@@ -25,16 +25,19 @@ interface RuntimeConfig {
   frontendSnowpackConfig: SnowpackConfig;
 }
 
+// info needed for collection generation
+type CollectionInfo = { additionalURLs: Set<string> };
+
 type LoadResultSuccess = {
   statusCode: 200;
   contents: string | Buffer;
   contentType?: string | false;
 };
-type LoadResultNotFound = { statusCode: 404; error: Error };
+type LoadResultNotFound = { statusCode: 404; error: Error; collectionInfo?: CollectionInfo };
-type LoadResultRedirect = { statusCode: 301 | 302; location: string };
+type LoadResultRedirect = { statusCode: 301 | 302; location: string; collectionInfo?: CollectionInfo };
 type LoadResultError = { statusCode: 500 } & ({ type: 'parse-error'; error: CompileError } | { type: 'unknown'; error: Error });
 
-export type LoadResult = LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError;
+export type LoadResult = (LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError) & { collectionInfo?: CollectionInfo };
 
 // Disable snowpack from writing to stdout/err.
 snowpackLogger.level = 'silent';
@@ -82,6 +85,8 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
 
     // handle collection
     let collection = {} as CollectionResult;
+    let additionalURLs = new Set<string>();
+
     if (mod.exports.createCollection) {
       const createCollection: CreateCollection = await mod.exports.createCollection();
       for (const key of Object.keys(createCollection)) {
@@ -100,6 +105,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
         }
         let requestedParams = routes.find((p) => {
           const baseURL = (permalink as any)({ params: p });
+          additionalURLs.add(baseURL);
           return baseURL === reqPath || `${baseURL}/${searchResult.currentPage || 1}` === reqPath;
         });
         if (requestedParams) {
@@ -135,6 +141,19 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
             .replace(/\/1$/, ''); // if end is `/1`, then just omit
         }
 
+        // from page 2 to the end, add all pages as additional URLs (needed for build)
+        for (let n = 1; n <= collection.page.last; n++) {
+          if (additionalURLs.size) {
+            // if this is a param-based collection, paginate all params
+            additionalURLs.forEach((url) => {
+              additionalURLs.add(url.replace(/(\/\d+)?$/, `/${n}`));
+            });
+          } else {
+            // if this has no params, simply add page
+            additionalURLs.add(reqPath.replace(/(\/\d+)?$/, `/${n}`));
+          }
+        }
+
         data = data.slice(start, end);
       } else if (createCollection.pageSize) {
         // TODO: fix bug where redirect doesn’t happen
@@ -142,15 +161,18 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
         return {
           statusCode: 301,
           location: reqPath + '/1',
+          collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
         };
       }
 
       // if we’ve paginated too far, this is a 404
-      if (!data.length)
+      if (!data.length) {
         return {
           statusCode: 404,
           error: new Error('Not Found'),
+          collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
         };
+      }
 
       collection.data = data;
     }
@@ -178,6 +200,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
       statusCode: 200,
       contentType: 'text/html; charset=utf-8',
       contents: html,
+      collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
     };
   } catch (err) {
     if (err.code === 'parse-error') {