From f28cebcf61ae6206383dabc957366b3ab6edb6e1 Mon Sep 17 00:00:00 2001 From: Drew Powers <1369770+drwpow@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:24:34 -0600 Subject: [PATCH] Add collections to build (#94) --- src/build.ts | 125 ++++++++++++++++++++++++++++++++++++++++--------- src/runtime.ts | 31 ++++++++++-- 2 files changed, 130 insertions(+), 26 deletions(-) diff --git a/src/build.ts b/src/build.ts index 51cdc6e56..a66d49ffa 100644 --- a/src/build.ts +++ b/src/build.ts @@ -1,6 +1,6 @@ import type { AstroConfig, RuntimeMode } from './@types/astro'; import type { LogOptions } from './logger'; -import type { LoadResult } from './runtime'; +import type { AstroRuntime, LoadResult } from './runtime'; import { existsSync, promises as fsPromises } from 'fs'; import { relative as pathRelative } from 'path'; @@ -13,6 +13,18 @@ import { collectStatics } from './build/static.js'; const { mkdir, readdir, readFile, stat, writeFile } = fsPromises; +interface PageBuildOptions { + astroRoot: URL; + dist: URL; + filepath: URL; + runtime: AstroRuntime; + statics: Set; +} + +interface PageResult { + statusCode: number; +} + const logging: LogOptions = { level: 'debug', dest: defaultLogDestination, @@ -55,6 +67,78 @@ async function writeResult(result: LoadResult, outPath: URL, encoding: null | 'u } } +/** Collection utility */ +function getPageType(filepath: URL): 'collection' | 'static' { + if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection'; + return 'static'; +} + +/** Build collection */ +async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise { + const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`; + const builtURLs = new Set(); // !important: internal cache that prevents building the same URLs + + /** Recursively build collection URLs */ + async function loadCollection(url: string): Promise { + if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over + const result = await runtime.load(url); + builtURLs.add(url); + if (result.statusCode === 200) { + const outPath = new URL('./' + url + '/index.html', dist); + await writeResult(result, outPath, 'utf-8'); + mergeSet(statics, collectStatics(result.contents.toString('utf-8'))); + } + return result; + } + + const result = (await loadCollection(pagePath)) as LoadResult; + if (result.statusCode === 200 && !result.collectionInfo) { + throw new Error(`[${rel}]: Collection page must export createCollection() function`); + } + + // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be + if (result.collectionInfo) { + await Promise.all( + [...result.collectionInfo.additionalURLs].map(async (url) => { + // for the top set of additional URLs, we render every new URL generated + const addlResult = await loadCollection(url); + if (addlResult && addlResult.collectionInfo) { + // believe it or not, we may still have a few unbuilt pages left. this is our last crawl: + await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2))); + } + }) + ); + } + + return { + statusCode: result.statusCode, + }; +} + +/** Build static page */ +async function buildStaticPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise { + const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`; + + let relPath = './' + rel.replace(/\.(astro|md)$/, '.html'); + if (!relPath.endsWith('index.html')) { + relPath = relPath.replace(/\.html$/, '/index.html'); + } + + const outPath = new URL(relPath, dist); + const result = await runtime.load(pagePath); + + await writeResult(result, outPath, 'utf-8'); + if (result.statusCode === 200) { + mergeSet(statics, collectStatics(result.contents.toString('utf-8'))); + } + + return { + statusCode: result.statusCode, + }; +} + /** The primary build action */ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { const { projectRoot, astroRoot } = astroConfig; @@ -77,30 +161,27 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { const statics = new Set(); const collectImportsOptions = { astroConfig, logging, resolve, mode }; - for (const pathname of await allPages(pageRoot)) { - const filepath = new URL(`file://${pathname}`); - const rel = pathRelative(astroRoot.pathname + '/pages', filepath.pathname); // pages/index.astro - const pagePath = `/${rel.replace(/\.(astro|md)/, '')}`; + const pages = await allPages(pageRoot); - try { - let relPath = './' + rel.replace(/\.(astro|md)$/, '.html'); - if (!relPath.endsWith('index.html')) { - relPath = relPath.replace(/\.html$/, '/index.html'); - } + try { + await Promise.all( + pages.map(async (pathname) => { + const filepath = new URL(`file://${pathname}`); - const outPath = new URL(relPath, dist); - const result = await runtime.load(pagePath); + const pageType = getPageType(filepath); + const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, statics }; + if (pageType === 'collection') { + await buildCollectionPage(pageOptions); + } else { + await buildStaticPage(pageOptions); + } - await writeResult(result, outPath, 'utf-8'); - if (result.statusCode === 200) { - mergeSet(statics, collectStatics(result.contents.toString('utf-8'))); - } - } catch (err) { - error(logging, 'generate', err); - return 1; - } - - mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions)); + mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions)); + }) + ); + } catch (err) { + error(logging, 'generate', err); + return 1; } for (const pathname of await allPages(componentRoot)) { diff --git a/src/runtime.ts b/src/runtime.ts index 24e186e1c..9d441aa34 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -25,16 +25,19 @@ interface RuntimeConfig { frontendSnowpackConfig: SnowpackConfig; } +// info needed for collection generation +type CollectionInfo = { additionalURLs: Set }; + type LoadResultSuccess = { statusCode: 200; contents: string | Buffer; contentType?: string | false; }; -type LoadResultNotFound = { statusCode: 404; error: Error }; -type LoadResultRedirect = { statusCode: 301 | 302; location: string }; +type LoadResultNotFound = { statusCode: 404; error: Error; collectionInfo?: CollectionInfo }; +type LoadResultRedirect = { statusCode: 301 | 302; location: string; collectionInfo?: CollectionInfo }; type LoadResultError = { statusCode: 500 } & ({ type: 'parse-error'; error: CompileError } | { type: 'unknown'; error: Error }); -export type LoadResult = LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError; +export type LoadResult = (LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError) & { collectionInfo?: CollectionInfo }; // Disable snowpack from writing to stdout/err. snowpackLogger.level = 'silent'; @@ -82,6 +85,8 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro // handle collection let collection = {} as CollectionResult; + let additionalURLs = new Set(); + if (mod.exports.createCollection) { const createCollection: CreateCollection = await mod.exports.createCollection(); for (const key of Object.keys(createCollection)) { @@ -100,6 +105,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro } let requestedParams = routes.find((p) => { const baseURL = (permalink as any)({ params: p }); + additionalURLs.add(baseURL); return baseURL === reqPath || `${baseURL}/${searchResult.currentPage || 1}` === reqPath; }); if (requestedParams) { @@ -135,6 +141,19 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro .replace(/\/1$/, ''); // if end is `/1`, then just omit } + // from page 2 to the end, add all pages as additional URLs (needed for build) + for (let n = 1; n <= collection.page.last; n++) { + if (additionalURLs.size) { + // if this is a param-based collection, paginate all params + additionalURLs.forEach((url) => { + additionalURLs.add(url.replace(/(\/\d+)?$/, `/${n}`)); + }); + } else { + // if this has no params, simply add page + additionalURLs.add(reqPath.replace(/(\/\d+)?$/, `/${n}`)); + } + } + data = data.slice(start, end); } else if (createCollection.pageSize) { // TODO: fix bug where redirect doesn’t happen @@ -142,15 +161,18 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro return { statusCode: 301, location: reqPath + '/1', + collectionInfo: additionalURLs.size ? { additionalURLs } : undefined, }; } // if we’ve paginated too far, this is a 404 - if (!data.length) + if (!data.length) { return { statusCode: 404, error: new Error('Not Found'), + collectionInfo: additionalURLs.size ? { additionalURLs } : undefined, }; + } collection.data = data; } @@ -178,6 +200,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro statusCode: 200, contentType: 'text/html; charset=utf-8', contents: html, + collectionInfo: additionalURLs.size ? { additionalURLs } : undefined, }; } catch (err) { if (err.code === 'parse-error') {