diff --git a/.changeset/metal-pens-decide.md b/.changeset/metal-pens-decide.md new file mode 100644 index 000000000..3cef0315a --- /dev/null +++ b/.changeset/metal-pens-decide.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Improvements performance for building sites with thousands of pages with the static build diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 3c1548698..1ec00c15f 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -177,7 +177,11 @@ export type GetHydrateCallback = () => Promise<(element: Element, innerHTML: str rss?: (...args: any[]) => any; } -export type GetStaticPathsResult = { params: Params; props?: Props }[]; +export type GetStaticPathsItem = { params: Params; props?: Props }; +export type GetStaticPathsResult = GetStaticPathsItem[]; +export type GetStaticPathsResultKeyed = GetStaticPathsResult & { + keyed: Map +}; export interface HydrateOptions { value?: string; @@ -313,7 +317,7 @@ export interface RouteData { type: 'page'; } -export type RouteCache = Record; +export type RouteCache = Record; export type RuntimeMode = 'development' | 'production'; diff --git a/packages/astro/src/core/build/page-data.ts b/packages/astro/src/core/build/page-data.ts index 8ba029ba3..f2d3472e5 100644 --- a/packages/astro/src/core/build/page-data.ts +++ b/packages/astro/src/core/build/page-data.ts @@ -10,6 +10,7 @@ import { preload as ssrPreload } from '../ssr/index.js'; import { validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js'; import { generatePaginateFunction } from '../ssr/paginate.js'; import { generateRssFunction } from '../ssr/rss.js'; +import { assignStaticPaths } from '../ssr/route-cache.js'; export interface CollectPagesDataOptions { astroConfig: AstroConfig; @@ -112,8 +113,8 @@ async function getStaticPathsForRoute(opts: CollectPagesDataOptions, route: Rout const mod = (await viteServer.ssrLoadModule(fileURLToPath(filePath))) as ComponentInstance; validateGetStaticPathsModule(mod); const rss = generateRssFunction(astroConfig.buildOptions.site, route); - const staticPaths: GetStaticPathsResult = (await mod.getStaticPaths!({ paginate: generatePaginateFunction(route), rss: rss.generator })).flat(); - routeCache[route.component] = staticPaths; + await assignStaticPaths(routeCache, route, mod, rss.generator); + const staticPaths = routeCache[route.component]; validateGetStaticPathsResult(staticPaths, logging); return { paths: staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean), diff --git a/packages/astro/src/core/build/static-build.ts b/packages/astro/src/core/build/static-build.ts index ee68e486c..403cbb7b7 100644 --- a/packages/astro/src/core/build/static-build.ts +++ b/packages/astro/src/core/build/static-build.ts @@ -31,6 +31,8 @@ export interface StaticBuildOptions { viteConfig: ViteConfigWithSSR; } +const MAX_CONCURRENT_RENDERS = 10; + function addPageName(pathname: string, opts: StaticBuildOptions): void { const pathrepl = opts.astroConfig.buildOptions.pageUrlFormat === 'directory' ? '/index.html' : pathname === '/' ? 'index.html' : '.html'; opts.pageNames.push(pathname.replace(/\/?$/, pathrepl).replace(/^\//, '')); @@ -45,6 +47,29 @@ function chunkIsPage(output: OutputAsset | OutputChunk, internals: BuildInternal return chunk.facadeModuleId && (internals.entrySpecifierToBundleMap.has(chunk.facadeModuleId) || internals.entrySpecifierToBundleMap.has('/' + chunk.facadeModuleId)); } +// Throttle the rendering a paths to prevents creating too many Promises on the microtask queue. +function *throttle(max: number, inPaths: string[]) { + let tmp = []; + let i = 0; + for(let path of inPaths) { + tmp.push(path); + if(i === max) { + yield tmp; + // Empties the array, to avoid allocating a new one. + tmp.length = 0; + i = 0; + } else { + i++; + } + } + + // If tmp has items in it, that means there were less than {max} paths remaining + // at the end, so we need to yield these too. + if(tmp.length) { + yield tmp; + } +} + export async function staticBuild(opts: StaticBuildOptions) { const { allPages, astroConfig } = opts; @@ -246,10 +271,17 @@ async function generatePage(output: OutputChunk, opts: StaticBuildOptions, inter renderers, }; - const renderPromises = pageData.paths.map((path) => { - return generatePath(path, opts, generationOptions); - }); - return await Promise.all(renderPromises); + const renderPromises = []; + // Throttle the paths to avoid overloading the CPU with too many tasks. + for(const paths of throttle(MAX_CONCURRENT_RENDERS, pageData.paths)) { + for(const path of paths) { + renderPromises.push(generatePath(path, opts, generationOptions)); + } + // This blocks generating more paths until these 10 complete. + await Promise.all(renderPromises); + // This empties the array without allocating a new one. + renderPromises.length = 0; + } } interface GeneratePathOptions { @@ -276,6 +308,9 @@ async function generatePath(pathname: string, opts: StaticBuildOptions, gopts: G logging, pathname, mod, + // Do not validate as validation already occurred for static routes + // and validation is relatively expensive. + validate: false }); debug(logging, 'generate', `Generating: ${pathname}`); diff --git a/packages/astro/src/core/ssr/index.ts b/packages/astro/src/core/ssr/index.ts index fd1b1694c..817dba933 100644 --- a/packages/astro/src/core/ssr/index.ts +++ b/packages/astro/src/core/ssr/index.ts @@ -1,6 +1,6 @@ import type { BuildResult } from 'esbuild'; import type vite from '../vite'; -import type { AstroConfig, ComponentInstance, GetStaticPathsResult, Params, Props, Renderer, RouteCache, RouteData, RuntimeMode, SSRElement, SSRError } from '../../@types/astro'; +import type { AstroConfig, ComponentInstance, GetStaticPathsResult, GetStaticPathsResultKeyed, Params, Props, Renderer, RouteCache, RouteData, RuntimeMode, SSRElement, SSRError } from '../../@types/astro'; import type { LogOptions } from '../logger'; import eol from 'eol'; @@ -14,6 +14,7 @@ import { injectTags } from './html.js'; import { generatePaginateFunction } from './paginate.js'; import { getParams, validateGetStaticPathsModule, validateGetStaticPathsResult } from './routing.js'; import { createResult } from './result.js'; +import { assignStaticPaths, ensureRouteCached, findPathItemByKey } from './route-cache.js'; const svelteStylesRE = /svelte\?svelte&type=style/; @@ -131,16 +132,18 @@ export async function getParamsAndProps({ logging, pathname, mod, + validate = true }: { route: RouteData | undefined; routeCache: RouteCache; pathname: string; mod: ComponentInstance; logging: LogOptions; + validate?: boolean; }): Promise<[Params, Props]> { // Handle dynamic routes let params: Params = {}; - let pageProps: Props = {}; + let pageProps: Props; if (route && !route.pathname) { if (route.params.length) { const paramsMatch = route.pattern.exec(pathname); @@ -148,24 +151,27 @@ export async function getParamsAndProps({ params = getParams(route.params)(paramsMatch); } } - validateGetStaticPathsModule(mod); - if (!routeCache[route.component]) { - routeCache[route.component] = await ( - await mod.getStaticPaths!({ - paginate: generatePaginateFunction(route), - rss: () => { - /* noop */ - }, - }) - ).flat(); + if(validate) { + validateGetStaticPathsModule(mod); } - validateGetStaticPathsResult(routeCache[route.component], logging); - const routePathParams: GetStaticPathsResult = routeCache[route.component]; - const matchedStaticPath = routePathParams.find(({ params: _params }) => JSON.stringify(_params) === JSON.stringify(params)); + if (!routeCache[route.component]) { + await assignStaticPaths(routeCache, route, mod); + } + if(validate) { + // This validation is expensive so we only want to do it in dev. + validateGetStaticPathsResult(routeCache[route.component], logging); + } + const staticPaths: GetStaticPathsResultKeyed = routeCache[route.component]; + const paramsKey = JSON.stringify(params); + const matchedStaticPath = findPathItemByKey(staticPaths, paramsKey, logging); if (!matchedStaticPath) { throw new Error(`[getStaticPaths] route pattern matched, but no matching static path found. (${pathname})`); } - pageProps = { ...matchedStaticPath.props } || {}; + // This is written this way for performance; instead of spreading the props + // which is O(n), create a new object that extends props. + pageProps = Object.create(matchedStaticPath.props || Object.prototype); + } else { + pageProps = {}; } return [params, pageProps]; } @@ -185,16 +191,7 @@ export async function render(renderers: Renderer[], mod: ComponentInstance, ssrO } } validateGetStaticPathsModule(mod); - if (!routeCache[route.component]) { - routeCache[route.component] = await ( - await mod.getStaticPaths!({ - paginate: generatePaginateFunction(route), - rss: () => { - /* noop */ - }, - }) - ).flat(); - } + await ensureRouteCached(routeCache, route, mod); validateGetStaticPathsResult(routeCache[route.component], logging); const routePathParams: GetStaticPathsResult = routeCache[route.component]; const matchedStaticPath = routePathParams.find(({ params: _params }) => JSON.stringify(_params) === JSON.stringify(params)); diff --git a/packages/astro/src/core/ssr/route-cache.ts b/packages/astro/src/core/ssr/route-cache.ts new file mode 100644 index 000000000..8ac7ae2b5 --- /dev/null +++ b/packages/astro/src/core/ssr/route-cache.ts @@ -0,0 +1,52 @@ +import type { ComponentInstance, GetStaticPathsItem, GetStaticPathsResult, GetStaticPathsResultKeyed, RouteCache, RouteData } from '../../@types/astro'; +import type { LogOptions } from '../logger'; + +import { debug } from '../logger.js'; +import { generatePaginateFunction } from '../ssr/paginate.js'; + +type RSSFn = (...args: any[]) => any; + +export async function callGetStaticPaths(mod: ComponentInstance, route: RouteData, rssFn?: RSSFn): Promise { + const staticPaths: GetStaticPathsResult = await ( + await mod.getStaticPaths!({ + paginate: generatePaginateFunction(route), + rss: rssFn || (() => { + /* noop */ + }), + }) + ).flat(); + + const keyedStaticPaths = staticPaths as GetStaticPathsResultKeyed; + keyedStaticPaths.keyed = new Map(); + for(const sp of keyedStaticPaths) { + const paramsKey = JSON.stringify(sp.params); + keyedStaticPaths.keyed.set(paramsKey, sp); + } + + return keyedStaticPaths; +} + +export async function assignStaticPaths(routeCache: RouteCache, route: RouteData, mod: ComponentInstance, rssFn?: RSSFn): Promise { + const staticPaths = await callGetStaticPaths(mod, route, rssFn); + routeCache[route.component] = staticPaths; +} + +export async function ensureRouteCached(routeCache: RouteCache, route: RouteData, mod: ComponentInstance, rssFn?: RSSFn): Promise { + if (!routeCache[route.component]) { + const staticPaths = await callGetStaticPaths(mod, route, rssFn); + routeCache[route.component] = staticPaths; + return staticPaths; + } else { + return routeCache[route.component]; + } +} + +export function findPathItemByKey(staticPaths: GetStaticPathsResultKeyed, paramsKey: string, logging: LogOptions) { + let matchedStaticPath = staticPaths.keyed.get(paramsKey); + if(matchedStaticPath) { + return matchedStaticPath; + } + + debug(logging, 'findPathItemByKey', `Unexpected cache miss looking for ${paramsKey}`); + matchedStaticPath = staticPaths.find(({ params: _params }) => JSON.stringify(_params) === paramsKey); +}