wip: add incremental plugin, collect timings from Rollup

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -37,6 +37,15 @@ export function astroContentVirtualModPlugin({
 	const contentEntryExts = [...contentEntryConfigByExt.keys()];
 	const dataEntryExts = getDataEntryExts(settings);
 
+	let contentEntryGlobPath = globWithUnderscoresIgnored(relContentDir, contentEntryExts);
+	// HACK(nate): filter contentEntryGlobPath to simulate incremental build
+	// contentEntryGlobPath = [contentEntryGlobPath[0].replace('**/*', '**/a-e*')]
+
+	const dataEntryGlobPath = globWithUnderscoresIgnored(relContentDir, dataEntryExts);
+	/** Note: data collections excluded */ 
+	let renderEntryGlobPath = globWithUnderscoresIgnored(relContentDir, contentEntryExts);
+	// renderEntryGlobPath = [renderEntryGlobPath[0].replace('**/*', '**/a-e*')]
+
 	const virtualModContents = fsMod
 		.readFileSync(contentPaths.virtualModTemplate, 'utf-8')
 		.replace(
@@ -46,20 +55,15 @@ export function astroContentVirtualModPlugin({
 		.replace('@@CONTENT_DIR@@', relContentDir)
 		.replace(
 			"'@@CONTENT_ENTRY_GLOB_PATH@@'",
-			JSON.stringify(globWithUnderscoresIgnored(relContentDir, contentEntryExts))
+			JSON.stringify(contentEntryGlobPath)
 		)
 		.replace(
 			"'@@DATA_ENTRY_GLOB_PATH@@'",
-			JSON.stringify(globWithUnderscoresIgnored(relContentDir, dataEntryExts))
+			JSON.stringify(dataEntryGlobPath)
 		)
 		.replace(
 			"'@@RENDER_ENTRY_GLOB_PATH@@'",
-			JSON.stringify(
+			JSON.stringify(renderEntryGlobPath)
-				globWithUnderscoresIgnored(
-					relContentDir,
-					/** Note: data collections excluded */ contentEntryExts
-				)
-			)
 		);
 
 	const astroContentVirtualModuleId = '\0' + VIRTUAL_MODULE_ID;

packages/astro/src/core/build/internal.ts

@@ -12,6 +12,8 @@ import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
 import type { PageBuildData, StylesheetAsset, ViteID } from './types.js';
 
 export interface BuildInternals {
+	cache: any;
+
 	/**
 	 * Each CSS module is named with a chunk id derived from the Astro pages they
 	 * are used in by default. It's easy to crawl this relation in the SSR build as
@@ -106,6 +108,8 @@ export function createBuildInternals(): BuildInternals {
 	const hoistedScriptIdToPagesMap = new Map<string, Set<string>>();
 
 	return {
+		cache: [],
+
 		cssModuleToChunkIdMap: new Map(),
 		hoistedScriptIdToHoistedMap,
 		hoistedScriptIdToPagesMap,

packages/astro/src/core/build/static-build.ts

@@ -33,6 +33,9 @@ import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
 import type { PageBuildData, StaticBuildOptions } from './types.js';
 import { getTimeStat } from './util.js';
 
+function isWatcher(value: unknown): value is vite.Rollup.RollupWatcher {
+	return true;
+}
 export async function viteBuild(opts: StaticBuildOptions) {
 	const { allPages, settings } = opts;
 
@@ -77,12 +80,54 @@ export async function viteBuild(opts: StaticBuildOptions) {
 	const container = createPluginContainer(opts, internals);
 	registerAllPlugins(container);
 
+	// HACK(nate): Begin cache restoration
+	// const cacheDir = new URL('./node_modules/.cache/astro/', settings.config.root);
+	// const cacheFile = new URL('./build.json', cacheDir);
+	// let restore = false;
+	// if (fs.existsSync(cacheDir)) {
+	// 	internals.cache = JSON.parse(fs.readFileSync(cacheFile, { encoding: 'utf8' }));
+	// 	restore = true;
+	// }
+
 	// Build your project (SSR application code, assets, client JS, etc.)
 	const ssrTime = performance.now();
 	opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
 	const ssrOutput = await ssrBuild(opts, internals, pageInput, container);
 	opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`));
 
+	if (isWatcher(ssrOutput)) {
+		await new Promise<void>(resolve => ssrOutput.on("event", (e) => {
+			if (e.code === "BUNDLE_END") {
+				const timings = e.result.getTimings?.();
+				for (const [key, [elapsed]] of Object.entries(timings!)) {
+					if (elapsed > 50) {
+						console.log(key, `${(elapsed).toFixed(0)}ms`)
+					}
+				}
+				resolve()
+			}
+		}));
+		return;
+	}
+
+	// HACK(nate): write to cache
+	// internals.cache = [];
+	// for (const output of ssrOutput.output) {
+	// 	const md = output.moduleIds.filter(id => id.endsWith('.md'))
+	// 	if (!!md) {
+	// 		// const { fileName: id, code: content } = output.moduleIds
+	// 		// internals.cache.push({ input, output: { id, content } })
+	// 	}
+	// }
+	// fs.mkdirSync(cacheDir, { recursive: true });
+	// fs.writeFileSync(cacheFile, JSON.stringify(internals.cache), { encoding: 'utf8' });
+	// if (restore) {
+	// 	// console.log('RESTORING CACHE');
+	// 	for (const cached of internals.cache) {
+	// 		fs.writeFileSync(new URL(`./${cached.output.id}`, settings.config.outDir), cached.output.content, { encoding: 'utf8' });
+	// 	}
+	// }
+
 	settings.timer.end('SSR build');
 	settings.timer.start('Client build');
 
@@ -123,7 +168,7 @@ export async function staticBuild(opts: StaticBuildOptions, internals: BuildInte
 		case settings.config.output === 'static': {
 			settings.timer.start('Static generate');
 			await generatePages(opts, internals);
-			await cleanServerOutput(opts);
+			// await cleanServerOutput(opts);
 			settings.timer.end('Static generate');
 			return;
 		}
@@ -166,8 +211,10 @@ async function ssrBuild(
 			manifest: false,
 			outDir: fileURLToPath(out),
 			copyPublicDir: !ssr,
+			watch: {},
 			rollupOptions: {
 				...viteConfig.build?.rollupOptions,
+				perf: true,
 				input: [],
 				output: {
 					format: 'esm',

packages/astro/src/core/create-vite.ts

@@ -11,6 +11,7 @@ import {
 	astroContentImportPlugin,
 	astroContentVirtualModPlugin,
 } from '../content/index.js';
+import { incremental } from '../vite-plugin-incremental/index.js';
 import astroTransitions from '../transitions/vite-plugin-transitions.js';
 import astroPostprocessVitePlugin from '../vite-plugin-astro-postprocess/index.js';
 import { vitePluginAstroServer } from '../vite-plugin-astro-server/index.js';
@@ -111,6 +112,7 @@ export async function createVite(
 			exclude: ['astro', 'node-fetch'],
 		},
 		plugins: [
+			incremental({ settings }),
 			configAliasVitePlugin({ settings }),
 			astroLoadFallbackPlugin({ fs, root: settings.config.root }),
 			astroVitePlugin({ settings, logger }),

packages/astro/src/vite-plugin-incremental/index.ts

@@ -0,0 +1,112 @@
+import type * as vite from 'vite';
+import type { AstroSettings } from '../@types/astro.js';
+import { existsSync, readFileSync, mkdirSync, writeFileSync,  } from 'node:fs';
+
+interface IncrementalOptions {
+	settings: AstroSettings;
+}
+
+export interface IncrementalAPI {
+	cache: Cache;
+}
+
+type RollupTransformCache = Map<string, vite.Rollup.SourceDescription>;
+
+export function incremental(opts: IncrementalOptions): vite.Plugin {
+	const { settings } = opts;
+	// let target: 'server' | 'client' = 'server';
+	const diskCache = new DiskCache(settings)
+	let cache: RollupTransformCache = new Map();
+
+	const plugin: vite.Plugin = {
+		name: 'astro:incremental',
+        apply: 'build',
+		config(viteConfig) {
+			diskCache.setTarget(!!viteConfig?.build?.ssr ? 'server' : 'client');
+			cache = diskCache.read();
+			if (cache.size > 0) {
+				console.log(`Cache restored!`, cache.size);
+			}
+			viteConfig.build!.rollupOptions!.perf = true;
+		},
+		buildStart(options) {
+			// Important! Skips any `transform` calls for modules in the cache
+			options.plugins = options.plugins.map(v => memoizePlugin(v, cache));
+		},
+		moduleParsed(info) {
+			const meta = info.meta ?? {};
+			meta.cached = true;
+			cache.set(info.id, {
+				ast: info.ast!,
+				code: info.code!,
+				assertions: info.assertions,
+				meta,
+				moduleSideEffects: info.moduleSideEffects,
+				syntheticNamedExports: info.syntheticNamedExports,
+			})
+		},
+		load(id) {
+			if (cache.has(id)) {
+				const cached = cache.get(id)!;
+				// const { code } = this.load({ id });
+				// No match, eject from cache and skip
+				// if (cached.code !== code) {
+				// 	cache.delete(id);
+				// 	return;
+				// }
+				return cached;
+			}
+		},
+		buildEnd() {
+			diskCache.write(cache);
+		}
+	}
+	return plugin;
+}
+
+class DiskCache {
+	#url: URL;
+	#raw: RollupTransformCache | undefined;
+	constructor(settings: AstroSettings) {
+		this.#url = new URL(`./node_modules/.astro/build.cache.json`, settings.config.root);
+	}
+	setTarget(target: 'server' | 'client') {
+		this.#url = new URL(`./build-${target}.cache.json`, this.#url);
+	}
+	read(): RollupTransformCache {
+		if (this.#raw) return this.#raw;
+		try {
+			if (existsSync(this.#url)) {
+				return new Map(JSON.parse(readFileSync(this.#url, { encoding: 'utf-8' })));
+			}
+		} catch {}
+		return new Map()
+	}
+	write(value: RollupTransformCache) {
+		this.#raw = value;
+		mkdirSync(new URL('./', this.#url), { recursive: true });
+		writeFileSync(this.#url, JSON.stringify(Array.from(value.entries())), { encoding: 'utf-8' });
+	}
+}
+
+// function checksum(code: string) {
+// 	return crypto.createHash('md5').update(code, 'utf-8').digest('hex');
+// }
+
+function memoizeHook<T extends (...args: any[]) => any>(original: T, override: (params: Parameters<T>, original: () => ReturnType<T>) => ReturnType<T>) {
+	return new Proxy(original, {
+		apply(target, thisArg, argArray) {
+			return override(argArray as Parameters<T>, () => Reflect.apply(target, thisArg, argArray))
+		}
+	})
+}
+
+function memoizePlugin(plugin: vite.Plugin, cache: RollupTransformCache) {
+	if (typeof plugin.transform === 'function') {
+		plugin.transform = memoizeHook(plugin.transform, function load([_code, id], next) {
+			if (cache.has(id)) return;
+			return next();
+		})
+	}
+	return plugin;
+}