From ca5517ba9156e94f8d2e1a0c7bc58f6f29f7c354 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Fri, 22 Sep 2023 18:52:52 -0500 Subject: [PATCH] wip: add incremental plugin, collect timings from Rollup --- .../vite-plugin-content-virtual-mod.ts | 4 +- packages/astro/src/core/build/static-build.ts | 65 ++++++---- packages/astro/src/core/create-vite.ts | 2 + .../src/vite-plugin-incremental/index.ts | 112 ++++++++++++++++++ 4 files changed, 158 insertions(+), 25 deletions(-) create mode 100644 packages/astro/src/vite-plugin-incremental/index.ts diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 0ea58a545..03d13ce0d 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -39,12 +39,12 @@ export function astroContentVirtualModPlugin({ let contentEntryGlobPath = globWithUnderscoresIgnored(relContentDir, contentEntryExts); // HACK(nate): filter contentEntryGlobPath to simulate incremental build - contentEntryGlobPath = [contentEntryGlobPath[0].replace('**/*', '**/a-e*')] + // contentEntryGlobPath = [contentEntryGlobPath[0].replace('**/*', '**/a-e*')] const dataEntryGlobPath = globWithUnderscoresIgnored(relContentDir, dataEntryExts); /** Note: data collections excluded */ let renderEntryGlobPath = globWithUnderscoresIgnored(relContentDir, contentEntryExts); - renderEntryGlobPath = [renderEntryGlobPath[0].replace('**/*', '**/a-e*')] + // renderEntryGlobPath = [renderEntryGlobPath[0].replace('**/*', '**/a-e*')] const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') diff --git a/packages/astro/src/core/build/static-build.ts b/packages/astro/src/core/build/static-build.ts index 7b0d6e378..69b6ba879 100644 --- a/packages/astro/src/core/build/static-build.ts +++ b/packages/astro/src/core/build/static-build.ts @@ -33,6 +33,9 @@ import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js'; import type { PageBuildData, StaticBuildOptions } from './types.js'; import { getTimeStat } from './util.js'; +function isWatcher(value: unknown): value is vite.Rollup.RollupWatcher { + return true; +} export async function viteBuild(opts: StaticBuildOptions) { const { allPages, settings } = opts; @@ -78,13 +81,13 @@ export async function viteBuild(opts: StaticBuildOptions) { registerAllPlugins(container); // HACK(nate): Begin cache restoration - const cacheDir = new URL('./node_modules/.cache/astro/', settings.config.root); - const cacheFile = new URL('./build.json', cacheDir); - let restore = false; - if (fs.existsSync(cacheDir)) { - internals.cache = JSON.parse(fs.readFileSync(cacheFile, { encoding: 'utf8' })); - restore = true; - } + // const cacheDir = new URL('./node_modules/.cache/astro/', settings.config.root); + // const cacheFile = new URL('./build.json', cacheDir); + // let restore = false; + // if (fs.existsSync(cacheDir)) { + // internals.cache = JSON.parse(fs.readFileSync(cacheFile, { encoding: 'utf8' })); + // restore = true; + // } // Build your project (SSR application code, assets, client JS, etc.) const ssrTime = performance.now(); @@ -92,24 +95,38 @@ export async function viteBuild(opts: StaticBuildOptions) { const ssrOutput = await ssrBuild(opts, internals, pageInput, container); opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`)); - // HACK(nate): write to cache - internals.cache = []; - for (const output of ssrOutput.output) { - const md = output.moduleIds.filter(id => id.endsWith('.md')) - if (!!md) { - // const { fileName: id, code: content } = output.moduleIds - // internals.cache.push({ input, output: { id, content } }) - } + if (isWatcher(ssrOutput)) { + await new Promise(resolve => ssrOutput.on("event", (e) => { + if (e.code === "BUNDLE_END") { + const timings = e.result.getTimings?.(); + for (const [key, [elapsed]] of Object.entries(timings!)) { + if (elapsed > 50) { + console.log(key, `${(elapsed).toFixed(0)}ms`) + } + } + resolve() + } + })); + return; } - fs.mkdirSync(cacheDir, { recursive: true }); - fs.writeFileSync(cacheFile, JSON.stringify(internals.cache), { encoding: 'utf8' }); - if (restore) { - // console.log('RESTORING CACHE'); - for (const cached of internals.cache) { - fs.writeFileSync(new URL(`./${cached.output.id}`, settings.config.outDir), cached.output.content, { encoding: 'utf8' }); - } - } + // HACK(nate): write to cache + // internals.cache = []; + // for (const output of ssrOutput.output) { + // const md = output.moduleIds.filter(id => id.endsWith('.md')) + // if (!!md) { + // // const { fileName: id, code: content } = output.moduleIds + // // internals.cache.push({ input, output: { id, content } }) + // } + // } + // fs.mkdirSync(cacheDir, { recursive: true }); + // fs.writeFileSync(cacheFile, JSON.stringify(internals.cache), { encoding: 'utf8' }); + // if (restore) { + // // console.log('RESTORING CACHE'); + // for (const cached of internals.cache) { + // fs.writeFileSync(new URL(`./${cached.output.id}`, settings.config.outDir), cached.output.content, { encoding: 'utf8' }); + // } + // } settings.timer.end('SSR build'); settings.timer.start('Client build'); @@ -194,8 +211,10 @@ async function ssrBuild( manifest: false, outDir: fileURLToPath(out), copyPublicDir: !ssr, + watch: {}, rollupOptions: { ...viteConfig.build?.rollupOptions, + perf: true, input: [], output: { format: 'esm', diff --git a/packages/astro/src/core/create-vite.ts b/packages/astro/src/core/create-vite.ts index 3c59b1fb4..e75fd8328 100644 --- a/packages/astro/src/core/create-vite.ts +++ b/packages/astro/src/core/create-vite.ts @@ -11,6 +11,7 @@ import { astroContentImportPlugin, astroContentVirtualModPlugin, } from '../content/index.js'; +import { incremental } from '../vite-plugin-incremental/index.js'; import astroTransitions from '../transitions/vite-plugin-transitions.js'; import astroPostprocessVitePlugin from '../vite-plugin-astro-postprocess/index.js'; import { vitePluginAstroServer } from '../vite-plugin-astro-server/index.js'; @@ -111,6 +112,7 @@ export async function createVite( exclude: ['astro', 'node-fetch'], }, plugins: [ + incremental({ settings }), configAliasVitePlugin({ settings }), astroLoadFallbackPlugin({ fs, root: settings.config.root }), astroVitePlugin({ settings, logger }), diff --git a/packages/astro/src/vite-plugin-incremental/index.ts b/packages/astro/src/vite-plugin-incremental/index.ts new file mode 100644 index 000000000..66c93fedb --- /dev/null +++ b/packages/astro/src/vite-plugin-incremental/index.ts @@ -0,0 +1,112 @@ +import type * as vite from 'vite'; +import type { AstroSettings } from '../@types/astro.js'; +import { existsSync, readFileSync, mkdirSync, writeFileSync, } from 'node:fs'; + +interface IncrementalOptions { + settings: AstroSettings; +} + +export interface IncrementalAPI { + cache: Cache; +} + +type RollupTransformCache = Map; + +export function incremental(opts: IncrementalOptions): vite.Plugin { + const { settings } = opts; + // let target: 'server' | 'client' = 'server'; + const diskCache = new DiskCache(settings) + let cache: RollupTransformCache = new Map(); + + const plugin: vite.Plugin = { + name: 'astro:incremental', + apply: 'build', + config(viteConfig) { + diskCache.setTarget(!!viteConfig?.build?.ssr ? 'server' : 'client'); + cache = diskCache.read(); + if (cache.size > 0) { + console.log(`Cache restored!`, cache.size); + } + viteConfig.build!.rollupOptions!.perf = true; + }, + buildStart(options) { + // Important! Skips any `transform` calls for modules in the cache + options.plugins = options.plugins.map(v => memoizePlugin(v, cache)); + }, + moduleParsed(info) { + const meta = info.meta ?? {}; + meta.cached = true; + cache.set(info.id, { + ast: info.ast!, + code: info.code!, + assertions: info.assertions, + meta, + moduleSideEffects: info.moduleSideEffects, + syntheticNamedExports: info.syntheticNamedExports, + }) + }, + load(id) { + if (cache.has(id)) { + const cached = cache.get(id)!; + // const { code } = this.load({ id }); + // No match, eject from cache and skip + // if (cached.code !== code) { + // cache.delete(id); + // return; + // } + return cached; + } + }, + buildEnd() { + diskCache.write(cache); + } + } + return plugin; +} + +class DiskCache { + #url: URL; + #raw: RollupTransformCache | undefined; + constructor(settings: AstroSettings) { + this.#url = new URL(`./node_modules/.astro/build.cache.json`, settings.config.root); + } + setTarget(target: 'server' | 'client') { + this.#url = new URL(`./build-${target}.cache.json`, this.#url); + } + read(): RollupTransformCache { + if (this.#raw) return this.#raw; + try { + if (existsSync(this.#url)) { + return new Map(JSON.parse(readFileSync(this.#url, { encoding: 'utf-8' }))); + } + } catch {} + return new Map() + } + write(value: RollupTransformCache) { + this.#raw = value; + mkdirSync(new URL('./', this.#url), { recursive: true }); + writeFileSync(this.#url, JSON.stringify(Array.from(value.entries())), { encoding: 'utf-8' }); + } +} + +// function checksum(code: string) { +// return crypto.createHash('md5').update(code, 'utf-8').digest('hex'); +// } + +function memoizeHook any>(original: T, override: (params: Parameters, original: () => ReturnType) => ReturnType) { + return new Proxy(original, { + apply(target, thisArg, argArray) { + return override(argArray as Parameters, () => Reflect.apply(target, thisArg, argArray)) + } + }) +} + +function memoizePlugin(plugin: vite.Plugin, cache: RollupTransformCache) { + if (typeof plugin.transform === 'function') { + plugin.transform = memoizeHook(plugin.transform, function load([_code, id], next) { + if (cache.has(id)) return; + return next(); + }) + } + return plugin; +}