[@astrojs/image] adding caching support for SSG builds (#4909)

* adds a caching feature for SSG builds

* chore: add changeset

* nit: eslint fix

* chore: add readme docs for caching

* adding basic test coverage for cached images

.changeset/seven-shrimps-hope.md

@@ -0,0 +1,24 @@
+---
+'@astrojs/image': patch
+---
+
+Adds caching support for transformed images :tada:
+
+Local images will be cached for 1 year and invalidated when the original image file is changed.
+
+Remote images will be cached based on the `fetch()` response's cache headers, similar to how a CDN would manage the cache.
+
+**cacheDir**
+
+By default, transformed images will be cached to `./node_modules/.astro/image`. This can be configured in the integration's config options.
+
+```
+export default defineConfig({
+	integrations: [image({
+    // may be useful if your hosting provider allows caching between CI builds
+    cacheDir: "./.cache/image"
+  })]
+});
+```
+
+Caching can also be disabled by using `cacheDir: false`.

packages/integrations/image/README.md

@@ -453,6 +453,25 @@ export default {
 }
 ```
 
+### config.cacheDir
+
+During static builds, the integration will cache transformed images to avoid rebuilding the same image for every build. This can be particularly helpful if you are using a hosting service that allows you to cache build assets for future deployments.
+
+Local images will be cached for 1 year and invalidated when the original image file is changed. Remote images will be cached based on the `fetch()` response's cache headers, similar to how a CDN would manage the cache.
+
+By default, transformed images will be cached to `./node_modules/.astro/image`. This can be configured in the integration's config options.
+
+```
+export default defineConfig({
+	integrations: [image({
+    // may be useful if your hosting provider allows caching between CI builds
+    cacheDir: "./.cache/image"
+  })]
+});
+```
+
+Caching can also be disabled by using `cacheDir: false`.
+
 ## Examples
 
 ### Local images

packages/integrations/image/package.json

@@ -49,12 +49,14 @@
     "slash": "^4.0.0"
   },
   "devDependencies": {
+    "@types/http-cache-semantics": "^4.0.1",
     "@types/mime": "^2.0.3",
     "@types/sharp": "^0.30.5",
     "astro": "workspace:*",
     "astro-scripts": "workspace:*",
     "chai": "^4.3.6",
     "cheerio": "^1.0.0-rc.11",
+    "http-cache-semantics": "^4.1.0",
     "kleur": "^4.1.4",
     "mocha": "^9.2.2",
     "rollup-plugin-copy": "^3.4.0",

packages/integrations/image/src/build/cache.ts

@@ -0,0 +1,85 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { debug, error, warn } from '../utils/logger.js';
+import type { LoggerLevel } from '../utils/logger.js';
+
+const CACHE_FILE = `cache.json`;
+
+interface Cache {
+	[filename: string]: { expires: number }
+}
+
+export class ImageCache {
+	#cacheDir: URL;
+	#cacheFile: URL;
+	#cache: Cache = { }
+	#logLevel: LoggerLevel;
+
+	constructor(dir: URL, logLevel: LoggerLevel) {
+		this.#logLevel = logLevel;
+		this.#cacheDir = dir;
+		this.#cacheFile = this.#toAbsolutePath(CACHE_FILE);
+	}
+
+	#toAbsolutePath(file: string) {
+		return new URL(path.join(this.#cacheDir.toString(), file));
+	}
+
+	async init() {
+		try {
+			const str = await fs.readFile(this.#cacheFile, 'utf-8');
+			this.#cache = JSON.parse(str) as Cache;
+		} catch {
+			// noop
+			debug({ message: 'no cache file found', level: this.#logLevel });
+		}
+	}
+
+	async finalize() {
+		try {
+			await fs.mkdir(path.dirname(fileURLToPath(this.#cacheFile)), { recursive: true });
+			await fs.writeFile(this.#cacheFile, JSON.stringify(this.#cache));
+		} catch {
+			// noop
+			warn({ message: 'could not save the cache file', level: this.#logLevel });
+		}
+	}
+
+	async get(file: string): Promise<Buffer | undefined> {
+		if (!this.has(file)) {
+			return undefined;
+		}
+
+		try {
+			const filepath = this.#toAbsolutePath(file);
+			return await fs.readFile(filepath);
+		} catch {
+			warn({ message: `could not load cached file for "${file}"`, level: this.#logLevel });
+			return undefined;
+		}
+	}
+
+	async set(file: string, buffer: Buffer, opts: Cache['string']): Promise<void> {
+		try {
+			const filepath = this.#toAbsolutePath(file);
+			await fs.mkdir(path.dirname(fileURLToPath(filepath)), { recursive: true });
+			await fs.writeFile(filepath, buffer);
+
+			this.#cache[file] = opts;
+		} catch {
+			// noop
+			warn({ message: `could not save cached copy of "${file}"`, level: this.#logLevel });
+		}
+	}
+
+	has(file: string): boolean {
+		if (!(file in this.#cache)) {
+			return false;
+		}
+
+		const { expires } = this.#cache[file];
+
+		return expires > Date.now();
+	}
+}

packages/integrations/image/src/build/ssg.ts

@@ -1,6 +1,7 @@
 import { doWork } from '@altano/tiny-async-pool';
 import type { AstroConfig } from 'astro';
 import { bgGreen, black, cyan, dim, green } from 'kleur/colors';
+import CachePolicy from 'http-cache-semantics';
 import fs from 'node:fs/promises';
 import OS from 'node:os';
 import path from 'node:path';
@@ -8,24 +9,66 @@ import { fileURLToPath } from 'node:url';
 import type { SSRImageService, TransformOptions } from '../loaders/index.js';
 import { debug, info, LoggerLevel, warn } from '../utils/logger.js';
 import { isRemoteImage } from '../utils/paths.js';
+import { ImageCache } from './cache.js';
 
 async function loadLocalImage(src: string | URL) {
 	try {
-		return await fs.readFile(src);
+		const data = await fs.readFile(src);
+
+		// Vite's file hash will change if the file is changed at all,
+		// we can safely cache local images here.
+		const timeToLive = new Date();
+		timeToLive.setFullYear(timeToLive.getFullYear() + 1);
+		
+		return {
+			data,
+			expires: timeToLive.getTime(),
+		}
 	} catch {
 		return undefined;
 	}
 }
 
+function webToCachePolicyRequest({ url, method, headers: _headers }: Request): CachePolicy.Request {
+	const headers: CachePolicy.Headers = {};
+	for (const [key, value] of _headers) {
+			headers[key] = value;
+	}
+	return {
+			method,
+			url,
+			headers,
+	};
+}
+
+function webToCachePolicyResponse({ status, headers: _headers }: Response): CachePolicy.Response {
+	const headers: CachePolicy.Headers = {};
+	for (const [key, value] of _headers) {
+			headers[key] = value;
+	}
+	return {
+			status,
+			headers,
+	};
+}
+
 async function loadRemoteImage(src: string) {
 	try {
-		const res = await fetch(src);
+		const req = new Request(src);
+		const res = await fetch(req);
 
 		if (!res.ok) {
 			return undefined;
 		}
 
-		return Buffer.from(await res.arrayBuffer());
+		// calculate an expiration date based on the response's TTL
+		const policy = new CachePolicy(webToCachePolicyRequest(req), webToCachePolicyResponse(res));
+		const expires = policy.storable() ? policy.timeToLive() : 0;
+
+		return {
+			data: Buffer.from(await res.arrayBuffer()),
+			expires: Date.now() + expires,
+		};
 	} catch {
 		return undefined;
 	}
@@ -42,9 +85,17 @@ export interface SSGBuildParams {
 	config: AstroConfig;
 	outDir: URL;
 	logLevel: LoggerLevel;
+	cacheDir?: URL;
 }
 
-export async function ssgBuild({ loader, staticImages, config, outDir, logLevel }: SSGBuildParams) {
+export async function ssgBuild({ loader, staticImages, config, outDir, logLevel, cacheDir }: SSGBuildParams) {
+	let cache: ImageCache | undefined = undefined;
+
+	if (cacheDir) {
+		cache = new ImageCache(cacheDir, logLevel);
+		await cache.init();
+	}
+
 	const timer = performance.now();
 	const cpuCount = OS.cpus().length;
 
@@ -67,6 +118,9 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel
 		let inputFile: string | undefined = undefined;
 		let inputBuffer: Buffer | undefined = undefined;
 
+		// tracks the cache duration for the original source image
+		let expires = 0;
+
 		// Vite will prefix a hashed image with the base path, we need to strip this
 		// off to find the actual file relative to /dist
 		if (config.base && src.startsWith(config.base)) {
@@ -75,11 +129,17 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel
 
 		if (isRemoteImage(src)) {
 			// try to load the remote image
-			inputBuffer = await loadRemoteImage(src);
+			const res = await loadRemoteImage(src);
+
+			inputBuffer = res?.data;
+			expires = res?.expires || 0;
 		} else {
 			const inputFileURL = new URL(`.${src}`, outDir);
 			inputFile = fileURLToPath(inputFileURL);
-			inputBuffer = await loadLocalImage(inputFile);
+
+			const res = await loadLocalImage(inputFile);
+			inputBuffer = res?.data;
+			expires = res?.expires || 0;
 		}
 
 		if (!inputBuffer) {
@@ -106,14 +166,32 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel
 				outputFile = fileURLToPath(outputFileURL);
 			}
 
-			const { data } = await loader.transform(inputBuffer, transform);
+			const pathRelative = outputFile.replace(fileURLToPath(outDir), '');
+
+			let data: Buffer | undefined;
+
+			// try to load the transformed image from cache, if available
+			if (cache?.has(pathRelative)) {
+				data = await cache.get(pathRelative);
+			}
+			
+			// a valid cache file wasn't found, transform the image and cache it
+			if (!data) {
+				const transformed = await loader.transform(inputBuffer, transform);
+				data = transformed.data;
+
+				// cache the image, if available
+				if (cache) {
+					await cache.set(pathRelative, data, { expires });
+				}
+			}
 
 			await fs.writeFile(outputFile, data);
 
 			const timeEnd = performance.now();
 			const timeChange = getTimeStat(timeStart, timeEnd);
 			const timeIncrease = `(+${timeChange})`;
-			const pathRelative = outputFile.replace(fileURLToPath(outDir), '');
+
 			debug({
 				level: logLevel,
 				prefix: false,
@@ -125,6 +203,11 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel
 	// transform each original image file in batches
 	await doWork(cpuCount, staticImages, processStaticImage);
 
+	// saves the cache's JSON manifest to file
+	if (cache) {
+		await cache.finalize();
+	}
+
 	info({
 		level: logLevel,
 		prefix: false,

packages/integrations/image/src/index.ts

@@ -27,14 +27,16 @@ export interface IntegrationOptions {
 	/**
 	 * Entry point for the @type {HostedImageService} or @type {LocalImageService} to be used.
 	 */
-	serviceEntryPoint?: string;
+	serviceEntryPoint?: '@astrojs/image/squoosh' | '@astrojs/image/sharp' | string;
 	logLevel?: LoggerLevel;
+	cacheDir?: false | string;
 }
 
 export default function integration(options: IntegrationOptions = {}): AstroIntegration {
 	const resolvedOptions = {
 		serviceEntryPoint: '@astrojs/image/squoosh',
 		logLevel: 'info' as LoggerLevel,
+		cacheDir: './node_modules/.astro/image',
 		...options,
 	};
 
@@ -127,12 +129,15 @@ export default function integration(options: IntegrationOptions = {}): AstroInte
 				}
 
 				if (loader && 'transform' in loader && staticImages.size > 0) {
+					const cacheDir = !!resolvedOptions.cacheDir ? new URL(resolvedOptions.cacheDir, _config.root) : undefined;
+
 					await ssgBuild({
 						loader,
 						staticImages,
 						config: _config,
 						outDir: dir,
 						logLevel: resolvedOptions.logLevel,
+						cacheDir,
 					});
 				}
 			},

packages/integrations/image/test/image-ssg.test.js

@@ -1,6 +1,7 @@
 import { expect } from 'chai';
 import * as cheerio from 'cheerio';
 import sizeOf from 'image-size';
+import fs from 'fs/promises';
 import { fileURLToPath } from 'url';
 import { loadFixture } from './test-utils.js';
 
@@ -253,7 +254,7 @@ describe('SSG images - build', function () {
 			size: { width: 544, height: 184, type: 'jpg' },
 		},
 	].forEach(({ title, id, regex, size }) => {
-		it(title, () => {
+		it(title, async () => {
 			const image = $(id);
 
 			expect(image.attr('src')).to.match(regex);
@@ -261,6 +262,9 @@ describe('SSG images - build', function () {
 			expect(image.attr('height')).to.equal(size.height.toString());
 
 			verifyImage(image.attr('src'), size);
+
+			const url = new URL('./fixtures/basic-image/node_modules/.astro/image' + image.attr('src'), import.meta.url);
+			expect(await fs.stat(url), 'transformed image was cached').to.not.be.undefined;
 		});
 	});
 });

pnpm-lock.yaml

@@ -2487,12 +2487,14 @@ importers:
   packages/integrations/image:
     specifiers:
       '@altano/tiny-async-pool': ^1.0.2
+      '@types/http-cache-semantics': ^4.0.1
       '@types/mime': ^2.0.3
       '@types/sharp': ^0.30.5
       astro: workspace:*
       astro-scripts: workspace:*
       chai: ^4.3.6
       cheerio: ^1.0.0-rc.11
+      http-cache-semantics: ^4.1.0
       image-size: ^1.0.2
       kleur: ^4.1.4
       magic-string: ^0.25.9
@@ -2510,12 +2512,14 @@ importers:
       mime: 3.0.0
       slash: 4.0.0
     devDependencies:
+      '@types/http-cache-semantics': 4.0.1
       '@types/mime': 2.0.3
       '@types/sharp': 0.30.5
       astro: link:../../astro
       astro-scripts: link:../../../scripts
       chai: 4.3.6
       cheerio: 1.0.0-rc.12
+      http-cache-semantics: 4.1.0
       kleur: 4.1.5
       mocha: 9.2.2
       rollup-plugin-copy: 3.4.0
@@ -9439,6 +9443,10 @@ packages:
     resolution: {integrity: sha512-OcJcvP3Yk8mjYwf/IdXZtTE1tb/u0WF0qa29ER07ZHCYUBZXSN29Z1mBS+/96+kNMGTFUAbSz9X+pHmHpZrTCw==}
     dev: false
 
+  /@types/http-cache-semantics/4.0.1:
+    resolution: {integrity: sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==}
+    dev: true
+
   /@types/is-ci/3.0.0:
     resolution: {integrity: sha512-Q0Op0hdWbYd1iahB+IFNQcWXFq4O0Q5MwQP7uN0souuQ4rPg1vEYcnIOfr1gY+M+6rc8FGoRaBO1mOOvL29sEQ==}
     dependencies: