Change how squoosh is loaded in the image integration (#6548)

* WIP fix image integration in Netlify

* Add a changeset

* Try another way

* Get the chunks folder

* try inlining

* deep sigh

* fix some things

* more

* Remove some stuff

* remove unused imports

* Add changeset

* remove unused --copy-wasm

* Remove unused post:build

.changeset/green-apes-approve.md

@@ -0,0 +1,7 @@
+---
+'@astrojs/image': patch
+---
+
+Use base64 encoded modules for Squoosh integration
+
+This moves `@astrojs/image` to use base64 encoded versions of the Squoosh wasm modules. This is in order to prevent breakage in SSR environments where your files are moved around. This will fix usage of the integration in Netlify.

packages/integrations/image/package.json

@@ -35,9 +35,8 @@
     "client.d.ts"
   ],
   "scripts": {
-    "build": "astro-scripts build \"src/**/*.ts\" && tsc && pnpm run postbuild",
-    "build:ci": "astro-scripts build \"src/**/*.ts\" && pnpm run postbuild",
-    "postbuild": "astro-scripts copy \"src/**/*.wasm\"",
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
     "dev": "astro-scripts dev \"src/**/*.ts\"",
     "test": "mocha --exit --timeout 20000 test"
   },
@@ -57,6 +56,7 @@
     "astro-scripts": "workspace:*",
     "chai": "^4.3.6",
     "cheerio": "^1.0.0-rc.11",
+    "fast-glob": "^3.2.11",
     "mocha": "^9.2.2",
     "rollup-plugin-copy": "^3.4.0",
     "sharp": "^0.31.0",

packages/integrations/image/src/vendor/squoosh/codecs.ts

@@ -1,5 +1,4 @@
-import { promises as fsp } from 'node:fs'
-import { getModuleURL, instantiateEmscriptenWasm, pathify } from './emscripten-utils.js'
+import { instantiateEmscriptenWasm } from './emscripten-utils.js'
 
 interface DecodeModule extends EmscriptenWasm.Module {
   decode: (data: Uint8Array) => ImageData
@@ -37,41 +36,41 @@ export interface RotateOptions {
 import type { MozJPEGModule as MozJPEGEncodeModule } from './mozjpeg/mozjpeg_enc'
 import mozDec from './mozjpeg/mozjpeg_node_dec.js'
 import mozEnc from './mozjpeg/mozjpeg_node_enc.js'
-const mozEncWasm = new URL('./mozjpeg/mozjpeg_node_enc.wasm', getModuleURL(import.meta.url))
-const mozDecWasm = new URL('./mozjpeg/mozjpeg_node_dec.wasm', getModuleURL(import.meta.url))
+import mozEncWasm from './mozjpeg/mozjpeg_node_enc.wasm.js';
+import mozDecWasm from './mozjpeg/mozjpeg_node_dec.wasm.js';
 
 // WebP
 import type { WebPModule as WebPEncodeModule } from './webp/webp_enc'
 import webpDec from './webp/webp_node_dec.js'
 import webpEnc from './webp/webp_node_enc.js'
-const webpEncWasm = new URL('./webp/webp_node_enc.wasm', getModuleURL(import.meta.url))
-const webpDecWasm = new URL('./webp/webp_node_dec.wasm', getModuleURL(import.meta.url))
+import webpEncWasm from './webp/webp_node_enc.wasm.js';
+import webpDecWasm from './webp/webp_node_dec.wasm.js';
 
 // AVIF
 import type { AVIFModule as AVIFEncodeModule } from './avif/avif_enc'
 import avifDec from './avif/avif_node_dec.js'
 import avifEnc from './avif/avif_node_enc.js'
-const avifEncWasm = new URL('./avif/avif_node_enc.wasm', getModuleURL(import.meta.url))
-const avifDecWasm = new URL('./avif/avif_node_dec.wasm', getModuleURL(import.meta.url))
+import avifEncWasm from './avif/avif_node_enc.wasm.js';
+import avifDecWasm from './avif/avif_node_dec.wasm.js';
 
 // PNG
-import * as pngEncDec from './png/squoosh_png.js'
-const pngEncDecWasm = new URL('./png/squoosh_png_bg.wasm', getModuleURL(import.meta.url))
+import * as pngEncDec from './png/squoosh_png.js';
+import pngEncDecWasm from './png/squoosh_png_bg.wasm.js';
 const pngEncDecInit = () =>
-  pngEncDec.default(fsp.readFile(pathify(pngEncDecWasm.toString())))
+  pngEncDec.default(pngEncDecWasm)
 
 // OxiPNG
-import * as oxipng from './png/squoosh_oxipng.js'
-const oxipngWasm = new URL('./png/squoosh_oxipng_bg.wasm', getModuleURL(import.meta.url))
-const oxipngInit = () => oxipng.default(fsp.readFile(pathify(oxipngWasm.toString())))
+import * as oxipng from './png/squoosh_oxipng.js';
+import oxipngWasm from './png/squoosh_oxipng_bg.wasm.js';
+const oxipngInit = () => oxipng.default(oxipngWasm)
 
 // Resize
 import * as resize from './resize/squoosh_resize.js'
-const resizeWasm = new URL('./resize/squoosh_resize_bg.wasm', getModuleURL(import.meta.url))
-const resizeInit = () => resize.default(fsp.readFile(pathify(resizeWasm.toString())))
+import resizeWasm from './resize/squoosh_resize_bg.wasm.js';
+const resizeInit = () => resize.default(resizeWasm)
 
 // rotate
-const rotateWasm = new URL('./rotate/rotate.wasm', getModuleURL(import.meta.url))
+import rotateWasm from './rotate/rotate.wasm.js';
 
 // Our decoders currently rely on a `ImageData` global.
 import ImageData from './image_data.js'
@@ -178,7 +177,7 @@ export const preprocessors = {
         const sameDimensions = degrees === 0 || degrees === 180
         const size = width * height * 4
         const instance = (
-          await WebAssembly.instantiate(await fsp.readFile(pathify(rotateWasm.toString())))
+          await WebAssembly.instantiate(rotateWasm)
         ).instance as RotateModuleInstance
         const { memory } = instance.exports
         const additionalPagesNeeded = Math.ceil(
@@ -209,11 +208,11 @@ export const codecs = {
     extension: 'jpg',
     detectors: [/^\xFF\xD8\xFF/],
     dec: () =>
-      instantiateEmscriptenWasm(mozDec as DecodeModuleFactory, mozDecWasm.toString()),
+      instantiateEmscriptenWasm(mozDec as DecodeModuleFactory, mozDecWasm),
     enc: () =>
       instantiateEmscriptenWasm(
         mozEnc as EmscriptenWasm.ModuleFactory<MozJPEGEncodeModule>,
-        mozEncWasm.toString()
+        mozEncWasm
       ),
     defaultEncoderOptions: {
       quality: 75,
@@ -244,11 +243,11 @@ export const codecs = {
     extension: 'webp',
     detectors: [/^RIFF....WEBPVP8[LX ]/s],
     dec: () =>
-      instantiateEmscriptenWasm(webpDec as DecodeModuleFactory, webpDecWasm.toString()),
+      instantiateEmscriptenWasm(webpDec as DecodeModuleFactory, webpDecWasm),
     enc: () =>
       instantiateEmscriptenWasm(
         webpEnc as EmscriptenWasm.ModuleFactory<WebPEncodeModule>,
-        webpEncWasm.toString()
+        webpEncWasm
       ),
     defaultEncoderOptions: {
       quality: 75,
@@ -291,11 +290,11 @@ export const codecs = {
     // eslint-disable-next-line no-control-regex
     detectors: [/^\x00\x00\x00 ftypavif\x00\x00\x00\x00/],
     dec: () =>
-      instantiateEmscriptenWasm(avifDec as DecodeModuleFactory, avifDecWasm.toString()),
+      instantiateEmscriptenWasm(avifDec as DecodeModuleFactory, avifDecWasm),
     enc: async () => {
       return instantiateEmscriptenWasm(
         avifEnc as EmscriptenWasm.ModuleFactory<AVIFEncodeModule>,
-        avifEncWasm.toString()
+        avifEncWasm
       )
     },
     defaultEncoderOptions: {

packages/integrations/image/src/vendor/squoosh/emscripten-utils.ts

@@ -10,19 +10,14 @@ export function pathify(path: string): string {
 
 export function instantiateEmscriptenWasm<T extends EmscriptenWasm.Module>(
   factory: EmscriptenWasm.ModuleFactory<T>,
-  path: string,
-  workerJS = ''
+  bytes: Uint8Array,
 ): Promise<T> {
   return factory({
-    locateFile(requestPath) {
-      // The glue code generated by emscripten uses the original
-      // file names of the worker file and the wasm binary.
-      // These will have changed in the bundling process and
-      // we need to inject them here.
-      if (requestPath.endsWith('.wasm')) return pathify(path)
-      if (requestPath.endsWith('.worker.js')) return pathify(workerJS)
-      return requestPath
-    },
+		// @ts-expect-error
+		wasmBinary: bytes,
+		locateFile(file: string) {
+			return file
+		}
   })
 }
 
@@ -32,12 +27,12 @@ export function dirname(url: string) {
 
 /**
  * On certain serverless hosts, our ESM bundle is transpiled to CJS before being run, which means
- * import.meta.url is undefined, so we'll fall back to __dirname in those cases
+ * import.meta.url is undefined, so we'll fall back to __filename in those cases
  * We should be able to remove this once https://github.com/netlify/zip-it-and-ship-it/issues/750 is fixed
  */
 export function getModuleURL(url: string | undefined): string {
 	if (!url) {
-		return pathToFileURL(__dirname).toString();
+		return pathToFileURL(__filename).toString();
 	}
 
 	return url

packages/integrations/image/src/vendor/squoosh/rotate/rotate.wasm.ts

@@ -0,0 +1 @@
+export default Buffer.from("AGFzbQEAAAABDAJgAn9/AGADf39/AAMGBQAAAAABBQMBABAGEQJ/AEGAgMAAC38AQYCAwAALBy4EBm1lbW9yeQIABnJvdGF0ZQAECl9fZGF0YV9lbmQDAAtfX2hlYXBfYmFzZQMBCpsJBUkBAX8gACABbCIAQf////8DcSICBEBBCCEBIABBAnRBCGohAANAIAAgASgCADYCACABQQRqIQEgAEEEaiEAIAJBf2oiAg0ACwsLzQMBFH8gAUECdCERIAAgAWwiDEECdEEEaiESA0ACQAJAAkACQCAEQQFxRQRAIAMgAU8NAiADQQFqIQgMAQsgA0EPaiICIANJIggNASACIAFJIgVFDQEgASADQRBqIAgbIAEgBRshCCACIQMLIAEgA0EQaiICIAIgAUsbIQ0gA0F/cyETIBIgA0ECdGshFEEAIQVBACEOA0ACQAJAIA5FBEAgBSAASQ0BQQEhBAwGCyAAIAVBEGogBUEPaiICIAVJIgcbIAAgAiAASRshBUEBIQQgByACIABPcg0FDAELIAUiAkEBaiEFC0EBIQ4gAyANTw0AIAAgAkEQaiIPIAAgD0kbQQJ0IAJBAnRrIRUgEyABIAJsaiEHIBQgASACQQFqbEECdGohCSADIQoDQCAAIApsIgYgAmoiBEEQaiAAIAZqIA8gAEkbIgYgBEkgDCAGSXINAyAEIAZHBEAgBEECdEEIaiELIBUhBiAHIRAgCSEEA0AgDCABIBBqIhBNDQUgBCALKAIANgIAIAQgEWohBCALQQRqIQsgBkF8aiIGDQALCyAHQX9qIQcgCUF8aiEJIA0gCkEBaiIKRw0ACwwACwALDwsACyAIIQMMAAsAC1MBAX8CQCAAIAFsQQJ0IgJBCGoiAEEIRg0AIAAgAmpBfGohAEEAIQEDQCABIAJGDQEgACABQQhqKAIANgIAIABBfGohACACIAFBBGoiAUcNAAsLC9oDARN/IABBf2ohEEEAIAFBAnRrIREgACABbCIMQQJ0QQhqIRIDQAJAAkACQAJAIARBAXFFBEAgAyABTw0CIANBAWohCQwBCyADQQ9qIgIgA0kiCQ0BIAIgAUkiBUUNASABIANBEGogCRsgASAFGyEJIAIhAwsgASADQRBqIgIgAiABSxshDSASIANBAnRqIRNBACEFQQAhBgNAAkACQCAGQQFxRQRAIAUgAEkNAUEBIQQMBgsgACAFQRBqIAVBD2oiAiAFSSIIGyAAIAIgAEkbIQVBASEEIAggAiAAT3INBQwBCyAFIgJBAWohBQtBASEGIAMgDU8NACAAIAJBEGoiDiAAIA5JG0ECdCACQQJ0ayEUIAMgASAAIAJrbGohCCATIAEgECACa2xBAnRqIQogAyELA0AgACALbCIHIAJqIgRBEGogACAHaiAOIABJGyIHIARJIAwgB0lyDQMgBCAHRwRAIARBAnRBCGohBiAUIQcgCCEPIAohBANAIAwgDyABayIPTQ0FIAQgBigCADYCACAEIBFqIQQgBkEEaiEGIAdBfGoiBw0ACwtBASEGIAhBAWohCCAKQQRqIQogDSALQQFqIgtHDQALDAALAAsPCwALIAkhAwwACwALUAACQAJAAkACQCACQbMBTARAIAJFDQIgAkHaAEcNASAAIAEQAQ8LIAJBtAFGDQIgAkGOAkYNAwsACyAAIAEQAA8LIAAgARACDwsgACABEAMLAE0JcHJvZHVjZXJzAghsYW5ndWFnZQEEUnVzdAAMcHJvY2Vzc2VkLWJ5AQVydXN0Yx0xLjQ3LjAgKDE4YmY2YjRmMCAyMDIwLTEwLTA3KQ==", 'base64');

packages/integrations/image/wasmize.mjs

@@ -0,0 +1,13 @@
+import fastglob from 'fast-glob';
+import { fileURLToPath } from 'node:url';
+import * as fs from 'node:fs';
+
+const result = await fastglob(fileURLToPath(new URL('./src/**/*.wasm', import.meta.url)));
+
+for(const filepath of result) {
+	const buffer = await fs.promises.readFile(filepath);
+	const base64 = buffer.toString('base64');
+	const source = `export default Buffer.from(${JSON.stringify(base64)}, 'base64');`;
+	const outpath = filepath + '.ts';
+	await fs.promises.writeFile(outpath, source, 'utf-8');
+}

pnpm-lock.yaml

@@ -2907,6 +2907,7 @@ importers:
       astro-scripts: workspace:*
       chai: ^4.3.6
       cheerio: ^1.0.0-rc.11
+      fast-glob: ^3.2.11
       http-cache-semantics: ^4.1.0
       image-size: ^1.0.2
       kleur: ^4.1.5
@@ -2931,6 +2932,7 @@ importers:
       astro-scripts: link:../../../scripts
       chai: 4.3.7
       cheerio: 1.0.0-rc.12
+      fast-glob: 3.2.12
       mocha: 9.2.2
       rollup-plugin-copy: 3.4.0
       sharp: 0.31.3