Allow passing fetch() Response to set:html (#4832)

* ALlow passing fetch() Response to set:html

* Only check for Symbol.iterator on objects

* oops

* Fix no-streaming case

* Remove old comment

.changeset/silent-comics-hang.md

@@ -0,0 +1,49 @@
+---
+'astro': minor
+---
+
+Allows Responses to be passed to set:html
+
+This expands the abilities of `set:html` to ultimate service this use-case:
+
+```astro
+<div set:html={fetch('/legacy-post.html')}></div>
+```
+
+This means you can take a legacy app that has been statically generated to HTML and directly consume that HTML within your templates. As is always the case with `set:html`, this should only be used on trusted content.
+
+To make this possible, you can also pass several other types into `set:html` now:
+
+* `Response` objects, since that is what fetch() returns:
+    ```astro
+    <div set:html={new Response('<span>Hello world</span>', {
+      headers: {
+        'content-type': 'text/html'
+      }
+    })}></div>
+    ```
+* `ReadableStream`s:
+    ```astro
+    <div set:html={new ReadableStream({
+      start(controller) {
+        controller.enqueue(`<span>read me</span>`);
+        controller.close();
+      }
+    })}></div>
+    ```
+* `AsyncIterable`s:
+    ```astro
+    <div set:html={(async function * () {
+      for await (const num of [1, 2, 3, 4, 5]) {
+        yield `<li>${num}</li>`;
+      }
+    })()}>
+    ```
+* `Iterable`s (non-async):
+    ```astro
+    <div set:html={(function * () {
+      for (const num of [1, 2, 3, 4, 5]) {
+        yield `<li>${num}</li>`;
+      }
+    })()}>
+    ```

packages/astro/src/runtime/server/escape.ts

@@ -3,11 +3,24 @@ import { escape } from 'html-escaper';
 // Leverage the battle-tested `html-escaper` npm package.
 export const escapeHTML = escape;
 
+export class HTMLBytes extends Uint8Array {
+	// @ts-ignore
+	get [Symbol.toStringTag]() {
+		return 'HTMLBytes';
+	}
+}
+
 /**
  * A "blessed" extension of String that tells Astro that the string
  * has already been escaped. This helps prevent double-escaping of HTML.
  */
-export class HTMLString extends String {}
+export class HTMLString extends String {
+	get [Symbol.toStringTag]() {
+		return 'HTMLString';
+	}
+}
+
+type BlessedType = string | HTMLBytes;
 
 /**
  * markHTMLString marks a string as raw or "already escaped" by returning
@@ -30,12 +43,52 @@ export const markHTMLString = (value: any) => {
 	return value;
 };
 
-export function unescapeHTML(str: any) {
-	// If a promise, await the result and mark that.
-	if (!!str && typeof str === 'object' && typeof str.then === 'function') {
-		return Promise.resolve(str).then((value) => {
-			return markHTMLString(value);
-		});
+export function isHTMLString(value: any): value is HTMLString {
+	return Object.prototype.toString.call(value) === '[object HTMLString]';
+}
+
+function markHTMLBytes(bytes: Uint8Array) {
+	return new HTMLBytes(bytes);
+}
+
+export function isHTMLBytes(value: any): value is HTMLBytes {
+	return Object.prototype.toString.call(value) === '[object HTMLBytes]';
+}
+
+async function * unescapeChunksAsync(iterable: AsyncIterable<Uint8Array>): any {
+	for await (const chunk of iterable) {
+		yield unescapeHTML(chunk as BlessedType);
+	}
+}
+
+function * unescapeChunks(iterable: Iterable<any>): any {
+	for(const chunk of iterable) {
+		yield unescapeHTML(chunk);
+	}
+}
+
+export function unescapeHTML(str: any): BlessedType | Promise<BlessedType | AsyncGenerator<BlessedType, void, unknown>> | AsyncGenerator<BlessedType, void, unknown> {
+	if (!!str && typeof str === 'object') {
+		if(str instanceof Uint8Array) {
+			return markHTMLBytes(str);
+		}
+		// If a response, stream out the chunks
+		else if(str instanceof Response && str.body) {
+			const body = str.body as unknown as AsyncIterable<Uint8Array>;
+			return unescapeChunksAsync(body);
+		}
+		// If a promise, await the result and mark that.
+		else if(typeof str.then === 'function') {
+			return Promise.resolve(str).then((value) => {
+				return unescapeHTML(value);
+			});
+		}
+		else if(Symbol.iterator in str) {
+			return unescapeChunks(str);
+		}
+		else if(Symbol.asyncIterator in str) {
+			return unescapeChunksAsync(str);
+		}
 	}
 	return markHTMLString(str);
 }

packages/astro/src/runtime/server/index.ts

@@ -1,6 +1,6 @@
 export { createAstro } from './astro-global.js';
 export { renderEndpoint } from './endpoint.js';
-export { escapeHTML, HTMLString, markHTMLString, unescapeHTML } from './escape.js';
+export { escapeHTML, HTMLString, HTMLBytes, markHTMLString, unescapeHTML } from './escape.js';
 export type { Metadata } from './metadata';
 export { createMetadata } from './metadata.js';
 export {

packages/astro/src/runtime/server/jsx.ts

@@ -4,14 +4,15 @@ import { AstroJSX, isVNode } from '../../jsx-runtime/index.js';
 import {
 	escapeHTML,
 	HTMLString,
+	HTMLBytes,
 	markHTMLString,
 	renderComponent,
 	RenderInstruction,
 	renderToString,
 	spreadAttributes,
-	stringifyChunk,
 	voidElementNames,
 } from './index.js';
+import { HTMLParts } from './render/common.js';
 
 const ClientOnlyPlaceholder = 'astro-client-only';
 
@@ -122,7 +123,7 @@ export async function renderJSX(result: SSRResult, vnode: any): Promise<any> {
 			}
 			await Promise.all(slotPromises);
 
-			let output: string | AsyncIterable<string | RenderInstruction>;
+			let output: string | AsyncIterable<string | HTMLBytes | RenderInstruction>;
 			if (vnode.type === ClientOnlyPlaceholder && vnode.props['client:only']) {
 				output = await renderComponent(
 					result,
@@ -141,12 +142,11 @@ export async function renderJSX(result: SSRResult, vnode: any): Promise<any> {
 				);
 			}
 			if (typeof output !== 'string' && Symbol.asyncIterator in output) {
-				let body = '';
+				let parts = new HTMLParts();
 				for await (const chunk of output) {
-					let html = stringifyChunk(result, chunk);
-					body += html;
+					parts.append(chunk, result);
 				}
-				return markHTMLString(body);
+				return markHTMLString(parts.toString());
 			} else {
 				return markHTMLString(output);
 			}

packages/astro/src/runtime/server/render/any.ts

@@ -27,7 +27,9 @@ export async function* renderChild(child: any): AsyncIterable<any> {
 		Object.prototype.toString.call(child) === '[object AstroComponent]'
 	) {
 		yield* renderAstroComponent(child);
-	} else if (typeof child === 'object' && Symbol.asyncIterator in child) {
+	} else if(ArrayBuffer.isView(child)) {
+		yield child;
+	} else if (typeof child === 'object' && (Symbol.asyncIterator in child || Symbol.iterator in child)) {
 		yield* child;
 	} else {
 		yield child;

packages/astro/src/runtime/server/render/astro.ts

@@ -2,10 +2,10 @@ import type { SSRResult } from '../../../@types/astro';
 import type { AstroComponentFactory } from './index';
 import type { RenderInstruction } from './types';
 
-import { markHTMLString } from '../escape.js';
+import { markHTMLString, HTMLBytes } from '../escape.js';
 import { HydrationDirectiveProps } from '../hydration.js';
 import { renderChild } from './any.js';
-import { stringifyChunk } from './common.js';
+import { HTMLParts } from './common.js';
 
 // In dev mode, check props and make sure they are valid for an Astro component
 function validateComponentProps(props: any, displayName: string) {
@@ -62,7 +62,7 @@ export function isAstroComponentFactory(obj: any): obj is AstroComponentFactory
 
 export async function* renderAstroComponent(
 	component: InstanceType<typeof AstroComponent>
-): AsyncIterable<string | RenderInstruction> {
+): AsyncIterable<string | HTMLBytes | RenderInstruction> {
 	for await (const value of component) {
 		if (value || value === 0) {
 			for await (const chunk of renderChild(value)) {
@@ -95,11 +95,11 @@ export async function renderToString(
 		throw response;
 	}
 
-	let html = '';
+	let parts = new HTMLParts();
 	for await (const chunk of renderAstroComponent(Component)) {
-		html += stringifyChunk(result, chunk);
+		parts.append(chunk, result);
 	}
-	return html;
+	return parts.toString();
 }
 
 export async function renderToIterable(
@@ -108,7 +108,7 @@ export async function renderToIterable(
 	displayName: string,
 	props: any,
 	children: any
-): Promise<AsyncIterable<string | RenderInstruction>> {
+): Promise<AsyncIterable<string | HTMLBytes | RenderInstruction>> {
 	validateComponentProps(props, displayName);
 	const Component = await componentFactory(result, props, children);
 

packages/astro/src/runtime/server/render/common.ts

@@ -1,7 +1,7 @@
 import type { SSRResult } from '../../../@types/astro';
 import type { RenderInstruction } from './types.js';
 
-import { markHTMLString } from '../escape.js';
+import { markHTMLString, HTMLBytes, isHTMLString } from '../escape.js';
 import {
 	determineIfNeedsHydrationScript,
 	determinesIfNeedsDirectiveScript,
@@ -12,6 +12,9 @@ import {
 export const Fragment = Symbol.for('astro:fragment');
 export const Renderer = Symbol.for('astro:renderer');
 
+export const encoder = new TextEncoder();
+export const decoder = new TextDecoder();
+
 // Rendering produces either marked strings of HTML or instructions for hydration.
 // These directive instructions bubble all the way up to renderPage so that we
 // can ensure they are added only once, and as soon as possible.
@@ -40,3 +43,55 @@ export function stringifyChunk(result: SSRResult, chunk: string | RenderInstruct
 		}
 	}
 }
+
+export class HTMLParts {
+	public parts: Array<HTMLBytes | string>;
+	constructor() {
+		this.parts = [];
+	}
+	append(part: string | HTMLBytes | RenderInstruction, result: SSRResult) {
+		if(ArrayBuffer.isView(part)) {
+			this.parts.push(part);
+		} else {
+			this.parts.push(stringifyChunk(result, part));
+		}
+	}
+	toString() {
+		let html = '';
+		for(const part of this.parts) {
+			if(ArrayBuffer.isView(part)) {
+				html += decoder.decode(part);
+			} else {
+				html += part;
+			}
+		}
+		return html;
+	}
+	toArrayBuffer() {
+		this.parts.forEach((part, i) => {
+			if(typeof part === 'string') {
+				this.parts[i] = encoder.encode(String(part));
+			}
+		});
+		return concatUint8Arrays(this.parts as Uint8Array[]);
+	}
+}
+
+export function chunkToByteArray(result: SSRResult, chunk: string | HTMLBytes | RenderInstruction): Uint8Array {
+	if(chunk instanceof Uint8Array) {
+		return chunk as Uint8Array;
+	}
+	return encoder.encode(stringifyChunk(result, chunk));
+}
+
+export function concatUint8Arrays(arrays: Array<Uint8Array>) {
+	let len = 0;
+	arrays.forEach(arr => len += arr.length);
+	let merged = new Uint8Array(len);
+	let offset = 0;
+	arrays.forEach(arr => {
+		merged.set(arr, offset);
+		offset += arr.length;
+	});
+	return merged;
+}

packages/astro/src/runtime/server/render/component.ts

@@ -1,7 +1,7 @@
 import type { AstroComponentMetadata, SSRLoadedRenderer, SSRResult } from '../../../@types/astro';
 import type { RenderInstruction } from './types.js';
 
-import { markHTMLString } from '../escape.js';
+import { markHTMLString, HTMLBytes } from '../escape.js';
 import { extractDirectives, generateHydrateScript } from '../hydration.js';
 import { serializeProps } from '../serialize.js';
 import { shorthash } from '../shorthash.js';
@@ -54,7 +54,7 @@ export async function renderComponent(
 	Component: unknown,
 	_props: Record<string | number, any>,
 	slots: any = {}
-): Promise<string | AsyncIterable<string | RenderInstruction>> {
+): Promise<string | AsyncIterable<string | HTMLBytes | RenderInstruction>> {
 	Component = await Component;
 
 	switch (getComponentType(Component)) {
@@ -84,7 +84,7 @@ export async function renderComponent(
 
 		case 'astro-factory': {
 			async function* renderAstroComponentInline(): AsyncGenerator<
-				string | RenderInstruction,
+				string | HTMLBytes | RenderInstruction,
 				void,
 				undefined
 			> {

packages/astro/src/runtime/server/render/page.ts

@@ -3,11 +3,11 @@ import type { AstroComponentFactory } from './index';
 
 import { createResponse } from '../response.js';
 import { isAstroComponent, isAstroComponentFactory, renderAstroComponent } from './astro.js';
-import { stringifyChunk } from './common.js';
+import { encoder, chunkToByteArray, HTMLParts } from './common.js';
 import { renderComponent } from './component.js';
+import { isHTMLString } from '../escape.js';
 import { maybeRenderHead } from './head.js';
 
-const encoder = new TextEncoder();
 const needsHeadRenderingSymbol = Symbol.for('astro.needsHeadRendering');
 
 type NonAstroPageComponent = {
@@ -72,17 +72,16 @@ export async function renderPage(
 						let i = 0;
 						try {
 							for await (const chunk of iterable) {
-								let html = stringifyChunk(result, chunk);
-
-								if (i === 0) {
-									if (!/<!doctype html/i.test(html)) {
-										controller.enqueue(encoder.encode('<!DOCTYPE html>\n'));
+								if(isHTMLString(chunk)) {
+									if (i === 0) {
+										if (!/<!doctype html/i.test(String(chunk))) {
+											controller.enqueue(encoder.encode('<!DOCTYPE html>\n'));
+										}
 									}
 								}
-								// Convert HTML object to string
-								// for environments that won't "toString" automatically
-								// (ex. Cloudflare and Vercel Edge)
-								controller.enqueue(encoder.encode(String(html)));
+								
+								let bytes = chunkToByteArray(result, chunk);
+								controller.enqueue(bytes);
 								i++;
 							}
 							controller.close();
@@ -94,20 +93,21 @@ export async function renderPage(
 				},
 			});
 		} else {
-			body = '';
+			let parts = new HTMLParts();
 			let i = 0;
 			for await (const chunk of iterable) {
-				let html = stringifyChunk(result, chunk);
-				if (i === 0) {
-					if (!/<!doctype html/i.test(html)) {
-						body += '<!DOCTYPE html>\n';
+				if(isHTMLString(chunk)) {
+					if (i === 0) {
+						if (!/<!doctype html/i.test(String(chunk))) {
+							parts.append('<!DOCTYPE html>\n', result);
+						}
 					}
 				}
-				body += html;
+				parts.append(chunk, result);
 				i++;
 			}
-			const bytes = encoder.encode(body);
-			headers.set('Content-Length', bytes.byteLength.toString());
+			body = parts.toArrayBuffer();
+			headers.set('Content-Length', body.byteLength.toString());
 		}
 
 		let response = createResponse(body, { ...init, headers });

packages/astro/src/runtime/server/response.ts

@@ -59,7 +59,7 @@ type CreateResponseFn = (body?: BodyInit | null, init?: ResponseInit) => Respons
 
 export const createResponse: CreateResponseFn = isNodeJS
 	? (body, init) => {
-			if (typeof body === 'string') {
+			if (typeof body === 'string' || ArrayBuffer.isView(body)) {
 				return new Response(body, init);
 			}
 			if (typeof StreamingCompatibleResponse === 'undefined') {

packages/astro/src/runtime/server/util.ts

@@ -1,9 +1,3 @@
-function formatList(values: string[]): string {
-	if (values.length === 1) {
-		return values[0];
-	}
-	return `${values.slice(0, -1).join(', ')} or ${values[values.length - 1]}`;
-}
 
 export function serializeListValue(value: any) {
 	const hash: Record<string, any> = {};
@@ -34,3 +28,7 @@ export function serializeListValue(value: any) {
 		}
 	}
 }
+
+export function isPromise<T = any>(value: any): value is Promise<T> {
+	return !!value && typeof value === 'object' && typeof value.then === 'function';
+}

packages/astro/test/fixtures/set-html/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@test/set-html",
+  "version": "1.0.0",
+  "dependencies": {
+    "astro": "workspace:*"
+  }
+}

packages/astro/test/fixtures/set-html/public/test.html

@@ -0,0 +1 @@
+<div id="fetched-html">works</div>

packages/astro/test/fixtures/set-html/src/pages/fetch.astro

@@ -0,0 +1,18 @@
+---
+// This is a dev only test
+const mode = import.meta.env.MODE;
+---
+<html>
+	<head>
+		<title>Testing</title>
+	</head>
+	<body>
+		<h1>Testing</h1>
+		<div id="fetch" set:html={
+			mode === 'development' ?
+				// @ts-ignore
+				TEST_FETCH(fetch, '/test.html') :
+				'build mode'
+		}></div>
+	</body>
+</html>

packages/astro/test/fixtures/set-html/src/pages/index.astro

@@ -0,0 +1,36 @@
+---
+function * iterator(id = 'iterator') {
+	for(const num of [1, 2, 3, 4, 5]) {
+		yield `<span id="${id}-num">${num}</span>`;
+	}
+}
+
+async function * asynciterator() {
+	for(const num of iterator('asynciterator')) {
+		yield Promise.resolve(num);
+	}
+}
+---
+<html>
+	<head>
+		<title>Testing</title>
+	</head>
+	<body>
+		<h1>Testing</h1>
+		<div id="html" set:html={`<span id="html-inner">works</span>`}></div>
+		<div id="promise-html" set:html={Promise.resolve(`<span id="promise-html-inner">works</span>`)}></div>
+		<div id="response" set:html={new Response(`<span id="response-html-inner"></span>`, {
+			headers: {
+				'content-type': 'text/html'
+			}
+		})}></div>
+		<div id="iterator" set:html={iterator()}></div>
+		<div id="asynciterator" set:html={asynciterator()}></div>
+		<div id="readablestream" set:html={new ReadableStream({
+			start(controller) {
+				controller.enqueue(`<span id="readable-inner">read me</span>`);
+				controller.close();
+			},
+		})}></div>
+	</body>
+</html>

packages/astro/test/set-html.test.js

@@ -0,0 +1,82 @@
+import { expect } from 'chai';
+import * as cheerio from 'cheerio';
+import { loadFixture } from './test-utils.js';
+
+
+describe('set:html', () => {
+	/** @type {import('./test-utils').Fixture} */
+	let fixture;
+
+	before(async () => {
+		fixture = await loadFixture({
+			root: './fixtures/set-html/',
+		});
+	});
+
+	describe('Development', () => {
+		/** @type {import('./test-utils').DevServer} */
+		let devServer;
+
+		before(async () => {
+			devServer = await fixture.startDevServer();
+			globalThis.TEST_FETCH = (fetch, url, init) => {
+				return fetch(fixture.resolveUrl(url), init);
+			};
+		});
+
+		after(async () => {
+			await devServer.stop();
+		});
+
+		it('can take a fetch()', async () => {
+			let res = await fixture.fetch('/fetch');
+			expect(res.status).to.equal(200);
+			let html = await res.text();
+			const $ = cheerio.load(html);
+			expect($('#fetched-html')).to.have.a.lengthOf(1);
+			expect($('#fetched-html').text()).to.equal('works');
+		});
+	});
+
+	describe('Build', () => {
+		before(async () => {
+			await fixture.build();
+		});
+
+		it('can take a string of HTML', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#html-inner')).to.have.a.lengthOf(1);
+		});
+
+		it('can take a Promise to a string of HTML', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#promise-html-inner')).to.have.a.lengthOf(1);
+		});
+
+		it('can take a Response to a string of HTML', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#response-html-inner')).to.have.a.lengthOf(1);
+		});
+
+		it('can take an Iterator', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#iterator-num')).to.have.a.lengthOf(5);
+		});
+
+		it('Can take an AsyncIterator', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#asynciterator-num')).to.have.a.lengthOf(5);
+		});
+
+		it('Can take a ReadableStream', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerio.load(html);
+			expect($('#readable-inner')).to.have.a.lengthOf(1);
+		});
+	});
+});

pnpm-lock.yaml

@@ -1838,6 +1838,12 @@ importers:
     dependencies:
       astro: link:../../..
 
+  packages/astro/test/fixtures/set-html:
+    specifiers:
+      astro: workspace:*
+    dependencies:
+      astro: link:../../..
+
   packages/astro/test/fixtures/slots-preact:
     specifiers:
       '@astrojs/mdx': workspace:*