Add page render benchmark (#6415)

.github/workflows/benchmark.yml

@@ -95,7 +95,7 @@ jobs:
         continue-on-error: true
         with:
           issue-number: ${{ github.event.issue.number }}
-          message: |
+          body: |
             ${{ needs.benchmark.outputs.PR-BENCH }}
 
             ${{ needs.benchmark.outputs.MAIN-BENCH }}

benchmark/bench/render.js

@@ -0,0 +1,122 @@
+import fs from 'fs/promises';
+import http from 'http';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { execaCommand } from 'execa';
+import { waitUntilBusy } from 'port-authority';
+import { markdownTable } from 'markdown-table';
+import { renderFiles } from '../make-project/render-default.js';
+import { astroBin } from './_util.js';
+
+const port = 4322;
+
+export const defaultProject = 'render-default';
+
+/** @typedef {{ avg: number, stdev: number, max: number }} Stat */
+
+/**
+ * @param {URL} projectDir
+ * @param {URL} outputFile
+ */
+export async function run(projectDir, outputFile) {
+	const root = fileURLToPath(projectDir);
+
+	console.log('Building...');
+	await execaCommand(`${astroBin} build`, {
+		cwd: root,
+		stdio: 'inherit',
+	});
+
+	console.log('Previewing...');
+	const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, {
+		cwd: root,
+		stdio: 'inherit',
+	});
+
+	console.log('Waiting for server ready...');
+	await waitUntilBusy(port, { timeout: 5000 });
+
+	console.log('Running benchmark...');
+	const result = await benchmarkRenderTime();
+
+	console.log('Killing server...');
+	if (!previewProcess.kill('SIGTERM')) {
+		console.warn('Failed to kill server process id:', previewProcess.pid);
+	}
+
+	console.log('Writing results to', fileURLToPath(outputFile));
+	await fs.writeFile(outputFile, JSON.stringify(result, null, 2));
+
+	console.log('Result preview:');
+	console.log('='.repeat(10));
+	console.log(`#### Render\n\n`);
+	console.log(printResult(result));
+	console.log('='.repeat(10));
+
+	console.log('Done!');
+}
+
+async function benchmarkRenderTime() {
+	/** @type {Record<string, number[]>} */
+	const result = {};
+	for (const fileName of Object.keys(renderFiles)) {
+		// Render each file 100 times and push to an array
+		for (let i = 0; i < 100; i++) {
+			const pathname = '/' + fileName.slice(0, -path.extname(fileName).length);
+			const renderTime = await fetchRenderTime(`http://localhost:${port}${pathname}`);
+			if (!result[pathname]) result[pathname] = [];
+			result[pathname].push(renderTime);
+		}
+	}
+	/** @type {Record<string, Stat>} */
+	const processedResult = {};
+	for (const [pathname, times] of Object.entries(result)) {
+		// From the 100 results, calculate average, standard deviation, and max value
+		const avg = times.reduce((a, b) => a + b, 0) / times.length;
+		const stdev = Math.sqrt(
+			times.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / times.length
+		);
+		const max = Math.max(...times);
+		processedResult[pathname] = { avg, stdev, max };
+	}
+	return processedResult;
+}
+
+/**
+ * @param {Record<string, Stat>} result
+ */
+function printResult(result) {
+	return markdownTable(
+		[
+			['Page', 'Avg (ms)', 'Stdev (ms)', 'Max (ms)'],
+			...Object.entries(result).map(([pathname, { avg, stdev, max }]) => [
+				pathname,
+				avg.toFixed(2),
+				stdev.toFixed(2),
+				max.toFixed(2),
+			]),
+		],
+		{
+			align: ['l', 'r', 'r', 'r'],
+		}
+	);
+}
+
+/**
+ * Simple fetch utility to get the render time sent by `@astrojs/timer` in plain text
+ * @param {string} url
+ * @returns {Promise<number>}
+ */
+function fetchRenderTime(url) {
+	return new Promise((resolve, reject) => {
+		const req = http.request(url, (res) => {
+			res.setEncoding('utf8');
+			let data = '';
+			res.on('data', (chunk) => (data += chunk));
+			res.on('error', (e) => reject(e));
+			res.on('end', () => resolve(+data));
+		});
+		req.on('error', (e) => reject(e));
+		req.end();
+	});
+}

benchmark/index.js

@@ -12,6 +12,7 @@ astro-benchmark <command> [options]
 Command
   [empty]         Run all benchmarks
   memory          Run build memory and speed test
+  render          Run rendering speed test
   server-stress   Run server stress test
 
 Options
@@ -24,6 +25,7 @@ Options
 const commandName = args._[0];
 const benchmarks = {
 	memory: () => import('./bench/memory.js'),
+	'render': () => import('./bench/render.js'),
 	'server-stress': () => import('./bench/server-stress.js'),
 };
 

benchmark/make-project/_util.js

@@ -1,2 +1,12 @@
 export const loremIpsum =
 	"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
+
+export const loremIpsumHtml = loremIpsum
+	.replace(/Lorem/g, '<strong>Lorem</strong>')
+	.replace(/Ipsum/g, '<em>Ipsum</em>')
+	.replace(/dummy/g, '<span>dummy</span>');
+
+export const loremIpsumMd = loremIpsum
+	.replace(/Lorem/g, '**Lorem**')
+	.replace(/Ipsum/g, '_Ipsum_')
+	.replace(/dummy/g, '`dummy`');

benchmark/make-project/render-default.js

@@ -0,0 +1,87 @@
+import fs from 'fs/promises';
+import { loremIpsumHtml, loremIpsumMd } from './_util.js';
+
+// Map of files to be generated and tested for rendering.
+// Ideally each content should be similar for comparison.
+export const renderFiles = {
+	'astro.astro': `\
+---
+const className = "text-red-500";
+const style = { color: "red" };
+const items = Array.from({ length: 1000 }, (_, i) => i);
+---
+
+<html>
+  <head>
+    <title>My Site</title>
+  </head>
+  <body>
+    <h1 class={className + ' text-lg'}>List</h1>
+    <ul style={style}>
+      {items.map((item) => (
+        <li class={className}>{item}</li>
+      ))}
+    </ul>
+    ${Array.from({ length: 1000 })
+			.map(() => `<p>${loremIpsumHtml}</p>`)
+			.join('\n')}
+  </body>
+</html>`,
+	'md.md': `\
+# List
+
+${Array.from({ length: 1000 }, (_, i) => i)
+	.map((v) => `- ${v}`)
+	.join('\n')}
+
+${Array.from({ length: 1000 })
+	.map(() => loremIpsumMd)
+	.join('\n\n')}
+`,
+	'mdx.mdx': `\
+export const className = "text-red-500";
+export const style = { color: "red" };
+export const items = Array.from({ length: 1000 }, (_, i) => i);
+
+# List
+
+<ul style={style}>
+  {items.map((item) => (
+    <li class={className}>{item}</li>
+  ))}
+</ul>
+
+${Array.from({ length: 1000 })
+	.map(() => loremIpsumMd)
+	.join('\n\n')}
+`,
+};
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages', projectDir), { recursive: true });
+
+	await Promise.all(
+		Object.entries(renderFiles).map(([name, content]) => {
+			return fs.writeFile(new URL(`./src/pages/${name}`, projectDir), content, 'utf-8');
+		})
+	);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+import timer from '@astrojs/timer';
+import mdx from '@astrojs/mdx';
+
+export default defineConfig({
+  integrations: [mdx()],
+	output: 'server',
+	adapter: timer(),
+});`,
+		'utf-8'
+	);
+}

benchmark/package.json

@@ -7,7 +7,9 @@
     "astro-benchmark": "./index.js"
   },
   "dependencies": {
+    "@astrojs/mdx": "workspace:*",
     "@astrojs/node": "workspace:*",
+    "@astrojs/timer": "workspace:*",
     "astro": "workspace:*",
     "autocannon": "^7.10.0",
     "execa": "^6.1.0",

packages/integrations/timer/README.md

@@ -0,0 +1,3 @@
+# @astrojs/timer
+
+Like `@astrojs/node`, but returns the rendered time in milliseconds for the page instead of the page content itself. This is used for internal benchmarks only.

packages/integrations/timer/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@astrojs/timer",
+  "description": "Preview server for benchmark",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "author": "withastro",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/withastro/astro.git",
+    "directory": "packages/integrations/timer"
+  },
+  "keywords": [
+    "withastro",
+    "astro-adapter"
+  ],
+  "bugs": "https://github.com/withastro/astro/issues",
+  "exports": {
+    ".": "./dist/index.js",
+    "./server.js": "./dist/server.js",
+    "./preview.js": "./dist/preview.js",
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
+    "dev": "astro-scripts dev \"src/**/*.ts\""
+  },
+  "dependencies": {
+    "@astrojs/webapi": "workspace:*",
+    "server-destroy": "^1.0.1"
+  },
+  "peerDependencies": {
+    "astro": "workspace:^2.0.17"
+  },
+  "devDependencies": {
+    "@types/server-destroy": "^1.0.1",
+    "astro": "workspace:*",
+    "astro-scripts": "workspace:*"
+  }
+}

packages/integrations/timer/src/index.ts

@@ -0,0 +1,34 @@
+import type { AstroAdapter, AstroIntegration } from 'astro';
+
+export function getAdapter(): AstroAdapter {
+	return {
+		name: '@astrojs/timer',
+		serverEntrypoint: '@astrojs/timer/server.js',
+		previewEntrypoint: '@astrojs/timer/preview.js',
+		exports: ['handler'],
+	};
+}
+
+export default function createIntegration(): AstroIntegration {
+	return {
+		name: '@astrojs/timer',
+		hooks: {
+			'astro:config:setup': ({ updateConfig }) => {
+				updateConfig({
+					vite: {
+						ssr: {
+							noExternal: ['@astrojs/timer'],
+						},
+					},
+				});
+			},
+			'astro:config:done': ({ setAdapter, config }) => {
+				setAdapter(getAdapter());
+
+				if (config.output === 'static') {
+					console.warn(`[@astrojs/timer] \`output: "server"\` is required to use this adapter.`);
+				}
+			},
+		},
+	};
+}

packages/integrations/timer/src/preview.ts

@@ -0,0 +1,36 @@
+import type { CreatePreviewServer } from 'astro';
+import { createServer } from 'http';
+import enableDestroy from 'server-destroy';
+
+const preview: CreatePreviewServer = async function ({ serverEntrypoint, host, port }) {
+	const ssrModule = await import(serverEntrypoint.toString());
+	const ssrHandler = ssrModule.handler;
+	const server = createServer(ssrHandler);
+	server.listen(port, host);
+	enableDestroy(server);
+
+	// eslint-disable-next-line no-console
+	console.log(`Preview server listening on http://${host}:${port}`);
+
+	// Resolves once the server is closed
+	const closed = new Promise<void>((resolve, reject) => {
+		server.addListener('close', resolve);
+		server.addListener('error', reject);
+	});
+
+	return {
+		host,
+		port,
+		closed() {
+			return closed;
+		},
+		server,
+		stop: async () => {
+			await new Promise((resolve, reject) => {
+				server.destroy((err) => (err ? reject(err) : resolve(undefined)));
+			});
+		},
+	};
+};
+
+export { preview as default };

packages/integrations/timer/src/server.ts

@@ -0,0 +1,21 @@
+import { polyfill } from '@astrojs/webapi';
+import type { IncomingMessage, ServerResponse } from 'http';
+import type { SSRManifest } from 'astro';
+import { NodeApp } from 'astro/app/node';
+
+polyfill(globalThis, {
+	exclude: 'window document',
+});
+
+export function createExports(manifest: SSRManifest) {
+	const app = new NodeApp(manifest);
+	return {
+		handler: async (req: IncomingMessage, res: ServerResponse) => {
+			const start = performance.now();
+			await app.render(req);
+			const end = performance.now();
+			res.write(end - start + '');
+			res.end();
+		},
+	};
+}

packages/integrations/timer/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "allowJs": true,
+    "module": "ES2020",
+    "outDir": "./dist",
+    "target": "ES2020"
+  }
+}

pnpm-lock.yaml

@@ -65,7 +65,9 @@ importers:
 
   benchmark:
     specifiers:
+      '@astrojs/mdx': workspace:*
       '@astrojs/node': workspace:*
+      '@astrojs/timer': workspace:*
       astro: workspace:*
       autocannon: ^7.10.0
       execa: ^6.1.0
@@ -74,7 +76,9 @@ importers:
       port-authority: ^2.0.1
       pretty-bytes: ^6.0.0
     dependencies:
+      '@astrojs/mdx': link:../packages/integrations/mdx
       '@astrojs/node': link:../packages/integrations/node
+      '@astrojs/timer': link:../packages/integrations/timer
       astro: link:../packages/astro
       autocannon: 7.10.0
       execa: 6.1.0
@@ -3375,6 +3379,21 @@ importers:
       tailwindcss: 3.2.6_postcss@8.4.21
       vite: 4.1.2
 
+  packages/integrations/timer:
+    specifiers:
+      '@astrojs/webapi': workspace:*
+      '@types/server-destroy': ^1.0.1
+      astro: workspace:*
+      astro-scripts: workspace:*
+      server-destroy: ^1.0.1
+    dependencies:
+      '@astrojs/webapi': link:../../webapi
+      server-destroy: 1.0.1
+    devDependencies:
+      '@types/server-destroy': 1.0.1
+      astro: link:../../astro
+      astro-scripts: link:../../../scripts
+
   packages/integrations/turbolinks:
     specifiers:
       astro: workspace:*