diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6f0b2a570..89934f492 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -95,7 +95,7 @@ jobs: continue-on-error: true with: issue-number: ${{ github.event.issue.number }} - message: | + body: | ${{ needs.benchmark.outputs.PR-BENCH }} ${{ needs.benchmark.outputs.MAIN-BENCH }} diff --git a/benchmark/bench/render.js b/benchmark/bench/render.js new file mode 100644 index 000000000..59214b788 --- /dev/null +++ b/benchmark/bench/render.js @@ -0,0 +1,122 @@ +import fs from 'fs/promises'; +import http from 'http'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { execaCommand } from 'execa'; +import { waitUntilBusy } from 'port-authority'; +import { markdownTable } from 'markdown-table'; +import { renderFiles } from '../make-project/render-default.js'; +import { astroBin } from './_util.js'; + +const port = 4322; + +export const defaultProject = 'render-default'; + +/** @typedef {{ avg: number, stdev: number, max: number }} Stat */ + +/** + * @param {URL} projectDir + * @param {URL} outputFile + */ +export async function run(projectDir, outputFile) { + const root = fileURLToPath(projectDir); + + console.log('Building...'); + await execaCommand(`${astroBin} build`, { + cwd: root, + stdio: 'inherit', + }); + + console.log('Previewing...'); + const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, { + cwd: root, + stdio: 'inherit', + }); + + console.log('Waiting for server ready...'); + await waitUntilBusy(port, { timeout: 5000 }); + + console.log('Running benchmark...'); + const result = await benchmarkRenderTime(); + + console.log('Killing server...'); + if (!previewProcess.kill('SIGTERM')) { + console.warn('Failed to kill server process id:', previewProcess.pid); + } + + console.log('Writing results to', fileURLToPath(outputFile)); + await fs.writeFile(outputFile, JSON.stringify(result, null, 2)); + + console.log('Result preview:'); + console.log('='.repeat(10)); + console.log(`#### Render\n\n`); + console.log(printResult(result)); + console.log('='.repeat(10)); + + console.log('Done!'); +} + +async function benchmarkRenderTime() { + /** @type {Record} */ + const result = {}; + for (const fileName of Object.keys(renderFiles)) { + // Render each file 100 times and push to an array + for (let i = 0; i < 100; i++) { + const pathname = '/' + fileName.slice(0, -path.extname(fileName).length); + const renderTime = await fetchRenderTime(`http://localhost:${port}${pathname}`); + if (!result[pathname]) result[pathname] = []; + result[pathname].push(renderTime); + } + } + /** @type {Record} */ + const processedResult = {}; + for (const [pathname, times] of Object.entries(result)) { + // From the 100 results, calculate average, standard deviation, and max value + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const stdev = Math.sqrt( + times.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / times.length + ); + const max = Math.max(...times); + processedResult[pathname] = { avg, stdev, max }; + } + return processedResult; +} + +/** + * @param {Record} result + */ +function printResult(result) { + return markdownTable( + [ + ['Page', 'Avg (ms)', 'Stdev (ms)', 'Max (ms)'], + ...Object.entries(result).map(([pathname, { avg, stdev, max }]) => [ + pathname, + avg.toFixed(2), + stdev.toFixed(2), + max.toFixed(2), + ]), + ], + { + align: ['l', 'r', 'r', 'r'], + } + ); +} + +/** + * Simple fetch utility to get the render time sent by `@astrojs/timer` in plain text + * @param {string} url + * @returns {Promise} + */ +function fetchRenderTime(url) { + return new Promise((resolve, reject) => { + const req = http.request(url, (res) => { + res.setEncoding('utf8'); + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('error', (e) => reject(e)); + res.on('end', () => resolve(+data)); + }); + req.on('error', (e) => reject(e)); + req.end(); + }); +} diff --git a/benchmark/index.js b/benchmark/index.js index 6ac76759c..c05fafefa 100755 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -12,6 +12,7 @@ astro-benchmark [options] Command [empty] Run all benchmarks memory Run build memory and speed test + render Run rendering speed test server-stress Run server stress test Options @@ -24,6 +25,7 @@ Options const commandName = args._[0]; const benchmarks = { memory: () => import('./bench/memory.js'), + 'render': () => import('./bench/render.js'), 'server-stress': () => import('./bench/server-stress.js'), }; diff --git a/benchmark/make-project/_util.js b/benchmark/make-project/_util.js index c0e17965b..65c91dbf3 100644 --- a/benchmark/make-project/_util.js +++ b/benchmark/make-project/_util.js @@ -1,2 +1,12 @@ export const loremIpsum = "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."; + +export const loremIpsumHtml = loremIpsum + .replace(/Lorem/g, 'Lorem') + .replace(/Ipsum/g, 'Ipsum') + .replace(/dummy/g, 'dummy'); + +export const loremIpsumMd = loremIpsum + .replace(/Lorem/g, '**Lorem**') + .replace(/Ipsum/g, '_Ipsum_') + .replace(/dummy/g, '`dummy`'); diff --git a/benchmark/make-project/render-default.js b/benchmark/make-project/render-default.js new file mode 100644 index 000000000..9dfe88609 --- /dev/null +++ b/benchmark/make-project/render-default.js @@ -0,0 +1,87 @@ +import fs from 'fs/promises'; +import { loremIpsumHtml, loremIpsumMd } from './_util.js'; + +// Map of files to be generated and tested for rendering. +// Ideally each content should be similar for comparison. +export const renderFiles = { + 'astro.astro': `\ +--- +const className = "text-red-500"; +const style = { color: "red" }; +const items = Array.from({ length: 1000 }, (_, i) => i); +--- + + + + My Site + + +

List

+
    + {items.map((item) => ( +
  • {item}
  • + ))} +
+ ${Array.from({ length: 1000 }) + .map(() => `

${loremIpsumHtml}

`) + .join('\n')} + +`, + 'md.md': `\ +# List + +${Array.from({ length: 1000 }, (_, i) => i) + .map((v) => `- ${v}`) + .join('\n')} + +${Array.from({ length: 1000 }) + .map(() => loremIpsumMd) + .join('\n\n')} +`, + 'mdx.mdx': `\ +export const className = "text-red-500"; +export const style = { color: "red" }; +export const items = Array.from({ length: 1000 }, (_, i) => i); + +# List + +
    + {items.map((item) => ( +
  • {item}
  • + ))} +
+ +${Array.from({ length: 1000 }) + .map(() => loremIpsumMd) + .join('\n\n')} +`, +}; + +/** + * @param {URL} projectDir + */ +export async function run(projectDir) { + await fs.rm(projectDir, { recursive: true, force: true }); + await fs.mkdir(new URL('./src/pages', projectDir), { recursive: true }); + + await Promise.all( + Object.entries(renderFiles).map(([name, content]) => { + return fs.writeFile(new URL(`./src/pages/${name}`, projectDir), content, 'utf-8'); + }) + ); + + await fs.writeFile( + new URL('./astro.config.js', projectDir), + `\ +import { defineConfig } from 'astro/config'; +import timer from '@astrojs/timer'; +import mdx from '@astrojs/mdx'; + +export default defineConfig({ + integrations: [mdx()], + output: 'server', + adapter: timer(), +});`, + 'utf-8' + ); +} diff --git a/benchmark/package.json b/benchmark/package.json index 34b486e97..85ba91e87 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -7,7 +7,9 @@ "astro-benchmark": "./index.js" }, "dependencies": { + "@astrojs/mdx": "workspace:*", "@astrojs/node": "workspace:*", + "@astrojs/timer": "workspace:*", "astro": "workspace:*", "autocannon": "^7.10.0", "execa": "^6.1.0", diff --git a/packages/integrations/timer/README.md b/packages/integrations/timer/README.md new file mode 100644 index 000000000..81124745d --- /dev/null +++ b/packages/integrations/timer/README.md @@ -0,0 +1,3 @@ +# @astrojs/timer + +Like `@astrojs/node`, but returns the rendered time in milliseconds for the page instead of the page content itself. This is used for internal benchmarks only. diff --git a/packages/integrations/timer/package.json b/packages/integrations/timer/package.json new file mode 100644 index 000000000..13203a2d5 --- /dev/null +++ b/packages/integrations/timer/package.json @@ -0,0 +1,43 @@ +{ + "name": "@astrojs/timer", + "description": "Preview server for benchmark", + "private": true, + "version": "0.0.0", + "type": "module", + "types": "./dist/index.d.ts", + "author": "withastro", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/withastro/astro.git", + "directory": "packages/integrations/timer" + }, + "keywords": [ + "withastro", + "astro-adapter" + ], + "bugs": "https://github.com/withastro/astro/issues", + "exports": { + ".": "./dist/index.js", + "./server.js": "./dist/server.js", + "./preview.js": "./dist/preview.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "astro-scripts build \"src/**/*.ts\" && tsc", + "build:ci": "astro-scripts build \"src/**/*.ts\"", + "dev": "astro-scripts dev \"src/**/*.ts\"" + }, + "dependencies": { + "@astrojs/webapi": "workspace:*", + "server-destroy": "^1.0.1" + }, + "peerDependencies": { + "astro": "workspace:^2.0.17" + }, + "devDependencies": { + "@types/server-destroy": "^1.0.1", + "astro": "workspace:*", + "astro-scripts": "workspace:*" + } +} diff --git a/packages/integrations/timer/src/index.ts b/packages/integrations/timer/src/index.ts new file mode 100644 index 000000000..feeaa2122 --- /dev/null +++ b/packages/integrations/timer/src/index.ts @@ -0,0 +1,34 @@ +import type { AstroAdapter, AstroIntegration } from 'astro'; + +export function getAdapter(): AstroAdapter { + return { + name: '@astrojs/timer', + serverEntrypoint: '@astrojs/timer/server.js', + previewEntrypoint: '@astrojs/timer/preview.js', + exports: ['handler'], + }; +} + +export default function createIntegration(): AstroIntegration { + return { + name: '@astrojs/timer', + hooks: { + 'astro:config:setup': ({ updateConfig }) => { + updateConfig({ + vite: { + ssr: { + noExternal: ['@astrojs/timer'], + }, + }, + }); + }, + 'astro:config:done': ({ setAdapter, config }) => { + setAdapter(getAdapter()); + + if (config.output === 'static') { + console.warn(`[@astrojs/timer] \`output: "server"\` is required to use this adapter.`); + } + }, + }, + }; +} diff --git a/packages/integrations/timer/src/preview.ts b/packages/integrations/timer/src/preview.ts new file mode 100644 index 000000000..1208830dd --- /dev/null +++ b/packages/integrations/timer/src/preview.ts @@ -0,0 +1,36 @@ +import type { CreatePreviewServer } from 'astro'; +import { createServer } from 'http'; +import enableDestroy from 'server-destroy'; + +const preview: CreatePreviewServer = async function ({ serverEntrypoint, host, port }) { + const ssrModule = await import(serverEntrypoint.toString()); + const ssrHandler = ssrModule.handler; + const server = createServer(ssrHandler); + server.listen(port, host); + enableDestroy(server); + + // eslint-disable-next-line no-console + console.log(`Preview server listening on http://${host}:${port}`); + + // Resolves once the server is closed + const closed = new Promise((resolve, reject) => { + server.addListener('close', resolve); + server.addListener('error', reject); + }); + + return { + host, + port, + closed() { + return closed; + }, + server, + stop: async () => { + await new Promise((resolve, reject) => { + server.destroy((err) => (err ? reject(err) : resolve(undefined))); + }); + }, + }; +}; + +export { preview as default }; diff --git a/packages/integrations/timer/src/server.ts b/packages/integrations/timer/src/server.ts new file mode 100644 index 000000000..0f609fd50 --- /dev/null +++ b/packages/integrations/timer/src/server.ts @@ -0,0 +1,21 @@ +import { polyfill } from '@astrojs/webapi'; +import type { IncomingMessage, ServerResponse } from 'http'; +import type { SSRManifest } from 'astro'; +import { NodeApp } from 'astro/app/node'; + +polyfill(globalThis, { + exclude: 'window document', +}); + +export function createExports(manifest: SSRManifest) { + const app = new NodeApp(manifest); + return { + handler: async (req: IncomingMessage, res: ServerResponse) => { + const start = performance.now(); + await app.render(req); + const end = performance.now(); + res.write(end - start + ''); + res.end(); + }, + }; +} diff --git a/packages/integrations/timer/tsconfig.json b/packages/integrations/timer/tsconfig.json new file mode 100644 index 000000000..44baf375c --- /dev/null +++ b/packages/integrations/timer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "allowJs": true, + "module": "ES2020", + "outDir": "./dist", + "target": "ES2020" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25643eafd..cebf0d4b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,7 +65,9 @@ importers: benchmark: specifiers: + '@astrojs/mdx': workspace:* '@astrojs/node': workspace:* + '@astrojs/timer': workspace:* astro: workspace:* autocannon: ^7.10.0 execa: ^6.1.0 @@ -74,7 +76,9 @@ importers: port-authority: ^2.0.1 pretty-bytes: ^6.0.0 dependencies: + '@astrojs/mdx': link:../packages/integrations/mdx '@astrojs/node': link:../packages/integrations/node + '@astrojs/timer': link:../packages/integrations/timer astro: link:../packages/astro autocannon: 7.10.0 execa: 6.1.0 @@ -3375,6 +3379,21 @@ importers: tailwindcss: 3.2.6_postcss@8.4.21 vite: 4.1.2 + packages/integrations/timer: + specifiers: + '@astrojs/webapi': workspace:* + '@types/server-destroy': ^1.0.1 + astro: workspace:* + astro-scripts: workspace:* + server-destroy: ^1.0.1 + dependencies: + '@astrojs/webapi': link:../../webapi + server-destroy: 1.0.1 + devDependencies: + '@types/server-destroy': 1.0.1 + astro: link:../../astro + astro-scripts: link:../../../scripts + packages/integrations/turbolinks: specifiers: astro: workspace:*