From 9d236c9417317f318639ce8d9c1cb24a34b4b0f5 Mon Sep 17 00:00:00 2001
From: Bjorn Lu <bjornlu.dev@gmail.com>
Date: Sat, 4 Mar 2023 13:16:53 +0800
Subject: [PATCH] Remove unused benchmarks (#6396)

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
---
 CONTRIBUTING.md                               | 19 +++--
 packages/astro/package.json                   |  1 -
 packages/astro/test/benchmark/benchmark.js    | 71 ----------------
 .../astro/test/benchmark/build-cached.json    |  4 -
 .../astro/test/benchmark/build-uncached.json  |  4 -
 packages/astro/test/benchmark/build.bench.js  | 85 -------------------
 .../test/benchmark/dev-server-cached.json     |  4 -
 .../test/benchmark/dev-server-uncached.json   |  4 -
 packages/astro/test/benchmark/dev.bench.js    | 63 --------------
 .../test/benchmark/simple/astro.config.mjs    |  7 --
 .../astro/test/benchmark/simple/package.json  | 12 ---
 .../astro/test/benchmark/simple/server.mjs    | 19 -----
 .../simple/src/components/Layout.astro        |  8 --
 .../benchmark/simple/src/pages/index.astro    | 79 -----------------
 scripts/memory/index.js                       | 85 -------------------
 scripts/memory/mk.js                          | 11 ---
 scripts/memory/project/.gitkeep               |  0
 scripts/memory/project/src/pages/.gitkeep     |  0
 18 files changed, 11 insertions(+), 465 deletions(-)
 delete mode 100644 packages/astro/test/benchmark/benchmark.js
 delete mode 100644 packages/astro/test/benchmark/build-cached.json
 delete mode 100644 packages/astro/test/benchmark/build-uncached.json
 delete mode 100644 packages/astro/test/benchmark/build.bench.js
 delete mode 100644 packages/astro/test/benchmark/dev-server-cached.json
 delete mode 100644 packages/astro/test/benchmark/dev-server-uncached.json
 delete mode 100644 packages/astro/test/benchmark/dev.bench.js
 delete mode 100644 packages/astro/test/benchmark/simple/astro.config.mjs
 delete mode 100644 packages/astro/test/benchmark/simple/package.json
 delete mode 100644 packages/astro/test/benchmark/simple/server.mjs
 delete mode 100644 packages/astro/test/benchmark/simple/src/components/Layout.astro
 delete mode 100644 packages/astro/test/benchmark/simple/src/pages/index.astro
 delete mode 100644 scripts/memory/index.js
 delete mode 100644 scripts/memory/mk.js
 delete mode 100644 scripts/memory/project/.gitkeep
 delete mode 100644 scripts/memory/project/src/pages/.gitkeep

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6af7c9859..3c0af74cb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -121,22 +121,25 @@ pnpm exec changeset
 
 ### Running benchmarks
 
-We have benchmarks to keep performance under control. You can run these by running (from the project root):
+We have benchmarks to keep performance under control. They are located in the `benchmarks` directory, and it exposes a CLI you can use to run them.
+
+You can run all available benchmarks sequentially by running (from the project root):
 
 ```shell
-pnpm run benchmark --filter astro
+pnpm run benchmark
 ```
 
-Which will fail if the performance has regressed by **10%** or more.
-
-To update the times cd into the `packages/astro` folder and run the following:
+To run a specific benchmark only, you can add the name of the benchmark after the command:
 
 ```shell
-node test/benchmark/build.bench.js --save
-node test/benchmark/dev.bench.js --save
+pnpm run benchmark memory
 ```
 
-Which will update the build and dev benchmarks.
+Use `pnpm run benchmark --help` to see all available options.
+
+To run these benchmarks in a PR on GitHub instead of using the CLI, you can comment `!bench`. The benchmarks will run on both the PR branch and the `main` branch, and the results will be posted as a new comment.
+
+To run only a specific benchmark on CI, add its name after the command in your comment, for example, `!bench memory`.
 
 ## Code Structure
 
diff --git a/packages/astro/package.json b/packages/astro/package.json
index f901ddf78..2809376d6 100644
--- a/packages/astro/package.json
+++ b/packages/astro/package.json
@@ -90,7 +90,6 @@
     "build:ci": "pnpm run prebuild && astro-scripts build \"src/**/*.ts\"",
     "dev": "astro-scripts dev --prebuild \"src/runtime/server/astro-island.ts\" --prebuild \"src/runtime/client/{idle,load,media,only,visible}.ts\" \"src/**/*.ts\"",
     "postbuild": "astro-scripts copy \"src/**/*.astro\"",
-    "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
     "test:unit": "mocha --exit --timeout 30000 ./test/units/**/*.test.js",
     "test:unit:match": "mocha --exit --timeout 30000 ./test/units/**/*.test.js -g",
     "test": "pnpm run test:unit && mocha --exit --timeout 20000 --ignore **/lit-element.test.js && mocha --timeout 20000 **/lit-element.test.js",
diff --git a/packages/astro/test/benchmark/benchmark.js b/packages/astro/test/benchmark/benchmark.js
deleted file mode 100644
index d8b1c72fc..000000000
--- a/packages/astro/test/benchmark/benchmark.js
+++ /dev/null
@@ -1,71 +0,0 @@
-import { promises as fsPromises, existsSync } from 'fs';
-import { performance } from 'perf_hooks';
-import * as assert from 'uvu/assert';
-
-const MUST_BE_AT_LEAST_PERC_OF = 90;
-
-const shouldSave = process.argv.includes('--save');
-
-export class Benchmark {
-	constructor(options) {
-		this.options = options;
-		this.setup = options.setup || Function.prototype;
-	}
-
-	async execute() {
-		const { run } = this.options;
-		const start = performance.now();
-		const end = await run(this.options);
-		const time = Math.floor(end - start);
-		return time;
-	}
-
-	async run() {
-		const { file } = this.options;
-
-		await this.setup();
-		const time = await this.execute();
-
-		if (existsSync(file)) {
-			const raw = await fsPromises.readFile(file, 'utf-8');
-			const data = JSON.parse(raw);
-			if (Math.floor((data.time / time) * 100) > MUST_BE_AT_LEAST_PERC_OF) {
-				this.withinPreviousRuns = true;
-			} else {
-				this.withinPreviousRuns = false;
-			}
-		}
-		this.time = time;
-	}
-
-	report() {
-		const { name } = this.options;
-		console.log(name, 'took', this.time, 'ms');
-	}
-
-	check() {
-		assert.ok(this.withinPreviousRuns !== false, `${this.options.name} ran too slowly`);
-	}
-
-	async save() {
-		const { file, name } = this.options;
-		const data = JSON.stringify(
-			{
-				name,
-				time: this.time,
-			},
-			null,
-			'  '
-		);
-		await fsPromises.writeFile(file, data, 'utf-8');
-	}
-
-	async test() {
-		await this.run();
-		if (shouldSave) {
-			await this.save();
-		}
-		this.report();
-		this.check();
-	}
-}
diff --git a/packages/astro/test/benchmark/build-cached.json b/packages/astro/test/benchmark/build-cached.json
deleted file mode 100644
index d0ec5912d..000000000
--- a/packages/astro/test/benchmark/build-cached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "name": "Snowpack Example Build Cached",
-  "time": 8496
-}
diff --git a/packages/astro/test/benchmark/build-uncached.json b/packages/astro/test/benchmark/build-uncached.json
deleted file mode 100644
index 101621c72..000000000
--- a/packages/astro/test/benchmark/build-uncached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "name": "Snowpack Example Build Uncached",
-  "time": 16200
-}
diff --git a/packages/astro/test/benchmark/build.bench.js b/packages/astro/test/benchmark/build.bench.js
deleted file mode 100644
index a140082a3..000000000
--- a/packages/astro/test/benchmark/build.bench.js
+++ /dev/null
@@ -1,85 +0,0 @@
-/** @todo migrate these to use the independent docs repository at https://github.com/withastro/docs */
-
-import { fileURLToPath } from 'url';
-import { performance } from 'perf_hooks';
-import { build as astroBuild } from '#astro/build';
-import { loadConfig } from '#astro/config';
-import { Benchmark } from './benchmark.js';
-import { deleteAsync } from 'del';
-import { Writable } from 'stream';
-import { format as utilFormat } from 'util';
-
-const snowpackExampleRoot = new URL('../../../../docs/', import.meta.url);
-
-export const errorWritable = new Writable({
-	objectMode: true,
-	write(event, _, callback) {
-		let dest = process.stderr;
-		dest.write(utilFormat(...event.args));
-		dest.write('\n');
-
-		callback();
-	},
-});
-
-let build;
-async function setupBuild() {
-	const astroConfig = await loadConfig(fileURLToPath(snowpackExampleRoot));
-
-	const logging = {
-		level: 'error',
-		dest: errorWritable,
-	};
-
-	build = () => astroBuild(astroConfig, logging);
-}
-
-async function runBuild() {
-	await build();
-	return performance.now();
-}
-
-const benchmarks = [
-	new Benchmark({
-		name: 'Snowpack Example Build Uncached',
-		root: snowpackExampleRoot,
-		file: new URL('./build-uncached.json', import.meta.url),
-		async setup() {
-			process.chdir(new URL('../../../../', import.meta.url).pathname);
-			const spcache = new URL('../../node_modules/.cache/', import.meta.url);
-			await Promise.all([deleteAsync(spcache.pathname, { force: true }), setupBuild()]);
-		},
-		run: runBuild,
-	}),
-	new Benchmark({
-		name: 'Snowpack Example Build Cached',
-		root: snowpackExampleRoot,
-		file: new URL('./build-cached.json', import.meta.url),
-		async setup() {
-			process.chdir(new URL('../../../../', import.meta.url).pathname);
-			await setupBuild();
-			await this.execute();
-		},
-		run: runBuild,
-	}),
-	/*new Benchmark({
-    name: 'Snowpack Example Dev Server Cached',
-    root: snowpackExampleRoot,
-    file: new URL('./dev-server-cached.json', import.meta.url),
-    async setup() {
-      // Execute once to make sure Snowpack is cached.
-      await this.execute();
-    }
-  })*/
-];
-
-async function run() {
-	for (const b of benchmarks) {
-		await b.test();
-	}
-}
-
-run().catch((err) => {
-	console.error(err);
-	process.exit(1);
-});
diff --git a/packages/astro/test/benchmark/dev-server-cached.json b/packages/astro/test/benchmark/dev-server-cached.json
deleted file mode 100644
index 16eb8a471..000000000
--- a/packages/astro/test/benchmark/dev-server-cached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "name": "Snowpack Example Dev Server Cached",
-  "time": 1229
-}
diff --git a/packages/astro/test/benchmark/dev-server-uncached.json b/packages/astro/test/benchmark/dev-server-uncached.json
deleted file mode 100644
index c0d72650d..000000000
--- a/packages/astro/test/benchmark/dev-server-uncached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "name": "Snowpack Example Dev Server Uncached",
-  "time": 3913
-}
diff --git a/packages/astro/test/benchmark/dev.bench.js b/packages/astro/test/benchmark/dev.bench.js
deleted file mode 100644
index 134992634..000000000
--- a/packages/astro/test/benchmark/dev.bench.js
+++ /dev/null
@@ -1,63 +0,0 @@
-/** @todo migrate these to use the independent docs repository at https://github.com/withastro/docs */
-
-import { performance } from 'perf_hooks';
-import { Benchmark } from './benchmark.js';
-import { runDevServer } from '../helpers.js';
-import { deleteAsync } from 'del';
-
-const docsExampleRoot = new URL('../../../../docs/', import.meta.url);
-
-async function runToStarted(root) {
-	const args = [];
-	const process = runDevServer(root, args);
-
-	let started = null;
-	process.stdout.setEncoding('utf8');
-	for await (const chunk of process.stdout) {
-		if (/Server started/.test(chunk)) {
-			started = performance.now();
-			break;
-		}
-	}
-
-	process.kill();
-	return started;
-}
-
-const benchmarks = [
-	new Benchmark({
-		name: 'Docs Site Example Dev Server Uncached',
-		root: docsExampleRoot,
-		file: new URL('./dev-server-uncached.json', import.meta.url),
-		async setup() {
-			const spcache = new URL('../../node_modules/.cache/', import.meta.url);
-			await deleteAsync(spcache.pathname);
-		},
-		run({ root }) {
-			return runToStarted(root);
-		},
-	}),
-	new Benchmark({
-		name: 'Docs Site Example Dev Server Cached',
-		root: docsExampleRoot,
-		file: new URL('./dev-server-cached.json', import.meta.url),
-		async setup() {
-			// Execute once to make sure Docs Site is cached.
-			await this.execute();
-		},
-		run({ root }) {
-			return runToStarted(root);
-		},
-	}),
-];
-
-async function run() {
-	for (const b of benchmarks) {
-		await b.test();
-	}
-}
-
-run().catch((err) => {
-	console.error(err);
-	process.exit(1);
-});
diff --git a/packages/astro/test/benchmark/simple/astro.config.mjs b/packages/astro/test/benchmark/simple/astro.config.mjs
deleted file mode 100644
index 16a331be3..000000000
--- a/packages/astro/test/benchmark/simple/astro.config.mjs
+++ /dev/null
@@ -1,7 +0,0 @@
-import { defineConfig } from 'astro/config';
-import nodejs from '@astrojs/node';
-
-export default defineConfig({
-	output: 'server',
-	adapter: nodejs({ mode: 'middleware' }),
-});
diff --git a/packages/astro/test/benchmark/simple/package.json b/packages/astro/test/benchmark/simple/package.json
deleted file mode 100644
index bec29ab55..000000000
--- a/packages/astro/test/benchmark/simple/package.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "name": "@benchmark/simple",
-  "scripts": {
-    "start": "node server.mjs",
-    "build": "astro build",
-    "dev": "astro dev"
-  },
-  "dependencies": {
-    "astro": "workspace:*",
-    "@astrojs/node": "workspace:*"
-  }
-}
diff --git a/packages/astro/test/benchmark/simple/server.mjs b/packages/astro/test/benchmark/simple/server.mjs
deleted file mode 100644
index 3fde151e1..000000000
--- a/packages/astro/test/benchmark/simple/server.mjs
+++ /dev/null
@@ -1,19 +0,0 @@
-import http from 'http';
-import { handler } from './dist/server/entry.mjs';
-
-const listener = (req, res) => {
-	handler(req, res, (err) => {
-		if (err) {
-			res.writeHead(500);
-			res.end(err.toString());
-		} else {
-			res.writeHead(404);
-			res.end('Not found');
-		}
-	});
-};
-
-const server = http.createServer(listener);
-server.listen(3002);
-// eslint-disable-next-line no-console
-console.log(`Listening at http://localhost:3002`);
diff --git a/packages/astro/test/benchmark/simple/src/components/Layout.astro b/packages/astro/test/benchmark/simple/src/components/Layout.astro
deleted file mode 100644
index 7224ba686..000000000
--- a/packages/astro/test/benchmark/simple/src/components/Layout.astro
+++ /dev/null
@@ -1,8 +0,0 @@
-<html>
-	<head>
-		<title>{Astro.props.title}</title>
-	</head>
-	<body>
-		<slot />
-	</body>
-</html>
diff --git a/packages/astro/test/benchmark/simple/src/pages/index.astro b/packages/astro/test/benchmark/simple/src/pages/index.astro
deleted file mode 100644
index 80f7e2eaa..000000000
--- a/packages/astro/test/benchmark/simple/src/pages/index.astro
+++ /dev/null
@@ -1,79 +0,0 @@
----
-const content =
-	"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
----
-
-<html lang="en">
-	<head>
-		<meta charset="utf-8" />
-		<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
-		<meta name="viewport" content="width=device-width" />
-		<meta name="generator" content={Astro.generator} />
-		<title>Astro</title>
-	</head>
-	<body>
-		<h1>Astro</h1>
-		<div>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-			<p>{content}</p>
-		</div>
-	</body>
-</html>
diff --git a/scripts/memory/index.js b/scripts/memory/index.js
deleted file mode 100644
index 8eaffb56b..000000000
--- a/scripts/memory/index.js
+++ /dev/null
@@ -1,85 +0,0 @@
-import { fileURLToPath } from 'url';
-import v8 from 'v8';
-import dev from '../../packages/astro/dist/core/dev/index.js';
-import { openConfig } from '../../packages/astro/dist/core/config.js';
-import { nodeLogDestination } from '../../packages/astro/dist/core/logger/node.js';
-import prettyBytes from 'pretty-bytes';
-
-if (!global.gc) {
-	console.error('ERROR: Node must be run with --expose-gc');
-	process.exit(1);
-}
-
-const isCI = process.argv.includes('--ci');
-
-/** URL directory containing the entire project. */
-const projDir = new URL('./project/', import.meta.url);
-
-let { astroConfig: config } = await openConfig({
-	cwd: fileURLToPath(projDir),
-	logging: {
-		dest: nodeLogDestination,
-		level: 'error',
-	},
-	cmd: 'dev',
-});
-
-const telemetry = {
-	record() {
-		return Promise.resolve();
-	},
-};
-const server = await dev(config, { logging: { level: 'error' }, telemetry });
-
-// Prime the server so initial memory is created
-await fetch(`http://localhost:3000/page-0`);
-
-async function run() {
-	for (let i = 0; i < 100; i++) {
-		let path = `/page-${i}`;
-		await fetch(`http://localhost:3000${path}`);
-	}
-}
-
-global.gc();
-const startSize = v8.getHeapStatistics().used_heap_size;
-
-// HUMAN mode: Runs forever. Optimized for accurate results on each snapshot Slower than CI.
-if (!isCI) {
-	console.log(
-		`Greetings, human. This test will run forever. Run with the "--ci" flag to finish with a result.`
-	);
-	let i = 1;
-	while (i++) {
-		await run();
-		global.gc();
-		const checkpoint = v8.getHeapStatistics().used_heap_size;
-		console.log(`Snapshot ${String(i).padStart(3, '0')}: ${(checkpoint / startSize) * 100}%`);
-	}
-}
-
-// CI mode: Runs 100 times. Optimized for speed with an accurate final result.
-for (let i = 0; i < 100; i++) {
-	await run();
-	const checkpoint = v8.getHeapStatistics().used_heap_size;
-	console.log(`Estimate ${String(i).padStart(3, '0')}/100: ${(checkpoint / startSize) * 100}%`);
-}
-
-console.log(`Test complete. Running final garbage collection...`);
-global.gc();
-const endSize = v8.getHeapStatistics().used_heap_size;
-
-// If the trailing average is higher than the median, see if it's more than 5% higher
-let percentage = endSize / startSize;
-const TEST_THRESHOLD = 1.5;
-const isPass = percentage < TEST_THRESHOLD;
-console.log(``);
-console.log(`Result: ${isPass ? 'PASS' : 'FAIL'} (${percentage * 100}%)`);
-console.log(
-	`Memory usage began at ${prettyBytes(startSize)} and finished at ${prettyBytes(endSize)}.`
-);
-console.log(`The threshold for a probable memory leak is ${TEST_THRESHOLD * 100}%`);
-console.log(``);
-console.log(`Exiting...`);
-await server.stop();
-process.exit(isPass ? 0 : 1);
diff --git a/scripts/memory/mk.js b/scripts/memory/mk.js
deleted file mode 100644
index ae14e1191..000000000
--- a/scripts/memory/mk.js
+++ /dev/null
@@ -1,11 +0,0 @@
-import fs from 'fs';
-
-const pages = new URL('./project/src/pages/', import.meta.url);
-
-for (let i = 0; i < 100; i++) {
-	let content = `---
-const i = ${i};
----
-<span>{i}</span>`;
-	await fs.promises.writeFile(new URL(`./page-${i}.astro`, pages), content, 'utf-8');
-}
diff --git a/scripts/memory/project/.gitkeep b/scripts/memory/project/.gitkeep
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scripts/memory/project/src/pages/.gitkeep b/scripts/memory/project/src/pages/.gitkeep
deleted file mode 100644
index e69de29bb..000000000