more efficient _routes.json for cloudflare integration (#7846)

* more efficient _routes.json for cloudflare integration

* added tests

* updated pnpm-lock.yaml

* added changeset

* cleaned up test

* fix: convert window path separators

* updated docs

* handle more cases

* Apply suggestions from code review

Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>

* incorporate feedback from code review

* used other pnpm version

* better fallback for empty include array

* adjust test case to changed fallback for empty include array

* updated docs

---------

Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>

.changeset/breezy-frogs-learn.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/cloudflare': minor
+---
+
+More efficient \_routes.json

packages/integrations/cloudflare/README.md

@@ -106,7 +106,10 @@ Cloudflare has support for adding custom [headers](https://developers.cloudflare
 
 ### Custom `_routes.json`
 
-By default, `@astrojs/cloudflare` will generate a `_routes.json` file that lists all files from your `dist/` folder and redirects from the `_redirects` file in the `exclude` array. This will enable Cloudflare to serve files and process static redirects without a function invocation. Creating a custom `_routes.json` will override this automatic optimization and, if not configured manually, cause function invocations that will count against the request limits of your Cloudflare plan.
+By default, `@astrojs/cloudflare` will generate a `_routes.json` file with `include` and `exclude` rules based on your applications's dynamic and static routes.
+This will enable Cloudflare to serve files and process static redirects without a function invocation. Creating a custom `_routes.json` will override this automatic optimization and, if not configured manually, cause function invocations that will count against the request limits of your Cloudflare plan.
+
+See [Cloudflare's documentation](https://developers.cloudflare.com/pages/platform/functions/routing/#create-a-_routesjson-file) for more details.
 
 ## Troubleshooting
 

packages/integrations/cloudflare/src/index.ts

@@ -39,6 +39,11 @@ const SHIM = `globalThis.process = {
 
 const SERVER_BUILD_FOLDER = '/$server_build/';
 
+/**
+ * These route types are candiates for being part of the `_routes.json` `include` array.
+ */
+const potentialFunctionRouteTypes = ['endpoint', 'page'];
+
 export default function createIntegration(args?: Options): AstroIntegration {
 	let _config: AstroConfig;
 	let _buildConfig: BuildConfig;
@@ -233,6 +238,32 @@ export default function createIntegration(args?: Options): AstroIntegration {
 				// cloudflare to handle static files and support _redirects configuration
 				// (without calling the function)
 				if (!routesExists) {
+					const functionEndpoints = routes
+						// Certain route types, when their prerender option is set to false, a run on the server as function invocations
+						.filter((route) => potentialFunctionRouteTypes.includes(route.type) && !route.prerender)
+						.map((route) => {
+							const includePattern =
+								'/' +
+								route.segments
+									.flat()
+									.map((segment) => (segment.dynamic ? '*' : segment.content))
+									.join('/');
+
+							const regexp = new RegExp(
+								'^\\/' +
+									route.segments
+										.flat()
+										.map((segment) => (segment.dynamic ? '(.*)' : segment.content))
+										.join('\\/') +
+									'$'
+							);
+
+							return {
+								includePattern,
+								regexp,
+							};
+						});
+
 					const staticPathList: Array<string> = (
 						await glob(`${fileURLToPath(_buildConfig.client)}/**/*`, {
 							cwd: fileURLToPath(_config.outDir),
@@ -240,7 +271,7 @@ export default function createIntegration(args?: Options): AstroIntegration {
 						})
 					)
 						.filter((file: string) => cloudflareSpecialFiles.indexOf(file) < 0)
-						.map((file: string) => `/${file}`);
+						.map((file: string) => `/${file.replace(/\\/g, '/')}`);
 
 					for (let page of pages) {
 						let pagePath = prependForwardSlash(page.pathname);
@@ -303,13 +334,41 @@ export default function createIntegration(args?: Options): AstroIntegration {
 						);
 					}
 
+					staticPathList.push(...routes.filter((r) => r.type === 'redirect').map((r) => r.route));
+
+					// In order to product the shortest list of patterns, we first try to
+					// include all function endpoints, and then exclude all static paths
+					let include = deduplicatePatterns(
+						functionEndpoints.map((endpoint) => endpoint.includePattern)
+					);
+					let exclude = deduplicatePatterns(
+						staticPathList.filter((file: string) =>
+							functionEndpoints.some((endpoint) => endpoint.regexp.test(file))
+						)
+					);
+
+					// Cloudflare requires at least one include pattern:
+					// https://developers.cloudflare.com/pages/platform/functions/routing/#limits
+					// So we add a pattern that we immediately exclude again
+					if (include.length === 0) {
+						include = ['/'];
+						exclude = ['/'];
+					}
+
+					// If using only an exclude list would produce a shorter list of patterns,
+					// we use that instead
+					if (include.length + exclude.length > staticPathList.length) {
+						include = ['/*'];
+						exclude = deduplicatePatterns(staticPathList);
+					}
+
 					await fs.promises.writeFile(
 						new URL('./_routes.json', _config.outDir),
 						JSON.stringify(
 							{
 								version: 1,
-								include: ['/*'],
+								include,
-								exclude: staticPathList,
+								exclude,
 							},
 							null,
 							2
@@ -324,3 +383,28 @@ export default function createIntegration(args?: Options): AstroIntegration {
 function prependForwardSlash(path: string) {
 	return path[0] === '/' ? path : '/' + path;
 }
+
+/**
+ * Remove duplicates and redundant patterns from an `include` or `exclude` list.
+ * Otherwise Cloudflare will throw an error on deployment. Plus, it saves more entries.
+ * E.g. `['/foo/*', '/foo/*', '/foo/bar'] => ['/foo/*']`
+ * @param patterns a list of `include` or `exclude` patterns
+ * @returns a deduplicated list of patterns
+ */
+function deduplicatePatterns(patterns: string[]) {
+	const openPatterns: RegExp[] = [];
+
+	return [...new Set(patterns)]
+		.sort((a, b) => a.length - b.length)
+		.filter((pattern) => {
+			if (openPatterns.some((p) => p.test(pattern))) {
+				return false;
+			}
+
+			if (pattern.endsWith('*')) {
+				openPatterns.push(new RegExp(`^${pattern.replace(/(\*\/)*\*$/g, '.*')}`));
+			}
+
+			return true;
+		});
+}

packages/integrations/cloudflare/test/fixtures/routesJson/astro.config.mjs

@@ -0,0 +1,11 @@
+import { defineConfig } from 'astro/config';
+import cloudflare from '@astrojs/cloudflare';
+
+export default defineConfig({
+	adapter: cloudflare({ mode: 'directory' }),
+	output: 'hybrid',
+	redirects: {
+		'/a/redirect': '/',
+	},
+	srcDir: process.env.SRC 
+});

packages/integrations/cloudflare/test/fixtures/routesJson/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@test/astro-cloudflare-routes-json",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "@astrojs/cloudflare": "workspace:*",
+    "astro": "workspace:*"
+  }
+}

packages/integrations/cloudflare/test/fixtures/routesJson/src/dynamicOnly/pages/another.astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/dynamicOnly/pages/index.astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/pages/a/[...rest].astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/pages/a/[id].astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/pages/a/endpoint.ts

@@ -0,0 +1 @@
+export const prerender = false;

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/pages/a/index.astro

@@ -0,0 +1 @@
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/pages/b/index.html

@@ -0,0 +1 @@
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/mixed/public/public.txt

@@ -0,0 +1 @@
+ok

packages/integrations/cloudflare/test/fixtures/routesJson/src/staticOnly/pages/index.astro

@@ -0,0 +1 @@
+ok

packages/integrations/cloudflare/test/prerender.test.js

@@ -18,13 +18,14 @@ describe('Prerendering', () => {
 		fixture.clean();
 	});
 
-	it('includes prerendered routes in the routes.json config', async () => {
+	it('includes non prerendered routes in the routes.json config', async () => {
-		const foundRoutes = JSON.parse(await fixture.readFile('/_routes.json')).exclude.map((r) =>
+		const foundRoutes = JSON.parse(await fixture.readFile('/_routes.json'));
-			r.replace(/\\/g, '/')
-		);
-		const expectedExcludedRoutes = ['/_worker.js', '/one/index.html', '/one/'];
 
-		expect(foundRoutes.every((element) => expectedExcludedRoutes.includes(element))).to.be.true;
+		expect(foundRoutes).to.deep.equal({
+			version: 1,
+			include: ['/'],
+			exclude: [],
+		});
 	});
 });
 
@@ -45,12 +46,13 @@ describe('Hybrid rendering', () => {
 		delete process.env.PRERENDER;
 	});
 
-	it('includes prerendered routes in the routes.json config', async () => {
+	it('includes non prerendered routes in the routes.json config', async () => {
-		const foundRoutes = JSON.parse(await fixture.readFile('/_routes.json')).exclude.map((r) =>
+		const foundRoutes = JSON.parse(await fixture.readFile('/_routes.json'));
-			r.replace(/\\/g, '/')
-		);
-		const expectedExcludedRoutes = ['/_worker.js', '/index.html', '/'];
 
-		expect(foundRoutes.every((element) => expectedExcludedRoutes.includes(element))).to.be.true;
+		expect(foundRoutes).to.deep.equal({
+			version: 1,
+			include: ['/one'],
+			exclude: [],
+		});
 	});
 });

packages/integrations/cloudflare/test/routesJson.js

@@ -0,0 +1,78 @@
+import { expect } from 'chai';
+import { loadFixture } from './test-utils.js';
+
+/** @type {import('./test-utils.js').Fixture} */
+describe('_routes.json generation', () => {
+	after(() => {
+		delete process.env.SRC;
+	});
+
+	describe('of both functions and static files', () => {
+		let fixture;
+
+		before(async () => {
+			process.env.SRC = './src/mixed';
+			fixture = await loadFixture({
+				root: './fixtures/routesJson/',
+			});
+			await fixture.build();
+		});
+
+		it('creates `include` for functions and `exclude` for static files where needed', async () => {
+			const _routesJson = await fixture.readFile('/_routes.json');
+			const routes = JSON.parse(_routesJson);
+
+			expect(routes).to.deep.equal({
+				version: 1,
+				include: ['/a/*'],
+				exclude: ['/a/', '/a/redirect', '/a/index.html'],
+			});
+		});
+	});
+
+	describe('of only functions', () => {
+		let fixture;
+
+		before(async () => {
+			process.env.SRC = './src/dynamicOnly';
+			fixture = await loadFixture({
+				root: './fixtures/routesJson/',
+			});
+			await fixture.build();
+		});
+
+		it('creates a wildcard `include` and `exclude` only for the redirect', async () => {
+			const _routesJson = await fixture.readFile('/_routes.json');
+			const routes = JSON.parse(_routesJson);
+
+			expect(routes).to.deep.equal({
+				version: 1,
+				include: ['/*'],
+				exclude: ['/a/redirect'],
+			});
+		});
+	});
+
+	describe('of only static files', () => {
+		let fixture;
+
+		before(async () => {
+			process.env.SRC = './src/staticOnly';
+			fixture = await loadFixture({
+				root: './fixtures/routesJson/',
+			});
+			await fixture.build();
+		});
+
+		it('create only one `include` and `exclude` that are supposed to match nothing', async () => {
+			const _routesJson = await fixture.readFile('/_routes.json');
+			const routes = JSON.parse(_routesJson);
+
+			expect(routes).to.deep.equal({
+				version: 1,
+				include: ['/'],
+				exclude: ['/'],
+			});
+		});
+	});
+});

pnpm-lock.yaml

@@ -3706,6 +3706,15 @@ importers:
         specifier: workspace:*
         version: link:../../../../../astro
 
+  packages/integrations/cloudflare/test/fixtures/routesJson:
+    dependencies:
+      '@astrojs/cloudflare':
+        specifier: workspace:*
+        version: link:../../..
+      astro:
+        specifier: workspace:*
+        version: link:../../../../../astro
+
   packages/integrations/cloudflare/test/fixtures/split:
     dependencies:
       '@astrojs/cloudflare':